From 5982792e46a452d92b18ebc1a26e5e615c7e8e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=AD=E5=BC=88?= Date: Wed, 10 Nov 2021 20:19:45 +0800 Subject: [PATCH 1/2] add cluster v2 --- types/api/v1/cluster_types.go | 1 + types/api/v2/cluster_types.go | 77 +++++++++++++++++ types/api/v2/groupversion_info.go | 36 ++++++++ types/api/v2/zz_generated.deepcopy.go | 114 ++++++++++++++++++++++++++ 4 files changed, 228 insertions(+) create mode 100644 types/api/v2/cluster_types.go create mode 100644 types/api/v2/groupversion_info.go create mode 100644 types/api/v2/zz_generated.deepcopy.go diff --git a/types/api/v1/cluster_types.go b/types/api/v1/cluster_types.go index 1bd37fc612c..2f214ffe0f9 100644 --- a/types/api/v1/cluster_types.go +++ b/types/api/v1/cluster_types.go @@ -23,6 +23,7 @@ type SSH struct { Passwd string `json:"passwd,omitempty"` Pk string `json:"pk,omitempty"` PkPasswd string `json:"pkPasswd,omitempty"` + Port string `json:"port,omitempty"` } type Network struct { diff --git a/types/api/v2/cluster_types.go b/types/api/v2/cluster_types.go new file mode 100644 index 00000000000..90e6f685980 --- /dev/null +++ b/types/api/v2/cluster_types.go @@ -0,0 +1,77 @@ +/* +Copyright 2021 alibaba. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1 "github.com/alibaba/sealer/types/api/v1" +) + +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// ClusterSpec defines the desired state of Cluster +type ClusterSpec struct { + // desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // Foo is an example field of Cluster. Edit Cluster_types.go to remove/update + Image string `json:"image,omitempty"` + Env map[string]string `json:"env,omitempty"` + Hosts []Hosts `json:"hosts,omitempty"` + SSH v1.SSH `json:"ssh,omitempty"` +} + +type Hosts struct { + IP string `json:"ip,omitempty"` + Roles []string `json:"roles,omitempty"` + // SSH config overwrite + SSH v1.SSH `json:"ssh,omitempty"` + // env overwrite + Env map[string]string `json:"env,omitempty"` +} + +// ClusterStatus defines the observed state of Cluster +type ClusterStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// Cluster is the Schema for the clusters API +type Cluster struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ClusterSpec `json:"spec,omitempty"` + Status ClusterStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ClusterList contains a list of Cluster +type ClusterList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Cluster `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Cluster{}, &ClusterList{}) +} diff --git a/types/api/v2/groupversion_info.go b/types/api/v2/groupversion_info.go new file mode 100644 index 00000000000..a3cd4cb46ff --- /dev/null +++ b/types/api/v2/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2021 alibaba. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v2 contains API Schema definitions for the cluster v2 API group +// +kubebuilder:object:generate=true +// +groupName=cluster.sealer.cool +package v2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "cluster.sealer.cool", Version: "v2"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/types/api/v2/zz_generated.deepcopy.go b/types/api/v2/zz_generated.deepcopy.go new file mode 100644 index 00000000000..58ff9927e45 --- /dev/null +++ b/types/api/v2/zz_generated.deepcopy.go @@ -0,0 +1,114 @@ +// +build !ignore_autogenerated + +/* +Copyright 2021 alibaba. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v2 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Cluster) DeepCopyInto(out *Cluster) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cluster. +func (in *Cluster) DeepCopy() *Cluster { + if in == nil { + return nil + } + out := new(Cluster) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Cluster) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterList) DeepCopyInto(out *ClusterList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Cluster, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterList. +func (in *ClusterList) DeepCopy() *ClusterList { + if in == nil { + return nil + } + out := new(ClusterList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpec. +func (in *ClusterSpec) DeepCopy() *ClusterSpec { + if in == nil { + return nil + } + out := new(ClusterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterStatus) DeepCopyInto(out *ClusterStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterStatus. +func (in *ClusterStatus) DeepCopy() *ClusterStatus { + if in == nil { + return nil + } + out := new(ClusterStatus) + in.DeepCopyInto(out) + return out +} From 1b24876f26d0aec30c8616ccced0e98b1e07c358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=AD=E5=BC=88?= Date: Thu, 11 Nov 2021 16:43:07 +0800 Subject: [PATCH 2/2] update clusterfile v2 design --- docs/design/clusterfile-v2.md | 368 ++++++++++++++++++++++++++++++++++ types/api/v2/cluster_types.go | 6 +- 2 files changed, 371 insertions(+), 3 deletions(-) create mode 100644 docs/design/clusterfile-v2.md diff --git a/docs/design/clusterfile-v2.md b/docs/design/clusterfile-v2.md new file mode 100644 index 00000000000..0fdaf04312c --- /dev/null +++ b/docs/design/clusterfile-v2.md @@ -0,0 +1,368 @@ +# Clusterfile v2 design + +## Motivations + +Clusterfile v1 not match some requirement. + +* Different node has different ssh config like passwd. +* Not clean, confused about what argument should put into Clusterfile. +* Coupling the infra config and cluster config. + +## Proposal + +* Delete provider field +* Add env field +* Modify hosts field, add ssh and env rewrite +* Delete all kubeadm config + +```yaml +apiVersion: sealer.cloud/v2 +kind: Cluster +metadata: + name: my-cluster +spec: + image: kubernetes:v1.19.8 + env: + key: value + ssh: + passwd: + pk: xxx + pkPasswd: xxx + user: root + port: 2222 + hosts: + - ips: [192.168.0.2] + roles: [master] # add role field to specify the node role + env: # rewrite some nodes has different env config + etcd-dir: /data/etcd + ssh: # rewrite ssh config if some node has different passwd... + user: xxx + passwd: xxx + port: 2222 + - ips: [192.168.0.3] + roles: [node,db] +``` + +## Use cases + +### Apply a simple cluster by default + +3 masters and a node, its so clearly and simple, cool + +```yaml +apiVersion: sealer.cloud/v2 +kind: Cluster +metadata: + name: default-kubernetes-cluster +spec: + image: kubernetes:v1.19.8 + ssh: + passwd: xxx + hosts: + - ips: [192.168.0.2,192.168.0.3,192.168.0.4] + roles: [master] + - ips: 192.168.0.3 + roles: [node] +``` + +### Overwite ssh config (for example password,and port) + +```yaml +apiVersion: sealer.cloud/v2 +kind: Cluster +metadata: + name: default-kubernetes-cluster +spec: + image: kubernetes:v1.19.8 + ssh: + passwd: xxx + port: 2222 + hosts: + - ips: 192.168.0.2 + roles: [master] + ssh: + passwd: yyy + port: 22 + - ips: [192.168.0.3,192.168.0.4] + roles: [master] + - ips: [192.168.0.3] + roles: [node] +``` + +### How to define your own kubeadm config + +The better way is add kubeadm config directly into Clusterfile, of course every CloudImage has it default config: +You can only define part of those configs, sealer will merge then into default config. + +```yaml +apiVersion: kubeadm.k8s.io/v1beta2 +kind: InitConfiguration +localAPIEndpoint: + advertiseAddress: 192.168.2.110 + bindPort: 6443 +nodeRegistration: + criSocket: /var/run/dockershim.sock + +--- +apiVersion: kubeadm.k8s.io/v1beta2 +kind: ClusterConfiguration +kubernetesVersion: v1.19.8 +controlPlaneEndpoint: "apiserver.cluster.local:6443" +imageRepository: sea.hub:5000/library +networking: + # dnsDomain: cluster.local + podSubnet: 100.64.0.0/10 + serviceSubnet: 10.96.0.0/22 +apiServer: + certSANs: + - 127.0.0.1 + - apiserver.cluster.local + - 192.168.2.110 + - aliyun-inc.com + - 10.0.0.2 + - 10.103.97.2 + extraArgs: + etcd-servers: https://192.168.2.110:2379 + feature-gates: TTLAfterFinished=true,EphemeralContainers=true + audit-policy-file: "/etc/kubernetes/audit-policy.yml" + audit-log-path: "/var/log/kubernetes/audit.log" + audit-log-format: json + audit-log-maxbackup: '"10"' + audit-log-maxsize: '"100"' + audit-log-maxage: '"7"' + enable-aggregator-routing: '"true"' + extraVolumes: + - name: "audit" + hostPath: "/etc/kubernetes" + mountPath: "/etc/kubernetes" + pathType: DirectoryOrCreate + - name: "audit-log" + hostPath: "/var/log/kubernetes" + mountPath: "/var/log/kubernetes" + pathType: DirectoryOrCreate + - name: localtime + hostPath: /etc/localtime + mountPath: /etc/localtime + readOnly: true + pathType: File +controllerManager: + extraArgs: + feature-gates: TTLAfterFinished=true,EphemeralContainers=true + experimental-cluster-signing-duration: 876000h + extraVolumes: + - hostPath: /etc/localtime + mountPath: /etc/localtime + name: localtime + readOnly: true + pathType: File +scheduler: + extraArgs: + feature-gates: TTLAfterFinished=true,EphemeralContainers=true + extraVolumes: + - hostPath: /etc/localtime + mountPath: /etc/localtime + name: localtime + readOnly: true + pathType: File +etcd: + local: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2381 + +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +mode: "ipvs" +ipvs: + excludeCIDRs: + - "10.103.97.2/32" + +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 2m0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt +authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 5m0s + cacheUnauthorizedTTL: 30s +cgroupDriver: +cgroupsPerQOS: true +clusterDomain: cluster.local +configMapAndSecretChangeDetectionStrategy: Watch +containerLogMaxFiles: 5 +containerLogMaxSize: 10Mi +contentType: application/vnd.kubernetes.protobuf +cpuCFSQuota: true +cpuCFSQuotaPeriod: 100ms +cpuManagerPolicy: none +cpuManagerReconcilePeriod: 10s +enableControllerAttachDetach: true +enableDebuggingHandlers: true +enforceNodeAllocatable: +- pods +eventBurst: 10 +eventRecordQPS: 5 +evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% +evictionPressureTransitionPeriod: 5m0s +failSwapOn: true +fileCheckFrequency: 20s +hairpinMode: promiscuous-bridge +healthzBindAddress: 127.0.0.1 +healthzPort: 10248 +httpCheckFrequency: 20s +imageGCHighThresholdPercent: 85 +imageGCLowThresholdPercent: 80 +imageMinimumGCAge: 2m0s +iptablesDropBit: 15 +iptablesMasqueradeBit: 14 +kubeAPIBurst: 10 +kubeAPIQPS: 5 +makeIPTablesUtilChains: true +maxOpenFiles: 1000000 +maxPods: 110 +nodeLeaseDurationSeconds: 40 +nodeStatusReportFrequency: 10s +nodeStatusUpdateFrequency: 10s +oomScoreAdj: -999 +podPidsLimit: -1 +port: 10250 +registryBurst: 10 +registryPullQPS: 5 +rotateCertificates: true +runtimeRequestTimeout: 2m0s +serializeImagePulls: true +staticPodPath: /etc/kubernetes/manifests +streamingConnectionIdleTimeout: 4h0m0s +syncFrequency: 1m0s +volumeStatsAggPeriod: 1m0s +``` + +### Using Kubeconfig to overwrite kubeadm configs + +If you don't want to care about so much Kubeadm configs, you can use `KubeConfig` object to overwrite(json patch merge) some fields. + +```yaml +apiVersion: sealer.cloud/v2 +kind: KubeConfig +metadata: + name: default-kubernetes-config +spec: + localAPIEndpoint: + advertiseAddress: 192.168.2.110 + bindPort: 6443 + nodeRegistration: + criSocket: /var/run/dockershim.sock + kubernetesVersion: v1.19.8 + controlPlaneEndpoint: "apiserver.cluster.local:6443" + imageRepository: sea.hub:5000/library + networking: + podSubnet: 100.64.0.0/10 + serviceSubnet: 10.96.0.0/22 + apiServer: + certSANs: + - sealer.cloud + - 127.0.0.1 + clusterDomain: cluster.local +``` + +### Using ENV in configs and script + +Using ENV in configs or yaml files [check this](https://github.com/alibaba/sealer/blob/main/docs/design/global-config.md#global-configuration) + +```yaml +apiVersion: sealer.cloud/v2 +kind: Cluster +metadata: + name: my-cluster +spec: + image: kubernetes:v1.19.8 + env: + docker-dir: /var/lib/docker + hosts: + - ips: [192.168.0.2] + roles: [master] # add role field to specify the node role + env: # overwrite some nodes has different env config + docker-dir: /data/docker + - ips: [192.168.0.3] + roles: [node] +``` + +Using ENV in init.sh script: + +```shell script +#!/bin/bash +echo $docker-dir +``` + +When sealer run the script will set ENV like this: `docker-dir=/data/docker sh init.sh` +In the case, master ENV is `/data/docker`, node ENV is by default `/var/lib/docker` + +### How to using cloud infra + +If you using public cloud, you needn't to config the ip field in Cluster Object. +The infra Object will tell sealer to apply resource from public cloud, then render the ip list to Cluster Object. + +```yaml +apiVersion: sealer.cloud/v2 +kind: Cluster +metadata: + name: default-kubernetes-cluster +spec: + image: kubernetes:v1.19.8 +--- +apiVersion: sealer.cloud/v2 +kind: Infra +metadata: + name: alicloud +spec: + provider: ALI_CLOUD + ssh: + passwd: xxx + port: 2222 + hosts: + - count: 3 + role: [master] + cpu: 4 + memory: 4 + systemDisk: 100 + dataDisk: [100,200] + - count: 3 + role: [node] + cpu: 4 + memory: 4 + systemDisk: 100 + dataDisk: [100, 200] +``` + +After `sealer apply -f Clusterfile`, The cluster object will update: + +```yaml +apiVersion: sealer.cloud/v2 +kind: Cluster +metadata: + name: default-kubernetes-cluster +spec: + image: kubernetes:v1.19.8 + ssh: + passwd: xxx + port: 2222 + - ips: [192.168.0.3] + roles: [master] +... +``` + +### Env render support + +[Env render](https://github.com/alibaba/sealer/blob/main/docs/design/global-config.md#global-configuration) \ No newline at end of file diff --git a/types/api/v2/cluster_types.go b/types/api/v2/cluster_types.go index 90e6f685980..5ba2f07a51b 100644 --- a/types/api/v2/cluster_types.go +++ b/types/api/v2/cluster_types.go @@ -37,11 +37,11 @@ type ClusterSpec struct { } type Hosts struct { - IP string `json:"ip,omitempty"` + IPS []string `json:"ips,omitempty"` Roles []string `json:"roles,omitempty"` - // SSH config overwrite + //overwrite SSH config SSH v1.SSH `json:"ssh,omitempty"` - // env overwrite + //overwrite env Env map[string]string `json:"env,omitempty"` }