clusterfile v2 runtime (#882)

* clusterfile v2 upgrade runtime, add init module * add runtime v2 join master
sealerio · Nov 25, 2021 · 592f5c1 · 592f5c1
1 parent a65d5da
commit 592f5c1
Show file tree

Hide file tree

Showing 23 changed files with 1,924 additions and 318 deletions.
diff --git a/common/common.go b/common/common.go
@@ -20,6 +20,11 @@ import (
 	"github.com/mitchellh/go-homedir"
 )
 
+const (
+	MASTER = "master"
+	NODE   = "node"
+)
+
 const (
 	FROMCOMMAND        = "FROM"
 	COPYCOMMAND        = "COPY"

diff --git a/docs/design/clusterfile-v2.md b/docs/design/clusterfile-v2.md
@@ -61,7 +61,7 @@ spec:
   hosts:
   - ips: [192.168.0.2,192.168.0.3,192.168.0.4]
     roles: [master]
-  - ips: 192.168.0.3
+  - ips: [192.168.0.3]
     roles: [node]
 ```
 
@@ -78,7 +78,7 @@ spec:
     passwd: xxx
     port: 2222
   hosts:
-  - ips: 192.168.0.2
+  - ips: [192.168.0.2]
     roles: [master]
     ssh:
       passwd: yyy

diff --git a/go.mod b/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/docker/go-connections v0.4.0
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
+	github.com/imdario/mergo v0.3.12
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/moby/term v0.0.0-20201216013528-df9cb8a40635
 	github.com/olekukonko/tablewriter v0.0.4
@@ -26,7 +27,7 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.0
 	go.uber.org/zap v1.17.0
 	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
-	golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect
+	golang.org/x/net v0.0.0-20210510120150-4163338589ed
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40
 	gotest.tools v2.2.0+incompatible

diff --git a/go.sum b/go.sum
@@ -437,8 +437,9 @@ github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmK
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
+github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=

diff --git a/pkg/runtime/init.go b/pkg/runtime/init.go
@@ -13,3 +13,261 @@
 // limitations under the License.
 
 package runtime
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/alibaba/sealer/cert"
+	"github.com/alibaba/sealer/common"
+	"github.com/alibaba/sealer/logger"
+	v2 "github.com/alibaba/sealer/types/api/v2"
+	"github.com/alibaba/sealer/utils"
+)
+
+const (
+	RemoteCmdCopyStatic            = "mkdir -p %s && cp -f %s %s"
+	RemoteApplyYaml                = `echo '%s' | kubectl apply -f -`
+	RemoteCmdGetNetworkInterface   = "ls /sys/class/net"
+	RemoteCmdExistNetworkInterface = "ip addr show %s | egrep \"%s\" || true"
+	WriteKubeadmConfigCmd          = "cd %s && echo \"%s\" > kubeadm-config.yaml"
+	DefaultVIP                     = "10.103.97.2"
+	DefaultAPIserverDomain         = "apiserver.cluster.local"
+	DefaultRegistryPort            = 5000
+)
+
+func (k *KubeadmRuntime) loadMetadata() error {
+	metadata, err := LoadMetadata(filepath.Join(k.getCloudImageDir(), common.DefaultMetadataName))
+	if err != nil {
+		return err
+	}
+	k.Metadata = metadata
+	return nil
+}
+
+func (k *KubeadmRuntime) ConfigKubeadmOnMaster0() error {
+	if err := k.LoadFromClusterfile(k.Config.Clusterfile); err != nil {
+		return fmt.Errorf("failed to load kubeadm config from clusterfile: %v", err)
+	}
+	// TODO handle the kubeadm config, like kubeproxy config
+
+	bs, err := k.Merge(k.getDefaultKubeadmConfig())
+	if err != nil {
+		return fmt.Errorf("failed to merge kubeadm config: %v", err)
+	}
+
+	if err := utils.WriteFile(k.getDefaultKubeadmConfig(), bs); err != nil {
+		return fmt.Errorf("failed to dump new kubeadm config: %v", err)
+	}
+	return nil
+}
+
+func (k *KubeadmRuntime) getCertSANS() []string {
+	return k.ClusterConfiguration.APIServer.CertSANs
+}
+
+//CmdToString is in host exec cmd and replace to spilt str
+func (k *KubeadmRuntime) CmdToString(host, cmd, split string) string {
+	ssh, err := k.getHostSSHClient(host)
+	if err != nil {
+		logger.Error("failed to get host ssh client, %s %v", cmd, err)
+		return ""
+	}
+	data, err := ssh.Cmd(host, cmd)
+	if err != nil {
+		logger.Error("exec remote cmd failed, %s %v", cmd, err)
+	}
+	if data != nil {
+		str := string(data)
+		str = strings.ReplaceAll(str, "\r\n", split)
+		return str
+	}
+	return ""
+}
+
+func (k *KubeadmRuntime) getRemoteHostName(hostIP string) string {
+	hostName := k.CmdToString(hostIP, "hostname", "")
+	return strings.ToLower(hostName)
+}
+
+func (k *KubeadmRuntime) GenerateCert() error {
+	err := cert.GenerateCert(
+		k.getCertPath(),
+		k.getEtcdCertPath(),
+		k.getCertSANS(),
+		k.getMaster0IP(),
+		k.getRemoteHostName(k.getMaster0IP()),
+		k.getSvcCIDR(),
+		k.getDNSDomain(),
+	)
+	if err != nil {
+		return fmt.Errorf("generate certs failed %v", err)
+	}
+	return k.sendNewCertAndKey([]string{k.getMaster0IP()})
+}
+
+func (k *KubeadmRuntime) CreateKubeConfig() error {
+	hostname := k.getRemoteHostName(k.getMaster0IP())
+	certConfig := cert.Config{
+		Path:     k.getCertPath(),
+		BaseName: "ca",
+	}
+
+	controlPlaneEndpoint := fmt.Sprintf("https://%s:6443", k.getAPIServerDomain())
+	err := cert.CreateJoinControlPlaneKubeConfigFiles(k.getBasePath(),
+		certConfig, hostname, controlPlaneEndpoint, "kubernetes")
+	if err != nil {
+		return fmt.Errorf("generator kubeconfig failed %s", err)
+	}
+	return nil
+}
+
+func (k *KubeadmRuntime) CopyStaticFiles(nodes []string) error {
+	errCh := make(chan error, len(nodes))
+	defer close(errCh)
+
+	for _, file := range MasterStaticFiles {
+		staticFilePath := filepath.Join(k.getStaticFileDir(), file.Name)
+		cmdLinkStatic := fmt.Sprintf(RemoteCmdCopyStatic, file.DestinationDir, staticFilePath, filepath.Join(file.DestinationDir, file.Name))
+		var wg sync.WaitGroup
+		for _, host := range nodes {
+			wg.Add(1)
+			go func(host string) {
+				defer wg.Done()
+				ssh, err := k.getHostSSHClient(host)
+				if err != nil {
+					errCh <- fmt.Errorf("new ssh client failed %v", err)
+					return
+				}
+				err = ssh.CmdAsync(host, cmdLinkStatic)
+				if err != nil {
+					errCh <- fmt.Errorf("[%s] link static file failed, error:%s", host, err.Error())
+				}
+			}(host)
+		}
+		wg.Wait()
+	}
+
+	return ReadChanError(errCh)
+}
+
+//decode output to join token  hash and key
+func (k *KubeadmRuntime) decodeMaster0Output(output []byte) {
+	s0 := string(output)
+	logger.Debug("[globals]decodeOutput: %s", s0)
+	slice := strings.Split(s0, "kubeadm join")
+	slice1 := strings.Split(slice[1], "Please note")
+	logger.Info("[globals]join command is: %s", slice1[0])
+	k.decodeJoinCmd(slice1[0])
+}
+
+//  192.168.0.200:6443 --token 9vr73a.a8uxyaju799qwdjv --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866 --experimental-control-plane --certificate-key f8902e114ef118304e561c3ecd4d0b543adc226b7a07f675f56564185ffe0c07
+func (k *KubeadmRuntime) decodeJoinCmd(cmd string) {
+	logger.Debug("[globals]decodeJoinCmd: %s", cmd)
+	stringSlice := strings.Split(cmd, " ")
+
+	for i, r := range stringSlice {
+		// upstream error, delete \t, \\, \n, space.
+		r = strings.ReplaceAll(r, "\t", "")
+		r = strings.ReplaceAll(r, "\n", "")
+		r = strings.ReplaceAll(r, "\\", "")
+		r = strings.TrimSpace(r)
+		if strings.Contains(r, "--token") {
+			k.JoinToken = stringSlice[i+1]
+		}
+		if strings.Contains(r, "--discovery-token-ca-cert-hash") {
+			k.TokenCaCertHash = stringSlice[i+1]
+		}
+		if strings.Contains(r, "--certificate-key") {
+			k.CertificateKey = stringSlice[i+1][:64]
+		}
+	}
+	logger.Debug("joinToken: %v\nTokenCaCertHash: %v\nCertificateKey: %v", k.JoinToken, k.TokenCaCertHash, k.CertificateKey)
+}
+
+//InitMaster0 is
+func (k *KubeadmRuntime) InitMaster0() error {
+	ssh, err := k.getHostSSHClient(k.getMaster0IP())
+	if err != nil {
+		return fmt.Errorf("failed to get master0 ssh client, %v", err)
+	}
+
+	if err := k.SendJoinMasterKubeConfigs([]string{k.getMaster0IP()}, AdminConf, ControllerConf, SchedulerConf, KubeletConf); err != nil {
+		return err
+	}
+	cmdAddEtcHost := fmt.Sprintf(RemoteAddEtcHosts, getAPIServerHost(k.getMaster0IP(), k.getAPIServerDomain()))
+	cmdAddRegistryHosts := fmt.Sprintf(RemoteAddEtcHosts, getRegistryHost(k.getRootfs(), k.getMaster0IP()))
+	err = ssh.CmdAsync(k.getMaster0IP(), cmdAddEtcHost, cmdAddRegistryHosts)
+	if err != nil {
+		return err
+	}
+
+	logger.Info("start to init master0...")
+	cmdInit := k.Command(k.getKubeVersion(), InitMaster)
+
+	// TODO skip docker version error check for test
+	output, err := ssh.Cmd(k.getMaster0IP(), cmdInit)
+	logger.Info("%s", output)
+	if err != nil {
+		return fmt.Errorf("init master0 failed, error: %s. Please clean and reinstall", err.Error())
+	}
+	k.decodeMaster0Output(output)
+	err = ssh.CmdAsync(k.getMaster0IP(), RemoteCopyKubeConfig)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (k *KubeadmRuntime) GetKubectlAndKubeconfig() error {
+	if utils.IsFileExist(common.DefaultKubeConfigFile()) {
+		return nil
+	}
+	ssh, err := k.getHostSSHClient(k.getMaster0IP())
+	if err != nil {
+		return fmt.Errorf("failed to get master0 ssh client when get kubbectl and kubeconfig %v", err)
+	}
+
+	return GetKubectlAndKubeconfig(ssh, k.getMaster0IP())
+}
+
+func (k *KubeadmRuntime) init(cluster *v2.Cluster) error {
+	if err := k.loadMetadata(); err != nil {
+		return fmt.Errorf("failed to load metadata %v", err)
+	}
+	//config kubeadm
+	if err := k.ConfigKubeadmOnMaster0(); err != nil {
+		return fmt.Errorf("failed to config kubeadmin on master0 %v", err)
+	}
+
+	//generate certs
+	if err := k.GenerateCert(); err != nil {
+		return fmt.Errorf("failed to gernerate cert %v", err)
+	}
+
+	//create kubeConfig for master0
+	if err := k.CreateKubeConfig(); err != nil {
+		return fmt.Errorf("failed to create kubeConfig for master0 %v", err)
+	}
+
+	if err := k.CopyStaticFiles(k.getMasterIPList()); err != nil {
+		return fmt.Errorf("failed to copy static files %v", err)
+	}
+
+	if err := k.ApplyRegistry(); err != nil {
+		return fmt.Errorf("failed to encsure registry %v", err)
+	}
+
+	if err := k.InitMaster0(); err != nil {
+		return fmt.Errorf("failed to init master0 %v", err)
+	}
+
+	if err := k.GetKubectlAndKubeconfig(); err != nil {
+		return fmt.Errorf("failed to get kubectl and kubeConfig %v", err)
+	}
+
+	return nil
+}