E2e/join (sealerio#397)

* add join test and optimize build test
fanux · Jul 7, 2021 · fb410c3 · fb410c3
1 parent dd2ecc8
commit fb410c3
Show file tree

Hide file tree

Showing 8 changed files with 123 additions and 49 deletions.
diff --git a/test/e2e_test.go b/test/e2e_test.go
@@ -18,6 +18,8 @@ import (
 	"os/exec"
 	"testing"
 
+	"github.com/alibaba/sealer/test/testhelper"
+
 	"github.com/alibaba/sealer/test/testhelper/settings"
 
 	. "github.com/onsi/ginkgo"
@@ -34,6 +36,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	Expect(err).NotTo(HaveOccurred(), output)
 	SetDefaultEventuallyTimeout(settings.DefaultWaiteTime)
 	settings.DefaultSealerBin = output
+	settings.DefaultTestEnvDir = testhelper.GetPwd()
 	return nil
 }, func(data []byte) {
 	SetDefaultEventuallyTimeout(settings.DefaultWaiteTime)

diff --git a/test/sealer_apply_test.go b/test/sealer_apply_test.go
@@ -15,6 +15,9 @@
 package test
 
 import (
+	"strconv"
+	"strings"
+
 	"github.com/alibaba/sealer/test/suites/apply"
 	"github.com/alibaba/sealer/test/testhelper"
 	"github.com/alibaba/sealer/test/testhelper/settings"
@@ -51,22 +54,18 @@ var _ = Describe("sealer apply", func() {
 				result := testhelper.GetFileDataLocally(settings.GetClusterWorkClusterfile(rawCluster.Name))
 				err = testhelper.WriteFile(tempFile, []byte(result))
 				Expect(err).NotTo(HaveOccurred())
-				usedCluster := apply.LoadClusterFileFromDisk(tempFile)
+				apply.LoadClusterFileFromDisk(tempFile)
 
 				//2,scale up cluster to 6 nodes and write to disk
-				By("start to scale up cluster")
-				usedCluster.Spec.Nodes.Count = "3"
-				usedCluster.Spec.Masters.Count = "3"
-				apply.WriteClusterFileToDisk(usedCluster, tempFile)
-				sess, err = testhelper.Start(apply.SealerApplyCmd(tempFile))
-				Expect(err).NotTo(HaveOccurred())
-				Eventually(sess, settings.MaxWaiteTime).Should(Exit(0))
-				apply.CheckNodeNumLocally(6)
+				By("Use join command to add 3master and 3node for scale up cluster in cloud mode", func() {
+					apply.SealerJoin(strconv.Itoa(2), strconv.Itoa(2))
+					apply.CheckNodeNumLocally(6)
+				})
 
 				result = testhelper.GetFileDataLocally(settings.GetClusterWorkClusterfile(rawCluster.Name))
 				err = testhelper.WriteFile(tempFile, []byte(result))
 				Expect(err).NotTo(HaveOccurred())
-				usedCluster = apply.LoadClusterFileFromDisk(tempFile)
+				usedCluster := apply.LoadClusterFileFromDisk(tempFile)
 
 				//3,scale down cluster to 4 nodes and write to disk
 				By("start to scale down cluster")
@@ -107,12 +106,17 @@ var _ = Describe("sealer apply", func() {
 				apply.SendAndApplyCluster(sshClient, tempFile)
 				apply.CheckNodeNumWithSSH(sshClient, 2)
 
-				By("start to scale up cluster")
-				usedCluster.Spec.Nodes.Count = "3"
-				usedCluster.Spec.Masters.Count = "3"
-				usedCluster = apply.CreateAliCloudInfraAndSave(usedCluster, tempFile)
-				apply.SendAndApplyCluster(sshClient, tempFile)
-				apply.CheckNodeNumWithSSH(sshClient, 6)
+				By("Use join command to add 3master and 3node for scale up cluster in baremetal mode", func() {
+					usedCluster.Spec.Nodes.Count = "3"
+					usedCluster.Spec.Masters.Count = "3"
+					usedCluster = apply.CreateAliCloudInfraAndSave(usedCluster, tempFile)
+					joinMasters := strings.Join(usedCluster.Spec.Masters.IPList[1:], ",")
+					joinNodes := strings.Join(usedCluster.Spec.Nodes.IPList[1:], ",")
+					//sealer join master and node
+					apply.SendAndJoinCluster(sshClient, tempFile, joinMasters, joinNodes)
+					//add 3 masters and 3 nodes
+					apply.CheckNodeNumWithSSH(sshClient, 6)
+				})
 
 				By("start to scale down cluster")
 				usedCluster.Spec.Nodes.Count = "1"

diff --git a/test/sealer_build_test.go b/test/sealer_build_test.go
@@ -36,14 +36,16 @@ var _ = Describe("sealer build", func() {
 		Context("build kube file", func() {
 			Context("testing the content of kube file", func() {
 				Context("testing local build scenario", func() {
-					err := os.Chdir(filepath.Join(build.GetFixtures(), build.GetLocalBuildDir()))
-					Expect(err).NotTo(HaveOccurred())
 
 					BeforeEach(func() {
 						registry.Login()
+						err := os.Chdir(filepath.Join(build.GetFixtures(), build.GetLocalBuildDir()))
+						Expect(err).NotTo(HaveOccurred())
 					})
 					AfterEach(func() {
 						registry.Logout()
+						err := os.Chdir(settings.DefaultTestEnvDir)
+						Expect(err).NotTo(HaveOccurred())
 					})
 
 					It("with all build instruct", func() {
@@ -82,11 +84,13 @@ var _ = Describe("sealer build", func() {
 				Context("testing cloud build scenario", func() {
 					BeforeEach(func() {
 						registry.Login()
-						err := os.Chdir(filepath.Join("..", build.GetCloudBuildDir()))
+						err := os.Chdir(filepath.Join(build.GetFixtures(), build.GetCloudBuildDir()))
 						Expect(err).NotTo(HaveOccurred())
 					})
 					AfterEach(func() {
 						registry.Logout()
+						err := os.Chdir(settings.DefaultTestEnvDir)
+						Expect(err).NotTo(HaveOccurred())
 					})
 
 					It("with all build instruct", func() {

diff --git a/test/sealer_images_test.go b/test/sealer_images_test.go
@@ -152,15 +152,11 @@ var _ = Describe("sealer image", func() {
 			image.DoImageOps(settings.SubCmdRmiOfSealer, settings.TestImageName)
 		})
 		It("show image metadata", func() {
-			sess, err := testhelper.Start(fmt.Sprintf("%s inspect %s", settings.DefaultSealerBin, settings.TestImageName))
-			Expect(err).NotTo(HaveOccurred())
-			Eventually(sess, settings.MaxWaiteTime).Should(Exit(0))
+			testhelper.RunCmdAndCheckResult(fmt.Sprintf("%s inspect %s", settings.DefaultSealerBin, settings.TestImageName), 0)
 		})
 
 		It("show image default Clusterfile", func() {
-			sess, err := testhelper.Start(fmt.Sprintf("%s inspect -c %s", settings.DefaultSealerBin, settings.TestImageName))
-			Expect(err).NotTo(HaveOccurred())
-			Eventually(sess, settings.MaxWaiteTime).Should(Exit(0))
+			testhelper.RunCmdAndCheckResult(fmt.Sprintf("%s inspect -c %s", settings.DefaultSealerBin, settings.TestImageName), 0)
 		})
 	})
 

diff --git a/test/sealer_run_test.go b/test/sealer_run_test.go
@@ -1,38 +1,62 @@
 package test
 
 import (
-	"fmt"
+	"strconv"
+	"strings"
 
 	"github.com/alibaba/sealer/test/suites/apply"
 	"github.com/alibaba/sealer/test/testhelper"
 	"github.com/alibaba/sealer/test/testhelper/settings"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	. "github.com/onsi/gomega/gexec"
 )
 
 var _ = Describe("sealer run", func() {
-	Context("start run", func() {
-		Context("regular scenario", func() {
-			Context("run on ali cloud", func() {
-				AfterEach(func() {
-					apply.DeleteClusterByFile(settings.GetClusterWorkClusterfile(settings.ClusterNameForRun))
-				})
-
-				It("exec sealer run", func() {
-					master := "1"
-					node := "1"
-					cmd := fmt.Sprintf("%s run %s -m %s -n %s", settings.DefaultSealerBin, settings.ImageNameForRun, master, node)
-					sess, err := testhelper.Start(cmd)
-					Expect(err).NotTo(HaveOccurred())
-					Eventually(sess, settings.MaxWaiteTime).Should(Exit(0))
-					apply.CheckNodeNumLocally(2)
-				})
-
-			})
+	Context("run on ali cloud", func() {
+		AfterEach(func() {
+			apply.DeleteClusterByFile(settings.GetClusterWorkClusterfile(settings.ClusterNameForRun))
+		})
 
+		It("exec sealer run", func() {
+			master := strconv.Itoa(1)
+			node := strconv.Itoa(1)
+			apply.SealerRun(master, node, "")
+			apply.CheckNodeNumLocally(2)
 		})
 
 	})
 
+	Context("run on bareMetal", func() {
+		var tempFile string
+		BeforeEach(func() {
+			tempFile = testhelper.CreateTempFile()
+		})
+
+		AfterEach(func() {
+			testhelper.RemoveTempFile(tempFile)
+		})
+
+		It("bareMetal run", func() {
+			rawCluster := apply.LoadClusterFileFromDisk(apply.GetRawClusterFilePath())
+			By("start to prepare infra")
+			usedCluster := apply.CreateAliCloudInfraAndSave(rawCluster, tempFile)
+			//defer to delete cluster
+			defer func() {
+				apply.CleanUpAliCloudInfra(usedCluster)
+			}()
+			sshClient := testhelper.NewSSHClientByCluster(usedCluster)
+			Eventually(func() bool {
+				err := sshClient.SSH.Copy(sshClient.RemoteHostIP, settings.DefaultSealerBin, settings.DefaultSealerBin)
+				return err == nil
+			}, settings.MaxWaiteTime).Should(BeTrue())
+
+			By("start to init cluster", func() {
+				masters := strings.Join(usedCluster.Spec.Masters.IPList, ",")
+				nodes := strings.Join(usedCluster.Spec.Nodes.IPList, ",")
+				apply.SendAndRunCluster(sshClient, tempFile, masters, nodes, usedCluster.Spec.SSH.Passwd)
+				apply.CheckNodeNumWithSSH(sshClient, 2)
+			})
+
+		})
+	})
 })
diff --git a/test/suites/apply/apply.go b/test/suites/apply/apply.go
@@ -32,7 +32,7 @@ import (
 )
 
 func getFixtures() string {
-	pwd := testhelper.GetPwd()
+	pwd := settings.DefaultTestEnvDir
 	return filepath.Join(pwd, "suites", "apply", "fixtures")
 }
 
@@ -67,6 +67,37 @@ func SealerApplyCmd(clusterFile string) string {
 	return fmt.Sprintf("%s apply -f %s", settings.DefaultSealerBin, clusterFile)
 }
 
+func SealerRunCmd(masters, nodes, passwd string) string {
+	if masters != "" {
+		masters = fmt.Sprintf("-m %s", masters)
+	}
+	if nodes != "" {
+		nodes = fmt.Sprintf("-n %s", nodes)
+	}
+	if passwd != "" {
+		passwd = fmt.Sprintf("-p %s", passwd)
+	}
+	return fmt.Sprintf("%s run %s %s %s %s", settings.DefaultSealerBin, settings.TestImageName, masters, nodes, passwd)
+}
+
+func SealerRun(masters, nodes, passwd string) {
+	testhelper.RunCmdAndCheckResult(SealerRunCmd(masters, nodes, passwd), 0)
+}
+
+func SealerJoinCmd(masters, nodes string) string {
+	if masters != "" {
+		masters = fmt.Sprintf("-m %s", masters)
+	}
+	if nodes != "" {
+		nodes = fmt.Sprintf("-n %s", nodes)
+	}
+	return fmt.Sprintf("%s join %s %s", settings.DefaultSealerBin, masters, nodes)
+}
+
+func SealerJoin(masters, nodes string) {
+	testhelper.RunCmdAndCheckResult(SealerJoinCmd(masters, nodes), 0)
+}
+
 func CreateAliCloudInfraAndSave(cluster *v1.Cluster, clusterFile string) *v1.Cluster {
 	gomega.Eventually(func() bool {
 		infraManager, err := infra.NewDefaultProvider(cluster)
@@ -86,14 +117,26 @@ func CreateAliCloudInfraAndSave(cluster *v1.Cluster, clusterFile string) *v1.Clu
 }
 
 func SendAndApplyCluster(sshClient *testhelper.SSHClient, clusterFile string) {
+	SendAndRemoteExecCluster(sshClient, clusterFile, SealerApplyCmd(clusterFile))
+}
+
+func SendAndJoinCluster(sshClient *testhelper.SSHClient, clusterFile string, joinMasters, joinNodes string) {
+	SendAndRemoteExecCluster(sshClient, clusterFile, SealerJoinCmd(joinMasters, joinNodes))
+}
+
+func SendAndRunCluster(sshClient *testhelper.SSHClient, clusterFile string, joinMasters, joinNodes, passwd string) {
+	SendAndRemoteExecCluster(sshClient, clusterFile, SealerRunCmd(joinMasters, joinNodes, passwd))
+}
+
+func SendAndRemoteExecCluster(sshClient *testhelper.SSHClient, clusterFile string, remoteCmd string) {
 	// send tmp cluster file to remote server and run apply cmd
 	gomega.Eventually(func() bool {
 		err := sshClient.SSH.Copy(sshClient.RemoteHostIP, clusterFile, clusterFile)
 		return err == nil
 	}, settings.MaxWaiteTime).Should(gomega.BeTrue())
 
 	gomega.Eventually(func() bool {
-		err := sshClient.SSH.CmdAsync(sshClient.RemoteHostIP, SealerApplyCmd(clusterFile))
+		err := sshClient.SSH.CmdAsync(sshClient.RemoteHostIP, remoteCmd)
 		return err == nil
 	}, settings.MaxWaiteTime).Should(gomega.BeTrue())
 }

diff --git a/test/suites/build/build.go b/test/suites/build/build.go
@@ -20,14 +20,13 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/alibaba/sealer/test/testhelper"
 	"github.com/alibaba/sealer/test/testhelper/settings"
 	"github.com/alibaba/sealer/utils"
 	"github.com/onsi/gomega"
 )
 
 func GetFixtures() string {
-	return filepath.Join(testhelper.GetPwd(), "suites", "build", "fixtures")
+	return filepath.Join(settings.DefaultTestEnvDir, "suites", "build", "fixtures")
 }
 
 func GetLocalBuildDir() string {

diff --git a/test/testhelper/settings/common.go b/test/testhelper/settings/common.go
@@ -64,6 +64,7 @@ var (
 	MaxWaiteTime           time.Duration
 	DefaultWaiteTime       time.Duration
 	DefaultSealerBin       = ""
+	DefaultTestEnvDir      = ""
 	RegistryURL            = os.Getenv("REGISTRY_URL")
 	RegistryUsername       = os.Getenv("REGISTRY_USERNAME")
 	RegistryPasswd         = os.Getenv("REGISTRY_PASSWORD")