diff --git a/Makefile b/Makefile index d9a6526700662..3acd88676729e 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,15 @@ test_e2e: go run hack/e2e.go -v --build --up --test --down .PHONY: test_e2e +# Build and run node end-to-end tests. +# +# Example: +# make test_e2e_node +test_e2e_node: + hack/e2e-node-test.sh +.PHONY: test_e2e_node + + # Remove all build artifacts. # # Example: diff --git a/hack/e2e-node-test.sh b/hack/e2e-node-test.sh new file mode 100755 index 0000000000000..3a4dbd9bb3e61 --- /dev/null +++ b/hack/e2e-node-test.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Provided for backwards compatibility +sudo -v +ginkgo "$(dirname $0)/../test/e2e_node/" -- --alsologtostderr --v 2 --node-name $(hostname) --build-services=true --start-services=true --stop-services=true + +exit $? diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index e7df21ec314dd..2166540c2a17f 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -87,6 +87,7 @@ kube::golang::test_targets() { examples/k8petstore/web-server/src github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test + test/e2e_node/e2e_node.test ) if [ -n "${KUBERNETES_CONTRIB:-}" ]; then for contrib in "${KUBERNETES_CONTRIB}"; do diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 1981750fb586c..e9c37e7dee58f 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -28,6 +28,7 @@ bench-workers bind-address bind-pods-burst bind-pods-qps +build-services cadvisor-port cert-dir certificate-authority @@ -162,6 +163,7 @@ ir-hawkular jenkins-host jenkins-jobs k8s-build-output +k8s-bin-dir keep-gogoproto km-path kube-api-burst @@ -323,6 +325,7 @@ scheduler-name schema-cache-dir secure-port serialize-image-pulls +server-start-timeout service-account-key-file service-account-lookup service-account-private-key-file @@ -344,10 +347,14 @@ skip-generated-rewrite skip-munges sort-by source-file +ssh-env ssh-keyfile +ssh-options ssh-user +start-services static-pods-config stats-port +stop-services storage-version storage-versions streaming-connection-idle-timeout diff --git a/test/e2e_node/doc.go b/test/e2e_node/doc.go index c5e79c4b2849d..b80586bcf0254 100644 --- a/test/e2e_node/doc.go +++ b/test/e2e_node/doc.go @@ -14,4 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ +// e2e_node contains e2e tests specific to the node +// TODO: rename this package e2e-node package e2e_node diff --git a/test/e2e_node/e2e_build.go b/test/e2e_node/e2e_build.go new file mode 100644 index 0000000000000..27676eb3fdd54 --- /dev/null +++ b/test/e2e_node/e2e_build.go @@ -0,0 +1,131 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/golang/glog" +) + +var k8sBinDir = flag.String("k8s-bin-dir", "", "Directory containing k8s kubelet and kube-apiserver binaries.") + +func buildGo() { + glog.Infof("Building k8s binaries...") + k8sRoot, err := getK8sRootDir() + if err != nil { + glog.Fatalf("Failed to locate kubernetes root directory %v.", err) + } + out, err := exec.Command(filepath.Join(k8sRoot, "hack/build-go.sh")).CombinedOutput() + if err != nil { + glog.Fatalf("Failed to build go packages %v. Output:\n%s", err, out) + } +} + +func getK8sBin(bin string) (string, error) { + // Use commandline specified path + if *k8sBinDir != "" { + absPath, err := filepath.Abs(*k8sBinDir) + if err != nil { + return "", err + } + if _, err := os.Stat(filepath.Join(*k8sBinDir, bin)); err != nil { + return "", fmt.Errorf("Could not find kube-apiserver under directory %s.", absPath) + } + return filepath.Join(absPath, bin), nil + } + + path, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + return "", fmt.Errorf("Could not find absolute path of directory containing the tests %s.", filepath.Dir(os.Args[0])) + } + if _, err := os.Stat(filepath.Join(path, bin)); err == nil { + return filepath.Join(path, bin), nil + } + + buildOutputDir, err := getK8sBuildOutputDir() + if err != nil { + return "", err + } + if _, err := os.Stat(filepath.Join(buildOutputDir, bin)); err == nil { + return filepath.Join(buildOutputDir, bin), nil + } + + // Give up with error + return "", fmt.Errorf("Unable to locate %s. Can be defined using --k8s-path.", bin) +} + +// TODO: Dedup / merge this with comparable utilities in e2e/util.go +func getK8sRootDir() (string, error) { + // Get the directory of the current executable + _, testExec, _, _ := runtime.Caller(0) + path := filepath.Dir(testExec) + + // Look for the kubernetes source root directory + if strings.Contains(path, "k8s.io/kubernetes") { + splitPath := strings.Split(path, "k8s.io/kubernetes") + return filepath.Join(splitPath[0], "k8s.io/kubernetes/"), nil + } + + return "", fmt.Errorf("Could not find kubernetes source root directory.") +} + +func getK8sBuildOutputDir() (string, error) { + k8sRoot, err := getK8sRootDir() + if err != nil { + return "", err + } + buildOutputDir := filepath.Join(k8sRoot, "_output/local/go/bin") + if _, err := os.Stat(buildOutputDir); err != nil { + return "", err + } + return buildOutputDir, nil +} + +func getK8sNodeTestDir() (string, error) { + k8sRoot, err := getK8sRootDir() + if err != nil { + return "", err + } + buildOutputDir := filepath.Join(k8sRoot, "test/e2e_node") + if _, err := os.Stat(buildOutputDir); err != nil { + return "", err + } + return buildOutputDir, nil +} + +func getKubeletServerBin() string { + bin, err := getK8sBin("kubelet") + if err != nil { + panic(fmt.Sprintf("Could not locate kubelet binary.")) + } + return bin +} + +func getApiServerBin() string { + bin, err := getK8sBin("kube-apiserver") + if err != nil { + panic(fmt.Sprintf("Could not locate kube-apiserver binary.")) + } + return bin +} diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index dba2be51f1ed4..58ceb4398ba04 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -15,21 +15,31 @@ limitations under the License. */ // To run tests in this suite -// Local: `$ ginkgo -- --logtostderr -v 2` -// Remote: `$ ginkgo -- --node-name --api-server-address= --kubelet-address= --logtostderr -v 2` +// NOTE: This test suite requires sudo capabilities to run the kubelet and kube-apiserver. +// $ sudo -v && ginkgo test/e2e_node/ -- --logtostderr --v 2 --node-name `hostname` --start-services package e2e_node import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - + "bytes" "flag" + "fmt" + "os/exec" + "strings" "testing" + + "github.com/golang/glog" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "Host and port of the kubelet") var apiServerAddress = flag.String("api-server-address", "http://127.0.0.1:8080", "Host and port of the api server") -var nodeName = flag.String("node-name", "127.0.0.1", "Name of the node") +var nodeName = flag.String("node-name", "", "Name of the node") +var buildServices = flag.Bool("build-services", true, "If true, build local executables") +var startServices = flag.Bool("start-services", true, "If true, start local node services") +var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tets") + +var e2es *e2eService func TestE2eNode(t *testing.T) { flag.Parse() @@ -39,8 +49,42 @@ func TestE2eNode(t *testing.T) { // Setup the kubelet on the node var _ = BeforeSuite(func() { + if *buildServices { + buildGo() + } + if *nodeName == "" { + output, err := exec.Command("hostname").CombinedOutput() + if err != nil { + glog.Fatal("Could not get node name from hostname %v. Output:\n%s", err, output) + } + *nodeName = strings.TrimSpace(fmt.Sprintf("%s", output)) + } + + if *startServices { + e2es = newE2eService() + if err := e2es.start(); err != nil { + Fail(fmt.Sprintf("Unable to start node services.\n%v", err)) + } + glog.Infof("Node services started. Running tests...") + } else { + glog.Infof("Running tests without starting services.") + } }) // Tear down the kubelet on the node var _ = AfterSuite(func() { + if e2es != nil && *startServices && *stopServices { + glog.Infof("Stopping node services...") + e2es.stop() + b := &bytes.Buffer{} + b.WriteString("-------------------------------------------------------------\n") + b.WriteString(fmt.Sprintf("kubelet output:\n%s\n", e2es.kubeletCombinedOut.String())) + b.WriteString("-------------------------------------------------------------\n") + b.WriteString(fmt.Sprintf("apiserver output:\n%s", e2es.apiServerCombinedOut.String())) + b.WriteString("-------------------------------------------------------------\n") + b.WriteString(fmt.Sprintf("etcd output:\n%s", e2es.etcdCombinedOut.String())) + b.WriteString("-------------------------------------------------------------\n") + glog.V(2).Infof(b.String()) + + } }) diff --git a/test/e2e_node/e2e_remote.go b/test/e2e_node/e2e_remote.go new file mode 100644 index 0000000000000..1de8d0600a3dd --- /dev/null +++ b/test/e2e_node/e2e_remote.go @@ -0,0 +1,182 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + "flag" + "fmt" + "io/ioutil" + "math/rand" + "os" + "os/exec" + "os/user" + "path/filepath" + "strings" + + "github.com/golang/glog" +) + +var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.") +var sshEnv = flag.String("ssh-env", "", "Use predefined ssh options for environment. Options: gce") + +var sshOptionsMap map[string]string + +const archiveName = "e2e_node_test.tar.gz" + +func init() { + usr, err := user.Current() + if err != nil { + glog.Fatal(err) + } + sshOptionsMap = map[string]string{ + "gce": fmt.Sprintf("-i %s/.ssh/google_compute_engine -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o CheckHostIP=no -o StrictHostKeyChecking=no", usr.HomeDir), + } +} + +// CreateTestArchive builds the local source and creates a tar archive e2e_node_test.tar.gz containing +// the binaries k8s required for node e2e tests +func CreateTestArchive() string { + // Build the executables + buildGo() + + // Build the e2e tests into an executable + glog.Infof("Building ginkgo k8s test binaries...") + testDir, err := getK8sNodeTestDir() + if err != nil { + glog.Fatalf("Failed to locate test/e2e_node directory %v.", err) + } + out, err := exec.Command("ginkgo", "build", testDir).CombinedOutput() + if err != nil { + glog.Fatalf("Failed to build e2e tests under %s %v. Output:\n%s", testDir, err, out) + } + ginkgoTest := filepath.Join(testDir, "e2e_node.test") + if _, err := os.Stat(ginkgoTest); err != nil { + glog.Fatalf("Failed to locate test binary %s", ginkgoTest) + } + defer os.Remove(ginkgoTest) + + // Make sure we can find the newly built binaries + buildOutputDir, err := getK8sBuildOutputDir() + if err != nil { + glog.Fatalf("Failed to locate kubernetes build output directory %v", err) + } + kubelet := filepath.Join(buildOutputDir, "kubelet") + if _, err := os.Stat(kubelet); err != nil { + glog.Fatalf("Failed to locate binary %s", kubelet) + } + apiserver := filepath.Join(buildOutputDir, "kube-apiserver") + if _, err := os.Stat(apiserver); err != nil { + glog.Fatalf("Failed to locate binary %s", apiserver) + } + + glog.Infof("Building archive...") + tardir, err := ioutil.TempDir("", "node-e2e-archive") + if err != nil { + glog.Fatalf("Failed to create temporary directory %v.", err) + } + defer os.RemoveAll(tardir) + + // Copy binaries + out, err = exec.Command("cp", ginkgoTest, filepath.Join(tardir, "e2e_node.test")).CombinedOutput() + if err != nil { + glog.Fatalf("Failed to copy e2e_node.test %v.", err) + } + out, err = exec.Command("cp", kubelet, filepath.Join(tardir, "kubelet")).CombinedOutput() + if err != nil { + glog.Fatalf("Failed to copy kubelet %v.", err) + } + out, err = exec.Command("cp", apiserver, filepath.Join(tardir, "kube-apiserver")).CombinedOutput() + if err != nil { + glog.Fatalf("Failed to copy kube-apiserver %v.", err) + } + + // Build the tar + out, err = exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput() + if err != nil { + glog.Fatalf("Failed to build tar %v. Output:\n%s", err, out) + } + + dir, err := os.Getwd() + if err != nil { + glog.Fatalf("Failed to get working directory %v.", err) + } + return filepath.Join(dir, archiveName) +} + +// RunRemote copies the archive file to a /tmp file on host, unpacks it, and runs the e2e_node.test +func RunRemote(archive string, host string) (string, error) { + // Create the temp staging directory + tmp := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31()) + _, err := runSshCommand("ssh", host, "--", "mkdir", tmp) + if err != nil { + return "", err + } + defer func() { + output, err := runSshCommand("ssh", host, "--", "rm", "-rf", tmp) + if err != nil { + glog.Errorf("Failed to cleanup tmp directory %s on host %v. Output:\n%s", tmp, err, output) + } + }() + + // Copy the archive to the staging directory + _, err = runSshCommand("scp", archive, fmt.Sprintf("%s:%s/", host, tmp)) + if err != nil { + return "", err + } + + // Kill any running node processes + cmd := getSshCommand(" ; ", + "sudo pkill kubelet", + "sudo pkill kube-apiserver", + "sudo pkill etcd") + // No need to log an error if pkill fails since pkill will fail if the commands are not running. + // If we are unable to stop existing running k8s processes, we should see messages in the kubelet/apiserver/etcd + // logs about failing to bind the required ports. + runSshCommand("ssh", host, "--", "sh", "-c", cmd) + + // Extract the archive and run the tests + cmd = getSshCommand(" && ", + fmt.Sprintf("cd %s", tmp), + fmt.Sprintf("tar -xzvf ./%s", archiveName), + "./e2e_node.test --logtostderr --v 2 --build-services=false --node-name `hostname`") + output, err := runSshCommand("ssh", host, "--", "sh", "-c", cmd) + if err != nil { + return "", err + } + + return output, nil +} + +// getSshCommand handles proper quoting so that multiple commands are executed in the same shell over ssh +func getSshCommand(sep string, args ...string) string { + return fmt.Sprintf("'%s'", strings.Join(args, sep)) +} + +// runSshCommand executes the ssh or scp command, adding the flag provided --ssh-options +func runSshCommand(cmd string, args ...string) (string, error) { + if env, found := sshOptionsMap[*sshEnv]; found { + args = append(strings.Split(env, " "), args...) + } + if *sshOptions != "" { + args = append(strings.Split(*sshOptions, " "), args...) + } + output, err := exec.Command(cmd, args...).CombinedOutput() + if err != nil { + return fmt.Sprintf("%s", output), fmt.Errorf("command %q %q failed with error: %v and output: %q", cmd, args, err, output) + } + return fmt.Sprintf("%s", output), nil +} diff --git a/test/e2e_node/e2e_service.go b/test/e2e_node/e2e_service.go new file mode 100644 index 0000000000000..56f525fb0d157 --- /dev/null +++ b/test/e2e_node/e2e_service.go @@ -0,0 +1,185 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "net/http" + "os" + "os/exec" + "strings" + "time" + + "github.com/golang/glog" +) + +var serverStartTimeout = flag.Duration("server-start-timeout", time.Second*30, "Time to wait for each server to become healthy.") + +type e2eService struct { + etcdCmd *exec.Cmd + etcdCombinedOut bytes.Buffer + etcdDataDir string + apiServerCmd *exec.Cmd + apiServerCombinedOut bytes.Buffer + kubeletCmd *exec.Cmd + kubeletCombinedOut bytes.Buffer +} + +func newE2eService() *e2eService { + return &e2eService{} +} + +func (es *e2eService) start() error { + if _, err := getK8sBin("kubelet"); err != nil { + return err + } + if _, err := getK8sBin("kube-apiserver"); err != nil { + return err + } + + cmd, err := es.startEtcd() + if err != nil { + return err + } + es.etcdCmd = cmd + + cmd, err = es.startApiServer() + if err != nil { + return err + } + es.apiServerCmd = cmd + + cmd, err = es.startKubeletServer() + if err != nil { + return err + } + es.kubeletCmd = cmd + + return nil +} + +func (es *e2eService) stop() { + if es.kubeletCmd != nil { + err := es.kubeletCmd.Process.Kill() + if err != nil { + glog.Errorf("Failed to stop kubelet.\n%v", err) + } + } + if es.apiServerCmd != nil { + err := es.apiServerCmd.Process.Kill() + if err != nil { + glog.Errorf("Failed to stop be-apiserver.\n%v", err) + } + } + if es.etcdCmd != nil { + err := es.etcdCmd.Process.Kill() + if err != nil { + glog.Errorf("Failed to stop etcd.\n%v", err) + } + } + if es.etcdDataDir != "" { + err := os.RemoveAll(es.etcdDataDir) + if err != nil { + glog.Errorf("Failed to delete etcd data directory %s.\n%v", es.etcdDataDir, err) + } + } +} + +func (es *e2eService) startEtcd() (*exec.Cmd, error) { + dataDir, err := ioutil.TempDir("", "node-e2e") + if err != nil { + return nil, err + } + es.etcdDataDir = dataDir + return es.startServer(healthCheckCommand{ + combinedOut: &es.etcdCombinedOut, + healthCheckUrl: "http://127.0.0.1:4001/v2/keys", + command: "etcd", + args: []string{"--data-dir", dataDir, "--name", "e2e-node"}, + }) +} + +func (es *e2eService) startApiServer() (*exec.Cmd, error) { + return es.startServer( + healthCheckCommand{ + combinedOut: &es.apiServerCombinedOut, + healthCheckUrl: "http://127.0.0.1:8080/healthz", + command: "sudo", + args: []string{getApiServerBin(), + "--v", "2", "--logtostderr", "--log_dir", "./", + "--etcd-servers", "http://127.0.0.1:4001", + "--insecure-bind-address", "0.0.0.0", + "--service-cluster-ip-range", "10.0.0.1/24", + "--kubelet-port", "10250"}, + }) +} + +func (es *e2eService) startKubeletServer() (*exec.Cmd, error) { + return es.startServer( + healthCheckCommand{ + combinedOut: &es.kubeletCombinedOut, + healthCheckUrl: "http://127.0.0.1:10255/healthz", + command: "sudo", + args: []string{getKubeletServerBin(), + "--v", "2", "--logtostderr", "--log_dir", "./", + "--api-servers", "http://127.0.0.1:8080", + "--address", "0.0.0.0", + "--port", "10250"}, + }) +} + +func (es *e2eService) startServer(hcc healthCheckCommand) (*exec.Cmd, error) { + cmdErrorChan := make(chan error) + cmd := exec.Command(hcc.command, hcc.args...) + cmd.Stdout = hcc.combinedOut + cmd.Stderr = hcc.combinedOut + go func() { + err := cmd.Run() + if err != nil { + cmdErrorChan <- fmt.Errorf("%v Exited with status %v. Output:\n%s", hcc, err, *hcc.combinedOut) + } + close(cmdErrorChan) + }() + + endTime := time.Now().Add(*serverStartTimeout) + for endTime.After(time.Now()) { + select { + case err := <-cmdErrorChan: + return nil, err + case <-time.After(time.Second): + resp, err := http.Get(hcc.healthCheckUrl) + if err == nil && resp.StatusCode == http.StatusOK { + return cmd, nil + } + } + } + return nil, fmt.Errorf("Timeout waiting for service %v", hcc) +} + +type healthCheckCommand struct { + healthCheckUrl string + command string + args []string + combinedOut *bytes.Buffer +} + +func (hcc *healthCheckCommand) String() string { + return fmt.Sprintf("`%s %s` %s", hcc.command, strings.Join(hcc.args, " "), hcc.healthCheckUrl) +} diff --git a/test/e2e_node/environment/conformance.go b/test/e2e_node/environment/conformance.go index e697e7a783a9c..9ac19a44dadb8 100644 --- a/test/e2e_node/environment/conformance.go +++ b/test/e2e_node/environment/conformance.go @@ -28,8 +28,9 @@ import ( "strings" "errors" - "k8s.io/kubernetes/pkg/kubelet/cadvisor" "os" + + "k8s.io/kubernetes/pkg/kubelet/cadvisor" ) const success = "\033[0;32mSUCESS\033[0m" diff --git a/test/e2e_node/gcloud/gcloud.go b/test/e2e_node/gcloud/gcloud.go deleted file mode 100644 index 1772a102b114b..0000000000000 --- a/test/e2e_node/gcloud/gcloud.go +++ /dev/null @@ -1,213 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gcloud - -import ( - "bytes" - "errors" - "fmt" - "math/rand" - "net" - "net/http" - "os/exec" - "path/filepath" - "regexp" - "strings" - "time" - - "github.com/golang/glog" -) - -var freePortRegexp = regexp.MustCompile(".+:([0-9]+)") - -type TearDown func() *RunResult - -type GCloudClient interface { - RunAndWaitTillHealthy( - sudo bool, copyBin bool, remotePort string, - timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error) -} - -type gCloudClientImpl struct { - host string - zone string -} - -type RunResult struct { - out []byte - err error - cmd string -} - -type CmdHandle struct { - TearDown TearDown - CombinedOutput bytes.Buffer - Output chan RunResult - LPort string -} - -func NewGCloudClient(host string, zone string) GCloudClient { - return &gCloudClientImpl{host, zone} -} - -func (gc *gCloudClientImpl) Command(cmd string, moreargs ...string) ([]byte, error) { - args := append([]string{"compute", "ssh"}) - if gc.zone != "" { - args = append(args, "--zone", gc.zone) - } - args = append(args, gc.host, "--", cmd) - args = append(args, moreargs...) - glog.V(2).Infof("Command gcloud %s", strings.Join(args, " ")) - return exec.Command("gcloud", args...).CombinedOutput() -} - -func (gc *gCloudClientImpl) TunnelCommand(sudo bool, lPort string, rPort string, dir string, cmd string, moreargs ...string) *exec.Cmd { - tunnelStr := fmt.Sprintf("-L %s:localhost:%s", lPort, rPort) - args := []string{"compute", "ssh"} - if gc.zone != "" { - args = append(args, "--zone", gc.zone) - } - args = append(args, "--ssh-flag", tunnelStr, gc.host, "--") - args = append(args, "cd", dir, ";") - if sudo { - args = append(args, "sudo") - } - args = append(args, cmd) - args = append(args, moreargs...) - glog.V(2).Infof("Command gcloud %s", strings.Join(args, " ")) - return exec.Command("gcloud", args...) -} - -func (gc *gCloudClientImpl) CopyToHost(from string, to string) ([]byte, error) { - rto := fmt.Sprintf("%s:%s", gc.host, to) - args := []string{"compute", "copy-files"} - if gc.zone != "" { - args = append(args, "--zone", gc.zone) - } - args = append(args, from, rto) - glog.V(2).Infof("Command gcloud %s", strings.Join(args, " ")) - return exec.Command("gcloud", args...).CombinedOutput() -} - -func (gc *gCloudClientImpl) Run( - sudo bool, copyBin bool, remotePort string, bin string, args ...string) *CmdHandle { - - h := &CmdHandle{} - h.Output = make(chan RunResult) - - rand.Seed(time.Now().UnixNano()) - - // Define where we will copy the temp binary - tDir := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31()) - _, f := filepath.Split(bin) - cmd := f - if copyBin { - cmd = filepath.Join(tDir, f) - } - h.LPort = getLocalPort() - - h.TearDown = func() *RunResult { - out, err := gc.Command("sudo", "pkill", f) - if err != nil { - return &RunResult{out, err, fmt.Sprintf("pkill %s", f)} - } - out, err = gc.Command("rm", "-rf", tDir) - if err != nil { - return &RunResult{out, err, fmt.Sprintf("rm -rf %s", tDir)} - } - return &RunResult{} - } - - // Run the commands in a Go fn so that this method doesn't block when writing to a channel - // to report an error - go func() { - // Create the tmp directory - out, err := gc.Command("mkdir", "-p", tDir) - - // Work around for gcloud flakiness - TODO: debug why gcloud sometimes cannot find credentials for some hosts - // If there was an error about credentials, retry making the directory 6 times to see if it can be resolved - // This is to help debug if the credential issues are persistent for a given host on a given run, or transient - // And if downstream gcloud commands are also impacted - for i := 0; i < 6 && err != nil && strings.Contains(string(out), "does not have any valid credentials"); i++ { - glog.Warningf("mkdir failed on host %s due to credential issues, retrying in 5 seconds %v %s", gc.host, err, out) - time.Sleep(5 * time.Second) - out, err = gc.Command("mkdir", "-p", tDir) - } - if err != nil { - glog.Errorf("mkdir failed %v %s", err, out) - h.Output <- RunResult{out, err, fmt.Sprintf("mkdir -p %s", tDir)} - return - } - - // Copy the binary - if copyBin { - out, err = gc.CopyToHost(bin, tDir) - if err != nil { - glog.Errorf("copy-files failed %v %s", err, out) - h.Output <- RunResult{out, err, fmt.Sprintf("copy-files %s %s", bin, tDir)} - return - } - } - - c := gc.TunnelCommand(sudo, h.LPort, remotePort, tDir, cmd, args...) - c.Stdout = &h.CombinedOutput - c.Stderr = &h.CombinedOutput - go func() { - // Start the process - err = c.Run() - if err != nil { - glog.Errorf("command failed %v %s", err, h.CombinedOutput.Bytes()) - h.Output <- RunResult{h.CombinedOutput.Bytes(), err, fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))} - return - } - }() - }() - return h -} - -func (gc *gCloudClientImpl) RunAndWaitTillHealthy( - sudo bool, copyBin bool, - remotePort string, timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error) { - h := gc.Run(sudo, copyBin, remotePort, bin, args...) - eTime := time.Now().Add(timeout) - done := false - for eTime.After(time.Now()) && !done { - select { - case r := <-h.Output: - glog.V(2).Infof("Error running %s Output:\n%s Error:\n%v", r.cmd, r.out, r.err) - return h, r.err - case <-time.After(2 * time.Second): - resp, err := http.Get(fmt.Sprintf("http://localhost:%s/%s", h.LPort, healthUrl)) - if err == nil && resp.StatusCode == http.StatusOK { - done = true - break - } - } - } - if !done { - return h, errors.New(fmt.Sprintf("Timeout waiting for service to be healthy at http://localhost:%s/%s", h.LPort, healthUrl)) - } - glog.Info("Healthz Success") - return h, nil -} - -// GetLocalPort returns a free local port that can be used for ssh tunneling -func getLocalPort() string { - l, _ := net.Listen("tcp", ":0") - defer l.Close() - return freePortRegexp.FindStringSubmatch(l.Addr().String())[1] -} diff --git a/test/e2e_node/kubelet_test.go b/test/e2e_node/kubelet_test.go index dd88defa803db..69240dad1b951 100644 --- a/test/e2e_node/kubelet_test.go +++ b/test/e2e_node/kubelet_test.go @@ -249,7 +249,7 @@ var _ = Describe("Kubelet", func() { Expect(*container.Logs.UsedBytes).NotTo(BeZero(), spew.Sdump(container)) } - Expect(podsList).To(ConsistOf(podNames)) + Expect(podsList).To(ConsistOf(podNames), spew.Sdump(summary)) }) }) diff --git a/test/e2e_node/runner/run_e2e.go b/test/e2e_node/runner/run_e2e.go index 26e4759797e7b..e2d5bb8d93c59 100644 --- a/test/e2e_node/runner/run_e2e.go +++ b/test/e2e_node/runner/run_e2e.go @@ -19,189 +19,72 @@ limitations under the License. package main import ( - "bufio" "flag" "fmt" "os" - "os/exec" "strings" - "sync" - "time" - "runtime" - - "github.com/golang/glog" - "k8s.io/kubernetes/test/e2e_node/gcloud" - "path/filepath" + "k8s.io/kubernetes/test/e2e_node" ) -type RunFunc func(host string, port string) ([]byte, error) - -type Result struct { - host string - output []byte - err error -} - -const gray = "\033[1;30m" -const blue = "\033[0;34m" -const noColour = "\033[0m" - -var u = sync.WaitGroup{} -var zone = flag.String("zone", "", "gce zone the hosts live in") var hosts = flag.String("hosts", "", "hosts to test") -var wait = flag.Bool("wait", false, "if true, wait for input before running tests") -var kubeOutputRelPath = flag.String("k8s-build-output", "_output/local/bin/linux/amd64", "Where k8s binary files are written") - -var kubeRoot = "" - -const buildScriptRelPath = "hack/build-go.sh" -const ginkgoTestRelPath = "test/e2e_node" -const healthyTimeoutDuration = time.Minute * 3 func main() { - flag.Parse() - if *hosts == "" { - glog.Fatalf("Must specific --hosts flag") - } - - // Figure out the kube root - _, path, _, _ := runtime.Caller(0) - kubeRoot, _ = filepath.Split(path) - kubeRoot = strings.Split(kubeRoot, "/test/e2e_node")[0] - - // Build the go code - out, err := exec.Command(filepath.Join(kubeRoot, buildScriptRelPath)).CombinedOutput() - if err != nil { - glog.Fatalf("Failed to build go packages %s: %v", out, err) + // Setup coloring + stat, _ := os.Stdout.Stat() + useColor := (stat.Mode() & os.ModeCharDevice) != 0 + blue := "" + noColour := "" + if useColor { + blue = "\033[0;34m" + noColour = "\033[0m" } - // Copy kubelet to each host and run test - if *wait { - u.Add(1) + flag.Parse() + if *hosts == "" { + fmt.Printf("Must specific --hosts flag") } + archive := e2e_node.CreateTestArchive() + defer os.Remove(archive) results := make(chan *TestResult) hs := strings.Split(*hosts, ",") for _, h := range hs { - go func(host string) { results <- runTests(host) }(h) - } - - // Maybe wait for user input before running tests - if *wait { - WaitForUser() + fmt.Printf("Starting tests on host %s.", h) + go func(host string) { + output, err := e2e_node.RunRemote(archive, host) + results <- &TestResult{ + output: output, + err: err, + host: host, + } + }(h) } // Wait for all tests to complete and emit the results errCount := 0 for i := 0; i < len(hs); i++ { tr := <-results - host := tr.fullhost + host := tr.host + fmt.Printf("%s================================================================%s\n", blue, noColour) if tr.err != nil { errCount++ - glog.Infof("%s================================================================%s", blue, noColour) - glog.Infof("Failure Finished Host %s Test Suite %s %v", host, tr.testCombinedOutput, tr.err) - glog.V(2).Infof("----------------------------------------------------------------") - glog.V(5).Infof("Host %s Etcd Logs\n%s%s%s", host, gray, tr.etcdCombinedOutput, noColour) - glog.V(5).Infof("----------------------------------------------------------------") - glog.V(5).Infof("Host %s Apiserver Logs\n%s%s%s", host, gray, tr.apiServerCombinedOutput, noColour) - glog.V(5).Infof("----------------------------------------------------------------") - glog.V(2).Infof("Host %s Kubelet Logs\n%s%s%s", host, gray, tr.kubeletCombinedOutput, noColour) - glog.Infof("%s================================================================%s", blue, noColour) + fmt.Printf("Failure Finished Host %s Test Suite %s %v\n", host, tr.output, tr.err) } else { - glog.Infof("================================================================") - glog.Infof("Success Finished Host %s Test Suite %s", host, tr.testCombinedOutput) - glog.Infof("================================================================") + fmt.Printf("Success Finished Host %s Test Suite %s\n", host, tr.output) } + fmt.Printf("%s================================================================%s\n", blue, noColour) } // Set the exit code if there were failures if errCount > 0 { - glog.Errorf("Failure: %d errors encountered.", errCount) + fmt.Printf("Failure: %d errors encountered.", errCount) os.Exit(1) } } -func WaitForUser() { - scanner := bufio.NewScanner(os.Stdin) - fmt.Printf("Enter \"y\" to run tests\n") - for scanner.Scan() { - if strings.ToUpper(scanner.Text()) != "Y\n" { - break - } - fmt.Printf("Enter \"y\" to run tests\n") - } - u.Done() -} - type TestResult struct { - fullhost string - err error - testCombinedOutput string - etcdCombinedOutput string - apiServerCombinedOutput string - kubeletCombinedOutput string -} - -func runTests(fullhost string) *TestResult { - result := &TestResult{fullhost: fullhost} - - host := strings.Split(fullhost, ".")[0] - c := gcloud.NewGCloudClient(host, *zone) - // TODO(pwittrock): Come up with something better for bootstrapping the environment. - eh, err := c.RunAndWaitTillHealthy( - false, false, "4001", healthyTimeoutDuration, "v2/keys/", "etcd", "--data-dir", "./", "--name", "e2e-node") - defer func() { - eh.TearDown() - result.etcdCombinedOutput = fmt.Sprintf("%s", eh.CombinedOutput.Bytes()) - }() - if err != nil { - result.err = fmt.Errorf("Host %s failed to run command %v", host, err) - return result - } - - apiBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kube-apiserver") - ah, err := c.RunAndWaitTillHealthy( - true, true, "8080", healthyTimeoutDuration, "healthz", apiBin, "--service-cluster-ip-range", - "10.0.0.1/24", "--insecure-bind-address", "0.0.0.0", "--etcd-servers", "http://127.0.0.1:4001", - "--v", "2", "--alsologtostderr", "--kubelet-port", "10250") - defer func() { - ah.TearDown() - result.apiServerCombinedOutput = fmt.Sprintf("%s", ah.CombinedOutput.Bytes()) - }() - if err != nil { - result.err = fmt.Errorf("Host %s failed to run command %v", host, err) - return result - } - - kubeletBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kubelet") - // TODO: Used --v 4 or higher and upload to gcs instead of printing to the console - // TODO: Copy /var/log/messages and upload to GCS for failed tests - kh, err := c.RunAndWaitTillHealthy( - true, true, "10255", healthyTimeoutDuration, "healthz", kubeletBin, "--api-servers", "http://127.0.0.1:8080", - "--v", "2", "--alsologtostderr", "--address", "0.0.0.0", "--port", "10250") - defer func() { - kh.TearDown() - result.kubeletCombinedOutput = fmt.Sprintf("%s", kh.CombinedOutput.Bytes()) - }() - if err != nil { - result.err = fmt.Errorf("Host %s failed to run command %v", host, err) - } - - // Run the tests - glog.Infof("Kubelet healthy on host %s", host) - glog.Infof("Kubelet host %s tunnel running on port %s", host, ah.LPort) - u.Wait() - glog.Infof("Running ginkgo tests against host %s", host) - ginkgoTests := filepath.Join(kubeRoot, ginkgoTestRelPath) - out, err := exec.Command( - "ginkgo", ginkgoTests, "--", - "--kubelet-address", fmt.Sprintf("http://127.0.0.1:%s", kh.LPort), - "--api-server-address", fmt.Sprintf("http://127.0.0.1:%s", ah.LPort), - "--node-name", fullhost, - "--v", "2", "--alsologtostderr").CombinedOutput() - - result.err = err - result.testCombinedOutput = fmt.Sprintf("%s", out) - return result + output string + err error + host string }