Skip to content

Commit

Permalink
Merge pull request kubernetes#30116 from Random-Liu/start-e2e-service…
Browse files Browse the repository at this point in the history 
…s-in-separate-process

Automatic merge from submit-queue

Node Conformance Test: Start e2e services in a separate process

For kubernetes#30122.

This is the first step of kubernetes#30174.

This PR added a `start-services-only` mode in the test suite, and start all e2e services in a separate process by run the test binary again in `start-services-only` mode.
  • Loading branch information
Kubernetes Submit Queue authored Aug 17, 2016
2 parents 60579ee + 3910a66 commit 457a151
Show file tree
Hide file tree
Showing 6 changed files with 309 additions and 171 deletions.
2 changes: 2 additions & 0 deletions hack/verify-flags/known-flags.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,7 @@ log-flush-frequency
long-running-request-regexp
low-diskspace-threshold-mb
make-symlinks
manifest-path
manifest-url
manifest-url-header
masquerade-all
Expand Down Expand Up @@ -421,6 +422,7 @@ rkt-stage1-image
root-ca-file
root-dir
run-proxy
run-services-mode
runtime-cgroups
runtime-config
runtime-request-timeout
Expand Down
20 changes: 17 additions & 3 deletions test/e2e/framework/test_context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,24 @@ type TestContextType struct {
CreateTestingNS CreateTestingNSFn
// If set to true test will dump data about the namespace in which test was running.
DumpLogsOnFailure bool
// If the garbage collector is enabled in the kube-apiserver and kube-controller-manager.
GarbageCollectorEnabled bool
// Node e2e specific test context
NodeTestContextType
}

// NodeTestContextType is part of TestContextType, it is shared by all node e2e test.
type NodeTestContextType struct {
// Name of the node to run tests on (node e2e suite only).
NodeName string
// DisableKubenet disables kubenet when starting kubelet.
DisableKubenet bool
// Whether to enable the QoS Cgroup Hierarchy or not
CgroupsPerQOS bool
// The hard eviction thresholds
EvictionHard string
// If the garbage collector is enabled in the kube-apiserver and kube-controller-manager.
GarbageCollectorEnabled bool
// ManifestPath is the static pod manifest path.
ManifestPath string
}

type CloudConfig struct {
Expand Down Expand Up @@ -112,7 +122,6 @@ func RegisterCommonFlags() {
flag.StringVar(&TestContext.Host, "host", "http://127.0.0.1:8080", "The host, or apiserver, to connect to")
flag.StringVar(&TestContext.ReportPrefix, "report-prefix", "", "Optional prefix for JUnit XML reports. Default is empty, which doesn't prepend anything to the default name.")
flag.StringVar(&TestContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.")
flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", false, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.")
}

// Register flags specific to the cluster e2e test suite.
Expand Down Expand Up @@ -149,11 +158,16 @@ func RegisterClusterFlags() {
flag.StringVar(&TestContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.")
flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")
flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", false, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.")
}

// Register flags specific to the node e2e test suite.
func RegisterNodeFlags() {
flag.StringVar(&TestContext.NodeName, "node-name", "", "Name of the node to run tests on (node e2e suite only).")
// TODO(random-liu): Remove kubelet related flags when we move the kubelet start logic out of the test.
// TODO(random-liu): Find someway to get kubelet configuration, and automatic config and filter test based on the configuration.
flag.BoolVar(&TestContext.DisableKubenet, "disable-kubenet", false, "If true, start kubelet without kubenet. (default false)")
flag.BoolVar(&TestContext.CgroupsPerQOS, "cgroups-per-qos", false, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.")
flag.StringVar(&TestContext.EvictionHard, "eviction-hard", "memory.available<250Mi", "The hard eviction thresholds. If set, pods get evicted when the specified resources drop below the thresholds.")
flag.StringVar(&TestContext.ManifestPath, "manifest-path", "", "The path to the static pod manifest file.")
}
71 changes: 29 additions & 42 deletions test/e2e_node/e2e_node_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,41 +45,30 @@ import (
"github.com/onsi/ginkgo/config"
more_reporters "github.com/onsi/ginkgo/reporters"
. "github.com/onsi/gomega"
"github.com/spf13/pflag"
)

var e2es *e2eService

// context is the test context shared by all parallel nodes.
// Originally we setup the test environment and initialize global variables
// in BeforeSuite, and then used the global variables in the test.
// However, after we make the test parallel, ginkgo will run all tests
// in several parallel test nodes. And for each test node, the BeforeSuite
// and AfterSuite will be run.
// We don't want to start services (kubelet, apiserver and etcd) for all
// parallel nodes, but we do want to set some globally shared variable which
// could be used in test.
// We have to use SynchronizedBeforeSuite to achieve that. The first
// function of SynchronizedBeforeSuite is only called once, and the second
// function is called in each parallel test node. The result returned by
// the first function will be the parameter of the second function.
// So we'll start all services and initialize the shared context in the first
// function, and propagate the context to all parallel test nodes in the
// second function.
// Notice no lock is needed for shared context, because context should only be
// initialized in the first function in SynchronizedBeforeSuite. After that
// it should never be modified.
var context SharedContext
var e2es *E2EServices

var prePullImages = flag.Bool("prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.")

func init() {
framework.RegisterCommonFlags()
framework.RegisterNodeFlags()
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
// Mark the run-services-mode flag as hidden to prevent user from using it.
pflag.CommandLine.MarkHidden("run-services-mode")
}

func TestE2eNode(t *testing.T) {
flag.Parse()

pflag.Parse()
if *runServicesMode {
// If run-services-mode is specified, only run services in current process.
RunE2EServices()
return
}
// If run-services-mode is not specified, run test.
rand.Seed(time.Now().UTC().UnixNano())
RegisterFailHandler(Fail)
reporters := []Reporter{}
Expand All @@ -103,14 +92,15 @@ var _ = SynchronizedBeforeSuite(func() []byte {
if *buildServices {
Expect(buildGo()).To(Succeed())
}

// Initialize node name here, so that the following code can get right node name.
if framework.TestContext.NodeName == "" {
hostname, err := os.Hostname()
if err != nil {
glog.Fatalf("Could not get node name: %v", err)
}
framework.TestContext.NodeName = hostname
}

// Pre-pull the images tests depend on so we can fail immediately if there is an image pull issue
// This helps with debugging test flakes since it is hard to tell when a test failure is due to image pulling.
if *prePullImages {
Expand All @@ -124,45 +114,42 @@ var _ = SynchronizedBeforeSuite(func() []byte {
// We should mask locksmithd when provisioning the machine.
maskLocksmithdOnCoreos()

shared := &SharedContext{}
if *startServices {
e2es = newE2eService(framework.TestContext.NodeName, framework.TestContext.CgroupsPerQOS, framework.TestContext.EvictionHard, shared)
if err := e2es.start(); err != nil {
Fail(fmt.Sprintf("Unable to start node services.\n%v", err))
e2es = NewE2EServices()
if err := e2es.Start(); err != nil {
glog.Fatalf("Unable to start node services: %v", err)
}
glog.Infof("Node services started. Running tests...")
} else {
glog.Infof("Running tests without starting services.")
}

glog.Infof("Starting namespace controller")
// TODO(random-liu): Move namespace controller into namespace services.
startNamespaceController()

// Reference common test to make the import valid.
commontest.CurrentSuite = commontest.NodeE2E

// Share the node name with the other test nodes.
shared.NodeName = framework.TestContext.NodeName
data, err := json.Marshal(shared)
Expect(err).NotTo(HaveOccurred())

data, err := json.Marshal(&framework.TestContext.NodeTestContextType)
if err != nil {
glog.Fatalf("Failed to serialize node test context: %v", err)
}
return data
}, func(data []byte) {
// Set the shared context got from the synchronized initialize function
shared := &SharedContext{}
Expect(json.Unmarshal(data, shared)).To(Succeed())
context = *shared

framework.TestContext.NodeName = shared.NodeName
// The node test context is updated in the first function, update it on every test node.
err := json.Unmarshal(data, &framework.TestContext.NodeTestContextType)
if err != nil {
glog.Fatalf("Failed to deserialize node test context: %v", err)
}
})

// Tear down the kubelet on the node
var _ = SynchronizedAfterSuite(func() {}, func() {
if e2es != nil {
e2es.getLogFiles()
if *startServices && *stopServices {
glog.Infof("Stopping node services...")
e2es.stop()
e2es.Stop()
}
}

Expand Down
Loading

0 comments on commit 457a151

Please sign in to comment.