Merge pull request #48899 from luxas/kubeadm_easy_upgrades_plan

Automatic merge from submit-queue Implement the `kubeadm upgrade` command **What this PR does / why we need it**: Implements the kubeadm upgrades proposal: https://docs.google.com/document/d/1PRrC2tvB-p7sotIA5rnHy5WAOGdJJOIXPPv23hUFGrY/edit# **Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes # fixes: kubernetes/kubeadm#14 **Special notes for your reviewer**: I'm gonna split out changes not directly related to the upgrade procedure into separate PRs as dependencies as we go. **Now ready for review. Please look at `cmd/kubeadm/app/phases/upgrade` first and give feedback on that. The rest kind of follows.** **Release note**: ```release-note Implemented `kubeadm upgrade plan` for checking whether you can upgrade your cluster to a newer version Implemented `kubeadm upgrade apply` for upgrading your cluster from one version to an other ``` cc @fabriziopandini @kubernetes/sig-cluster-lifecycle-pr-reviews @craigtracey @mattmoyer
kubernetes · Sep 3, 2017 · b3efdeb · b3efdeb
2 parents ea1d105 + c575626
commit b3efdeb
Show file tree

Hide file tree

Showing 28 changed files with 2,796 additions and 81 deletions.
diff --git a/cmd/kubeadm/app/cmd/upgrade/BUILD b/cmd/kubeadm/app/cmd/upgrade/BUILD
@@ -13,14 +13,19 @@ go_library(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
         "//cmd/kubeadm/app/cmd/util:go_default_library",
+        "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/phases/controlplane:go_default_library",
         "//cmd/kubeadm/app/phases/upgrade:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
+        "//cmd/kubeadm/app/util/apiclient:go_default_library",
+        "//cmd/kubeadm/app/util/dryrun:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
         "//pkg/api:go_default_library",
         "//pkg/util/version:go_default_library",
         "//vendor/github.com/ghodss/yaml:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
+        "//vendor/k8s.io/client-go/discovery/fake:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
     ],
 )

diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go
@@ -18,6 +18,7 @@ package upgrade
 
 import (
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
@@ -26,12 +27,20 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
+	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/util/version"
 )
 
+const (
+	upgradeManifestTimeout = 1 * time.Minute
+)
+
 // applyFlags holds the information about the flags that can be passed to apply
 type applyFlags struct {
 	nonInteractiveMode bool
@@ -102,7 +111,7 @@ func NewCmdApply(parentFlags *cmdUpgradeFlags) *cobra.Command {
 func RunApply(flags *applyFlags) error {
 
 	// Start with the basics, verify that the cluster is healthy and get the configuration from the cluster (using the ConfigMap)
-	upgradeVars, err := enforceRequirements(flags.parent.kubeConfigPath, flags.parent.cfgPath, flags.parent.printConfig)
+	upgradeVars, err := enforceRequirements(flags.parent.kubeConfigPath, flags.parent.cfgPath, flags.parent.printConfig, flags.dryRun)
 	if err != nil {
 		return err
 	}
@@ -126,16 +135,24 @@ func RunApply(flags *applyFlags) error {
 		}
 	}
 
-	// TODO: Implement a prepulling mechanism here
+	// Use a prepuller implementation based on creating DaemonSets
+	// and block until all DaemonSets are ready; then we know for sure that all control plane images are cached locally
+	prepuller := upgrade.NewDaemonSetPrepuller(upgradeVars.client, upgradeVars.waiter, internalcfg)
+	upgrade.PrepullImagesInParallel(prepuller, flags.imagePullTimeout)
 
 	// Now; perform the upgrade procedure
-	if err := PerformControlPlaneUpgrade(flags, upgradeVars.client, internalcfg); err != nil {
+	if err := PerformControlPlaneUpgrade(flags, upgradeVars.client, upgradeVars.waiter, internalcfg); err != nil {
 		return fmt.Errorf("[upgrade/apply] FATAL: %v", err)
 	}
 
 	// Upgrade RBAC rules and addons. Optionally, if needed, perform some extra task for a specific version
 	if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, internalcfg, flags.newK8sVersion); err != nil {
-		return fmt.Errorf("[upgrade/postupgrade] FATAL: %v", err)
+		return fmt.Errorf("[upgrade/postupgrade] FATAL post-upgrade error: %v", err)
+	}
+
+	if flags.dryRun {
+		fmt.Println("[dryrun] Finished dryrunning successfully!")
+		return nil
 	}
 
 	fmt.Println("")
@@ -182,7 +199,7 @@ func EnforceVersionPolicies(flags *applyFlags, versionGetter upgrade.VersionGett
 		if len(versionSkewErrs.Skippable) > 0 {
 			// Return the error if the user hasn't specified the --force flag
 			if !flags.force {
-				return fmt.Errorf("The --version argument is invalid due to these errors: %v. Can be bypassed if you pass the --force flag", versionSkewErrs.Mandatory)
+				return fmt.Errorf("The --version argument is invalid due to these errors: %v. Can be bypassed if you pass the --force flag", versionSkewErrs.Skippable)
 			}
 			// Soft errors found, but --force was specified
 			fmt.Printf("[upgrade/version] Found %d potential version compatibility errors but skipping since the --force flag is set: %v\n", len(versionSkewErrs.Skippable), versionSkewErrs.Skippable)
@@ -192,21 +209,60 @@ func EnforceVersionPolicies(flags *applyFlags, versionGetter upgrade.VersionGett
 }
 
 // PerformControlPlaneUpgrade actually performs the upgrade procedure for the cluster of your type (self-hosted or static-pod-hosted)
-func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, internalcfg *kubeadmapi.MasterConfiguration) error {
+func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.MasterConfiguration) error {
 
 	// Check if the cluster is self-hosted and act accordingly
 	if upgrade.IsControlPlaneSelfHosted(client) {
 		fmt.Printf("[upgrade/apply] Upgrading your Self-Hosted control plane to version %q...\n", flags.newK8sVersionStr)
 
-		// Upgrade a self-hosted cluster
-		// TODO(luxas): Implement this later when we have the new upgrade strategy
-		return fmt.Errorf("not implemented")
+		// Upgrade the self-hosted cluster
+		return upgrade.SelfHostedControlPlane(client, waiter, internalcfg, flags.newK8sVersion)
 	}
 
 	// OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster
 	fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q...\n", flags.newK8sVersionStr)
 
-	if err := upgrade.PerformStaticPodControlPlaneUpgrade(client, internalcfg, flags.newK8sVersion); err != nil {
+	if flags.dryRun {
+		return DryRunStaticPodUpgrade(internalcfg)
+	}
+	return PerformStaticPodUpgrade(client, waiter, internalcfg)
+}
+
+// PerformStaticPodUpgrade performs the upgrade of the control plane components for a static pod hosted cluster
+func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.MasterConfiguration) error {
+	pathManager, err := upgrade.NewKubeStaticPodPathManagerUsingTempDirs(constants.GetStaticPodDirectory())
+	if err != nil {
+		return err
+	}
+
+	if err := upgrade.StaticPodControlPlane(waiter, pathManager, internalcfg); err != nil {
+		return err
+	}
+	return nil
+}
+
+// DryRunStaticPodUpgrade fakes an upgrade of the control plane
+func DryRunStaticPodUpgrade(internalcfg *kubeadmapi.MasterConfiguration) error {
+
+	dryRunManifestDir, err := constants.CreateTempDirForKubeadm("kubeadm-upgrade-dryrun")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(dryRunManifestDir)
+
+	if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, internalcfg); err != nil {
+		return err
+	}
+
+	// Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests
+	files := []dryrunutil.FileToPrint{}
+	for _, component := range constants.MasterComponents {
+		realPath := constants.GetStaticPodFilepath(component, dryRunManifestDir)
+		outputPath := constants.GetStaticPodFilepath(component, constants.GetStaticPodDirectory())
+		files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath))
+	}
+
+	if err := dryrunutil.PrintDryRunFiles(files, os.Stdout); err != nil {
 		return err
 	}
 	return nil

diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go
@@ -26,10 +26,13 @@ import (
 
 	"github.com/ghodss/yaml"
 
+	fakediscovery "k8s.io/client-go/discovery/fake"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
+	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 )
 
@@ -39,11 +42,12 @@ type upgradeVariables struct {
 	client        clientset.Interface
 	cfg           *kubeadmapiext.MasterConfiguration
 	versionGetter upgrade.VersionGetter
+	waiter        apiclient.Waiter
 }
 
 // enforceRequirements verifies that it's okay to upgrade and then returns the variables needed for the rest of the procedure
-func enforceRequirements(kubeConfigPath, cfgPath string, printConfig bool) (*upgradeVariables, error) {
-	client, err := kubeconfigutil.ClientSetFromFile(kubeConfigPath)
+func enforceRequirements(kubeConfigPath, cfgPath string, printConfig, dryRun bool) (*upgradeVariables, error) {
+	client, err := getClient(kubeConfigPath, dryRun)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't create a Kubernetes client from file %q: %v", kubeConfigPath, err)
 	}
@@ -69,6 +73,8 @@ func enforceRequirements(kubeConfigPath, cfgPath string, printConfig bool) (*upg
 		cfg:    cfg,
 		// Use a real version getter interface that queries the API server, the kubeadm client and the Kubernetes CI system for latest versions
 		versionGetter: upgrade.NewKubeVersionGetter(client, os.Stdout),
+		// Use the waiter conditionally based on the dryrunning variable
+		waiter: getWaiter(dryRun, client),
 	}, nil
 }
 
@@ -101,6 +107,46 @@ func runPreflightChecks(skipPreFlight bool) error {
 	return preflight.RunRootCheckOnly()
 }
 
+// getClient gets a real or fake client depending on whether the user is dry-running or not
+func getClient(file string, dryRun bool) (clientset.Interface, error) {
+	if dryRun {
+		dryRunGetter, err := apiclient.NewClientBackedDryRunGetterFromKubeconfig(file)
+		if err != nil {
+			return nil, err
+		}
+
+		// In order for fakeclient.Discovery().ServerVersion() to return the backing API Server's
+		// real version; we have to do some clever API machinery tricks. First, we get the real
+		// API Server's version
+		realServerVersion, err := dryRunGetter.Client().Discovery().ServerVersion()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get server version: %v", err)
+		}
+
+		// Get the fake clientset
+		fakeclient := apiclient.NewDryRunClient(dryRunGetter, os.Stdout)
+		// As we know the return of Discovery() of the fake clientset is of type *fakediscovery.FakeDiscovery
+		// we can convert it to that struct.
+		fakeclientDiscovery, ok := fakeclient.Discovery().(*fakediscovery.FakeDiscovery)
+		if !ok {
+			return nil, fmt.Errorf("couldn't set fake discovery's server version")
+		}
+		// Lastly, set the right server version to be used
+		fakeclientDiscovery.FakedServerVersion = realServerVersion
+		// return the fake clientset used for dry-running
+		return fakeclient, nil
+	}
+	return kubeconfigutil.ClientSetFromFile(file)
+}
+
+// getWaiter gets the right waiter implementation
+func getWaiter(dryRun bool, client clientset.Interface) apiclient.Waiter {
+	if dryRun {
+		return dryrunutil.NewWaiter()
+	}
+	return apiclient.NewKubeWaiter(client, upgradeManifestTimeout, os.Stdout)
+}
+
 // InteractivelyConfirmUpgrade asks the user whether they _really_ want to upgrade.
 func InteractivelyConfirmUpgrade(question string) error {
 

diff --git a/cmd/kubeadm/app/cmd/upgrade/plan.go b/cmd/kubeadm/app/cmd/upgrade/plan.go
@@ -50,16 +50,16 @@ func NewCmdPlan(parentFlags *cmdUpgradeFlags) *cobra.Command {
 // RunPlan takes care of outputting available versions to upgrade to for the user
 func RunPlan(parentFlags *cmdUpgradeFlags) error {
 
-	// Start with the basics, verify that the cluster is healthy, build a client and a versionGetter.
-	upgradeVars, err := enforceRequirements(parentFlags.kubeConfigPath, parentFlags.cfgPath, parentFlags.printConfig)
+	// Start with the basics, verify that the cluster is healthy, build a client and a versionGetter. Never set dry-run for plan.
+	upgradeVars, err := enforceRequirements(parentFlags.kubeConfigPath, parentFlags.cfgPath, parentFlags.printConfig, false)
 	if err != nil {
 		return err
 	}
 
 	// Compute which upgrade possibilities there are
 	availUpgrades, err := upgrade.GetAvailableUpgrades(upgradeVars.versionGetter, parentFlags.allowExperimentalUpgrades, parentFlags.allowRCUpgrades)
 	if err != nil {
-		return err
+		return fmt.Errorf("[upgrade/versions] FATAL: %v", err)
 	}
 
 	// Tell the user which upgrades are available

diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go
@@ -18,6 +18,8 @@ package constants
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
 	"path/filepath"
 	"time"
 
@@ -31,6 +33,7 @@ var KubernetesDir = "/etc/kubernetes"
 
 const (
 	ManifestsSubDirName = "manifests"
+	TempDirForKubeadm   = "/etc/kubernetes/tmp"
 
 	CACertAndKeyBaseName = "ca"
 	CACertName           = "ca.crt"
@@ -181,3 +184,17 @@ func GetAdminKubeConfigPath() string {
 func AddSelfHostedPrefix(componentName string) string {
 	return fmt.Sprintf("%s%s", SelfHostingPrefix, componentName)
 }
+
+// CreateTempDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp (not using /tmp as that would potentially be dangerous)
+func CreateTempDirForKubeadm(dirName string) (string, error) {
+	// creates target folder if not already exists
+	if err := os.MkdirAll(TempDirForKubeadm, 0700); err != nil {
+		return "", fmt.Errorf("failed to create directory %q: %v", TempDirForKubeadm, err)
+	}
+
+	tempDir, err := ioutil.TempDir(TempDirForKubeadm, dirName)
+	if err != nil {
+		return "", fmt.Errorf("couldn't create a temporary directory: %v", err)
+	}
+	return tempDir, nil
+}
diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go
@@ -21,6 +21,7 @@ import (
 
 	"k8s.io/api/core/v1"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 )
 
@@ -34,8 +35,8 @@ const (
 // PodSpecMutatorFunc is a function capable of mutating a PodSpec
 type PodSpecMutatorFunc func(*v1.PodSpec)
 
-// getDefaultMutators gets the mutator functions that alwasy should be used
-func getDefaultMutators() map[string][]PodSpecMutatorFunc {
+// GetDefaultMutators gets the mutator functions that alwasy should be used
+func GetDefaultMutators() map[string][]PodSpecMutatorFunc {
 	return map[string][]PodSpecMutatorFunc{
 		kubeadmconstants.KubeAPIServer: {
 			addNodeSelectorToPodSpec,
@@ -55,6 +56,22 @@ func getDefaultMutators() map[string][]PodSpecMutatorFunc {
 	}
 }
 
+// GetMutatorsFromFeatureGates returns all mutators needed based on the feature gates passed
+func GetMutatorsFromFeatureGates(featureGates map[string]bool) map[string][]PodSpecMutatorFunc {
+	// Here the map of different mutators to use for the control plane's podspec is stored
+	mutators := GetDefaultMutators()
+
+	// Some extra work to be done if we should store the control plane certificates in Secrets
+	if features.Enabled(featureGates, features.StoreCertsInSecrets) {
+
+		// Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them
+		mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer)
+		mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager)
+		mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler)
+	}
+	return mutators
+}
+
 // mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting
 func mutatePodSpec(mutators map[string][]PodSpecMutatorFunc, name string, podSpec *v1.PodSpec) {
 	// Get the mutator functions for the component in question, then loop through and execute them

diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go
@@ -73,7 +73,7 @@ func TestMutatePodSpec(t *testing.T) {
 	}
 
 	for _, rt := range tests {
-		mutatePodSpec(getDefaultMutators(), rt.component, rt.podSpec)
+		mutatePodSpec(GetDefaultMutators(), rt.component, rt.podSpec)
 
 		if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
 			t.Errorf("failed mutatePodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)