From f5a359a3af72d1e795758d65834160db769aafff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Sat, 26 Aug 2017 19:25:30 +0300 Subject: [PATCH] kubeadm: Implement self-hosted upgrades --- cmd/kubeadm/app/cmd/upgrade/apply.go | 5 +- .../phases/selfhosting/podspec_mutation.go | 21 +- .../selfhosting/podspec_mutation_test.go | 2 +- .../app/phases/selfhosting/selfhosting.go | 36 +-- .../phases/selfhosting/selfhosting_test.go | 2 +- cmd/kubeadm/app/phases/upgrade/prepull.go | 6 +- cmd/kubeadm/app/phases/upgrade/selfhosted.go | 272 ++++++++++++++++++ cmd/kubeadm/app/util/apiclient/idempotency.go | 9 + cmd/kubeadm/app/util/apiclient/wait.go | 2 +- 9 files changed, 325 insertions(+), 30 deletions(-) create mode 100644 cmd/kubeadm/app/phases/upgrade/selfhosted.go diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index 818d159068a92..d2d7d84aa8326 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -215,9 +215,8 @@ func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, w if upgrade.IsControlPlaneSelfHosted(client) { fmt.Printf("[upgrade/apply] Upgrading your Self-Hosted control plane to version %q...\n", flags.newK8sVersionStr) - // Upgrade a self-hosted cluster - // TODO(luxas): Implement this later when we have the new upgrade strategy - return fmt.Errorf("not implemented") + // Upgrade the self-hosted cluster + return upgrade.SelfHostedControlPlane(client, waiter, internalcfg, flags.newK8sVersion) } // OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go index 65575bb27b4b2..78c54fd51b3e1 100644 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go +++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go @@ -21,6 +21,7 @@ import ( "k8s.io/api/core/v1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) @@ -34,8 +35,8 @@ const ( // PodSpecMutatorFunc is a function capable of mutating a PodSpec type PodSpecMutatorFunc func(*v1.PodSpec) -// getDefaultMutators gets the mutator functions that alwasy should be used -func getDefaultMutators() map[string][]PodSpecMutatorFunc { +// GetDefaultMutators gets the mutator functions that alwasy should be used +func GetDefaultMutators() map[string][]PodSpecMutatorFunc { return map[string][]PodSpecMutatorFunc{ kubeadmconstants.KubeAPIServer: { addNodeSelectorToPodSpec, @@ -55,6 +56,22 @@ func getDefaultMutators() map[string][]PodSpecMutatorFunc { } } +// GetMutatorsFromFeatureGates returns all mutators needed based on the feature gates passed +func GetMutatorsFromFeatureGates(featureGates map[string]bool) map[string][]PodSpecMutatorFunc { + // Here the map of different mutators to use for the control plane's podspec is stored + mutators := GetDefaultMutators() + + // Some extra work to be done if we should store the control plane certificates in Secrets + if features.Enabled(featureGates, features.StoreCertsInSecrets) { + + // Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them + mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer) + mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager) + mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler) + } + return mutators +} + // mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting func mutatePodSpec(mutators map[string][]PodSpecMutatorFunc, name string, podSpec *v1.PodSpec) { // Get the mutator functions for the component in question, then loop through and execute them diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go index b5e2483c94ebe..391ea4bad0ea7 100644 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go +++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go @@ -73,7 +73,7 @@ func TestMutatePodSpec(t *testing.T) { } for _, rt := range tests { - mutatePodSpec(getDefaultMutators(), rt.component, rt.podSpec) + mutatePodSpec(GetDefaultMutators(), rt.component, rt.podSpec) if !reflect.DeepEqual(*rt.podSpec, rt.expected) { t.Errorf("failed mutatePodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go index 7ad39f6d69d7e..96fb4641c9586 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go @@ -60,7 +60,7 @@ func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubea waiter.SetTimeout(selfHostingWaitTimeout) // Here the map of different mutators to use for the control plane's podspec is stored - mutators := getDefaultMutators() + mutators := GetMutatorsFromFeatureGates(cfg.FeatureFlags) // Some extra work to be done if we should store the control plane certificates in Secrets if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) { @@ -72,10 +72,6 @@ func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubea if err := uploadKubeConfigSecrets(client, kubeConfigDir); err != nil { return err } - // Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them - mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer) - mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager) - mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler) } for _, componentName := range kubeadmconstants.MasterComponents { @@ -95,7 +91,7 @@ func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubea } // Build a DaemonSet object from the loaded PodSpec - ds := buildDaemonSet(componentName, podSpec, mutators) + ds := BuildDaemonSet(componentName, podSpec, mutators) // Create or update the DaemonSet in the API Server, and retry selfHostingFailureThreshold times if it errors out if err := apiclient.TryRunCommand(func() error { @@ -105,7 +101,7 @@ func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubea } // Wait for the self-hosted component to come up - if err := waiter.WaitForPodsWithLabel(buildSelfHostedWorkloadLabelQuery(componentName)); err != nil { + if err := waiter.WaitForPodsWithLabel(BuildSelfHostedComponentLabelQuery(componentName)); err != nil { return err } @@ -132,8 +128,8 @@ func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubea return nil } -// buildDaemonSet is responsible for mutating the PodSpec and return a DaemonSet which is suitable for the self-hosting purporse -func buildDaemonSet(name string, podSpec *v1.PodSpec, mutators map[string][]PodSpecMutatorFunc) *extensions.DaemonSet { +// BuildDaemonSet is responsible for mutating the PodSpec and return a DaemonSet which is suitable for the self-hosting purporse +func BuildDaemonSet(name string, podSpec *v1.PodSpec, mutators map[string][]PodSpecMutatorFunc) *extensions.DaemonSet { // Mutate the PodSpec so it's suitable for self-hosting mutatePodSpec(mutators, name, podSpec) @@ -143,19 +139,19 @@ func buildDaemonSet(name string, podSpec *v1.PodSpec, mutators map[string][]PodS ObjectMeta: metav1.ObjectMeta{ Name: kubeadmconstants.AddSelfHostedPrefix(name), Namespace: metav1.NamespaceSystem, - Labels: map[string]string{ - "k8s-app": kubeadmconstants.AddSelfHostedPrefix(name), - }, + Labels: BuildSelfhostedComponentLabels(name), }, Spec: extensions.DaemonSetSpec{ Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "k8s-app": kubeadmconstants.AddSelfHostedPrefix(name), - }, + Labels: BuildSelfhostedComponentLabels(name), }, Spec: *podSpec, }, + UpdateStrategy: extensions.DaemonSetUpdateStrategy{ + // Make the DaemonSet utilize the RollingUpdate rollout strategy + Type: extensions.RollingUpdateDaemonSetStrategyType, + }, }, } } @@ -176,7 +172,13 @@ func loadPodSpecFromFile(manifestPath string) (*v1.PodSpec, error) { return &staticPod.Spec, nil } -// buildSelfHostedWorkloadLabelQuery creates the right query for matching a self-hosted Pod -func buildSelfHostedWorkloadLabelQuery(componentName string) string { +func BuildSelfhostedComponentLabels(component string) map[string]string { + return map[string]string{ + "k8s-app": kubeadmconstants.AddSelfHostedPrefix(component), + } +} + +// BuildSelfHostedComponentLabelQuery creates the right query for matching a self-hosted Pod +func BuildSelfHostedComponentLabelQuery(componentName string) string { return fmt.Sprintf("k8s-app=%s", kubeadmconstants.AddSelfHostedPrefix(componentName)) } diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go index 3ac5a6e6ad7e6..1844a1de0569a 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go @@ -471,7 +471,7 @@ func TestBuildDaemonSet(t *testing.T) { t.Fatalf("couldn't load the specified Pod") } - ds := buildDaemonSet(rt.component, podSpec, getDefaultMutators()) + ds := BuildDaemonSet(rt.component, podSpec, GetDefaultMutators()) dsBytes, err := yaml.Marshal(ds) if err != nil { t.Fatalf("failed to marshal daemonset to YAML: %v", err) diff --git a/cmd/kubeadm/app/phases/upgrade/prepull.go b/cmd/kubeadm/app/phases/upgrade/prepull.go index 99f2c7de6f2bf..9ff8f62e56b88 100644 --- a/cmd/kubeadm/app/phases/upgrade/prepull.go +++ b/cmd/kubeadm/app/phases/upgrade/prepull.go @@ -78,11 +78,7 @@ func (d *DaemonSetPrepuller) WaitFunc(component string) { // DeleteFunc deletes the DaemonSet used for making the image available on every relevant node func (d *DaemonSetPrepuller) DeleteFunc(component string) error { dsName := addPrepullPrefix(component) - foregroundDelete := metav1.DeletePropagationForeground - deleteOptions := &metav1.DeleteOptions{ - PropagationPolicy: &foregroundDelete, - } - if err := d.client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Delete(dsName, deleteOptions); err != nil { + if err := apiclient.DeleteDaemonSetForeground(d.client, metav1.NamespaceSystem, dsName); err != nil { return fmt.Errorf("unable to cleanup the DaemonSet used for prepulling %s: %v", component, err) } fmt.Printf("[upgrade/prepull] Prepulled image for component %s.\n", component) diff --git a/cmd/kubeadm/app/phases/upgrade/selfhosted.go b/cmd/kubeadm/app/phases/upgrade/selfhosted.go new file mode 100644 index 0000000000000..6cd4822077eb1 --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/selfhosted.go @@ -0,0 +1,272 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "fmt" + "time" + + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + "k8s.io/kubernetes/pkg/util/version" +) + +const ( + // upgradeTempDSPrefix is the prefix added to the temporary DaemonSet's name used during the upgrade + upgradeTempDSPrefix = "temp-upgrade-" + + // upgradeTempLabel is the label key used for identifying the temporary component's DaemonSet + upgradeTempLabel = "temp-upgrade-component" + + // selfHostingWaitTimeout describes the maximum amount of time a self-hosting wait process should wait before timing out + selfHostingWaitTimeout = 2 * time.Minute + + // selfHostingFailureThreshold describes how many times kubeadm will retry creating the DaemonSets + selfHostingFailureThreshold uint8 = 10 +) + +// controlPlaneComponentResources holds the relevant Pod and DaemonSet associated with a control plane component +type controlPlaneComponentResources struct { + pod *v1.Pod + daemonSet *extensions.DaemonSet +} + +// SelfHostedControlPlane upgrades a self-hosted control plane +// It works as follows: +// - The client gets the currently running DaemonSets and their associated Pods used for self-hosting the control plane +// - A temporary DaemonSet for the component in question is created; but nearly identical to the DaemonSet for the self-hosted component running right now +// - Why use this temporary DaemonSet? Because, the RollingUpdate strategy for upgrading DaemonSets first kills the old Pod, and then adds the new one +// - This doesn't work for self-hosted upgrades, as if you remove the only API server for instance you have in the cluster, the cluster essentially goes down +// - So instead, a nearly identical copy of the pre-upgrade DaemonSet is created and applied to the cluster. In the beginning, this duplicate DS is just idle +// - kubeadm waits for the temporary DaemonSet's Pod to become Running +// - kubeadm updates the real, self-hosted component. This will result in the pre-upgrade component Pod being removed from the cluster +// - Luckily, the temporary, backup DaemonSet now kicks in and takes over and acts as the control plane. It recognizes that a new Pod should be created, +// - as the "real" DaemonSet is being updated. +// - kubeadm waits for the pre-upgrade Pod to become deleted. It now takes advantage of the backup/temporary component +// - kubeadm waits for the new, upgraded DaemonSet to become Running. +// - Now that the new, upgraded DaemonSet is Running, we can delete the backup/temporary DaemonSet +// - Lastly, make sure the API /healthz endpoint still is reachable +// +// TL;DR; This is what the flow looks like in pseudo-code: +// for [kube-apiserver, kube-controller-manager, kube-scheduler], do: +// 1. Self-Hosted component v1 Running +// -> Duplicate the DaemonSet manifest +// 2. Self-Hosted component v1 Running (active). Backup component v1 Running (passive) +// -> Upgrade the Self-Hosted component v1 to v2. +// -> Self-Hosted component v1 is Deleted from the cluster +// 3. Backup component v1 Running becomes active and completes the upgrade by creating the Self-Hosted component v2 Pod (passive) +// -> Wait for Self-Hosted component v2 to become Running +// 4. Backup component v1 Running (active). Self-Hosted component v2 Running (passive) +// -> Backup component v1 is Deleted +// 5. Wait for Self-Hosted component v2 Running to become active +// 6. Repeat for all control plane components +func SelfHostedControlPlane(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) error { + + // Adjust the timeout slightly to something self-hosting specific + waiter.SetTimeout(selfHostingWaitTimeout) + + // This function returns a map of DaemonSet objects ready to post to the API server + newControlPlaneDaemonSets := BuildUpgradedDaemonSetsFromConfig(cfg, k8sVersion) + + controlPlaneResources, err := getCurrentControlPlaneComponentResources(client) + if err != nil { + return err + } + + for _, component := range constants.MasterComponents { + // Make a shallow copy of the current DaemonSet in order to create a new, temporary one + tempDS := *controlPlaneResources[component].daemonSet + + // Mutate the temp daemonset a little to be suitable for this usage (change label selectors, etc) + mutateTempDaemonSet(&tempDS, component) + + // Create or update the DaemonSet in the API Server, and retry selfHostingFailureThreshold times if it errors out + if err := apiclient.TryRunCommand(func() error { + return apiclient.CreateOrUpdateDaemonSet(client, &tempDS) + }, selfHostingFailureThreshold); err != nil { + return err + } + + // Wait for the temporary/backup self-hosted component to come up + if err := waiter.WaitForPodsWithLabel(buildTempUpgradeDSLabelQuery(component)); err != nil { + return err + } + + newDS := newControlPlaneDaemonSets[component] + + // Upgrade the component's self-hosted resource + // During this upgrade; the temporary/backup component will take over + if err := apiclient.TryRunCommand(func() error { + + if _, err := client.ExtensionsV1beta1().DaemonSets(newDS.ObjectMeta.Namespace).Update(newDS); err != nil { + return fmt.Errorf("couldn't update self-hosted component's DaemonSet: %v", err) + } + return nil + }, selfHostingFailureThreshold); err != nil { + return err + } + + // Wait for the component's old Pod to disappear + oldPod := controlPlaneResources[component].pod + if err := waiter.WaitForPodToDisappear(oldPod.ObjectMeta.Name); err != nil { + return err + } + + // Wait for the main, upgraded self-hosted component to come up + // Here we're talking to the temporary/backup component; the upgraded component is in the process of starting up + if err := waiter.WaitForPodsWithLabel(selfhosting.BuildSelfHostedComponentLabelQuery(component)); err != nil { + return err + } + + // Delete the temporary DaemonSet, and retry selfHostingFailureThreshold times if it errors out + // In order to pivot back to the upgraded API server, we kill the temporary/backup component + if err := apiclient.TryRunCommand(func() error { + return apiclient.DeleteDaemonSetForeground(client, tempDS.ObjectMeta.Namespace, tempDS.ObjectMeta.Name) + }, selfHostingFailureThreshold); err != nil { + return err + } + + // Just as an extra safety check; make sure the API server is returning ok at the /healthz endpoint + if err := waiter.WaitForAPI(); err != nil { + return err + } + + fmt.Printf("[upgrade/apply] Self-hosted component %q upgraded successfully!\n", component) + } + return nil +} + +// BuildUpgradedDaemonSetsFromConfig takes a config object and the current version and returns the DaemonSet objects to post to the master +func BuildUpgradedDaemonSetsFromConfig(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) map[string]*extensions.DaemonSet { + // Here the map of different mutators to use for the control plane's podspec is stored + mutators := selfhosting.GetMutatorsFromFeatureGates(cfg.FeatureFlags) + // Get the new PodSpecs to use + controlPlanePods := controlplane.GetStaticPodSpecs(cfg, k8sVersion) + // Store the created DaemonSets in this map + controlPlaneDaemonSets := map[string]*extensions.DaemonSet{} + + for _, component := range constants.MasterComponents { + podSpec := controlPlanePods[component].Spec + + // Build the full DaemonSet object from the PodSpec generated from the control plane phase and + // using the self-hosting mutators available from the selfhosting phase + ds := selfhosting.BuildDaemonSet(component, &podSpec, mutators) + controlPlaneDaemonSets[component] = ds + } + return controlPlaneDaemonSets +} + +// addTempUpgradeDSPrefix adds the upgradeTempDSPrefix to the specified DaemonSet name +func addTempUpgradeDSPrefix(currentName string) string { + return fmt.Sprintf("%s%s", upgradeTempDSPrefix, currentName) +} + +// buildTempUpgradeLabels returns the label string-string map for identifying the temporary +func buildTempUpgradeLabels(component string) map[string]string { + return map[string]string{ + upgradeTempLabel: component, + } +} + +// buildTempUpgradeDSLabelQuery creates the right query for matching +func buildTempUpgradeDSLabelQuery(component string) string { + return fmt.Sprintf("%s=%s", upgradeTempLabel, component) +} + +// mutateTempDaemonSet mutates the specified self-hosted DaemonSet for the specified component +// in a way that makes it possible to post a nearly identical, temporary DaemonSet as a backup +func mutateTempDaemonSet(tempDS *extensions.DaemonSet, component string) { + // Prefix the name of the temporary DaemonSet with upgradeTempDSPrefix + tempDS.ObjectMeta.Name = addTempUpgradeDSPrefix(tempDS.ObjectMeta.Name) + // Set .Labels to something else than the "real" self-hosted components have + tempDS.ObjectMeta.Labels = buildTempUpgradeLabels(component) + tempDS.Spec.Selector.MatchLabels = buildTempUpgradeLabels(component) + tempDS.Spec.Template.ObjectMeta.Labels = buildTempUpgradeLabels(component) + // Clean all unnecessary ObjectMeta fields + tempDS.ObjectMeta = extractRelevantObjectMeta(tempDS.ObjectMeta) + // Reset .Status as we're posting a new object + tempDS.Status = extensions.DaemonSetStatus{} +} + +// extractRelevantObjectMeta returns only the relevant parts of ObjectMeta required when creating +// a new, identical resource. We should not POST ResourceVersion, UUIDs, etc., only the name, labels, +// namespace and annotations should be preserved. +func extractRelevantObjectMeta(ob metav1.ObjectMeta) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Name: ob.Name, + Namespace: ob.Namespace, + Labels: ob.Labels, + Annotations: ob.Annotations, + } +} + +// listPodsWithLabelSelector returns the relevant Pods for the given LabelSelector +func listPodsWithLabelSelector(client clientset.Interface, kvLabel string) (*v1.PodList, error) { + return client.CoreV1().Pods(metav1.NamespaceSystem).List(metav1.ListOptions{ + LabelSelector: kvLabel, + }) +} + +// getCurrentControlPlaneComponentResources returns a string-(Pod|DaemonSet) map for later use +func getCurrentControlPlaneComponentResources(client clientset.Interface) (map[string]controlPlaneComponentResources, error) { + controlPlaneResources := map[string]controlPlaneComponentResources{} + + for _, component := range constants.MasterComponents { + var podList *v1.PodList + var currentDS *extensions.DaemonSet + + // Get the self-hosted pod associated with the component + podLabelSelector := selfhosting.BuildSelfHostedComponentLabelQuery(component) + if err := apiclient.TryRunCommand(func() error { + var tryrunerr error + podList, tryrunerr = listPodsWithLabelSelector(client, podLabelSelector) + return tryrunerr // note that tryrunerr is most likely nil here (in successful cases) + }, selfHostingFailureThreshold); err != nil { + return nil, err + } + + // Make sure that there are only one Pod with this label selector; otherwise unexpected things can happen + if len(podList.Items) > 1 { + return nil, fmt.Errorf("too many pods with label selector %q found in the %s namespace", podLabelSelector, metav1.NamespaceSystem) + } + + // Get the component's DaemonSet object + dsName := constants.AddSelfHostedPrefix(component) + if err := apiclient.TryRunCommand(func() error { + var tryrunerr error + // Try to get the current self-hosted component + currentDS, tryrunerr = client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Get(dsName, metav1.GetOptions{}) + return tryrunerr // note that tryrunerr is most likely nil here (in successful cases) + }, selfHostingFailureThreshold); err != nil { + return nil, err + } + + // Add the associated resources to the map to return later + controlPlaneResources[component] = controlPlaneComponentResources{ + pod: &podList.Items[0], + daemonSet: currentDS, + } + } + return controlPlaneResources, nil +} diff --git a/cmd/kubeadm/app/util/apiclient/idempotency.go b/cmd/kubeadm/app/util/apiclient/idempotency.go index ae5886bf4d39c..7929c75e65f21 100644 --- a/cmd/kubeadm/app/util/apiclient/idempotency.go +++ b/cmd/kubeadm/app/util/apiclient/idempotency.go @@ -97,6 +97,15 @@ func CreateOrUpdateDaemonSet(client clientset.Interface, ds *extensions.DaemonSe return nil } +// DeleteDaemonSetForeground deletes the specified DaemonSet in foreground mode; i.e. it blocks until/makes sure all the managed Pods are deleted +func DeleteDaemonSetForeground(client clientset.Interface, namespace, name string) error { + foregroundDelete := metav1.DeletePropagationForeground + deleteOptions := &metav1.DeleteOptions{ + PropagationPolicy: &foregroundDelete, + } + return client.ExtensionsV1beta1().DaemonSets(namespace).Delete(name, deleteOptions) +} + // CreateOrUpdateRole creates a Role if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. func CreateOrUpdateRole(client clientset.Interface, role *rbac.Role) error { if _, err := client.RbacV1beta1().Roles(role.ObjectMeta.Namespace).Create(role); err != nil { diff --git a/cmd/kubeadm/app/util/apiclient/wait.go b/cmd/kubeadm/app/util/apiclient/wait.go index 4dddffed22925..5690572b18d7f 100644 --- a/cmd/kubeadm/app/util/apiclient/wait.go +++ b/cmd/kubeadm/app/util/apiclient/wait.go @@ -116,7 +116,7 @@ func (w *KubeWaiter) WaitForPodToDisappear(podName string) error { return wait.PollImmediate(constants.APICallRetryInterval, w.timeout, func() (bool, error) { _, err := w.client.CoreV1().Pods(metav1.NamespaceSystem).Get(podName, metav1.GetOptions{}) if apierrors.IsNotFound(err) { - fmt.Printf("[apiclient] The Static Pod %q is now removed\n", podName) + fmt.Printf("[apiclient] The old Pod %q is now removed (which is desired)\n", podName) return true, nil } return false, nil