Merge pull request #914 from ironcladlou/api-wait-for-etcd

kas: wait for etcd client service name before allowing startup
openshift · Jan 24, 2022 · 036ac65 · 036ac65
2 parents ca5f6d6 + 99e829f
commit 036ac65
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 3 deletions.
diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go
@@ -15,7 +15,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 
-	//TODO: Switch to k8s.io/api/policy/v1 when all management clusters at 1.21+ OR 4.8_openshift+
+	// TODO: Switch to k8s.io/api/policy/v1 when all management clusters at 1.21+ OR 4.8_openshift+
 	policyv1beta1 "k8s.io/api/policy/v1beta1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -1263,7 +1263,7 @@ func (r *HostedControlPlaneReconciler) reconcileManagedEtcd(ctx context.Context,
 }
 
 func (r *HostedControlPlaneReconciler) reconcileUnmanagedEtcd(ctx context.Context, hcp *hyperv1.HostedControlPlane) error {
-	//reconcile client secret over
+	// reconcile client secret over
 	if hcp.Spec.Etcd.Unmanaged == nil || len(hcp.Spec.Etcd.Unmanaged.TLS.ClientSecret.Name) == 0 || len(hcp.Spec.Etcd.Unmanaged.Endpoint) == 0 {
 		return fmt.Errorf("etcd metadata not specified for unmanaged deployment")
 	}
@@ -1505,6 +1505,7 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex
 			hcp.Spec.SecretEncryption,
 			aesCBCActiveKey,
 			aesCBCBackupKey,
+			hcp.Spec.Etcd.ManagementType,
 		)
 	}); err != nil {
 		return fmt.Errorf("failed to reconcile api server deployment: %w", err)

diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go
@@ -91,6 +91,7 @@ func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment,
 	secretEncryptionData *hyperv1.SecretEncryptionSpec,
 	aesCBCActiveKey []byte,
 	aesCBCBackupKey []byte,
+	etcdMgmtType hyperv1.EtcdManagementType,
 ) error {
 
 	configBytes, ok := config.Data[KubeAPIServerConfigKey]
@@ -171,6 +172,15 @@ func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment,
 			},
 		},
 	}
+
+	// With managed etcd, we should wait for the known etcd client service name to
+	// at least resolve before starting up to avoid futile connection attempts and
+	// pod crashing. For unmanaged, make no assumptions.
+	if etcdMgmtType == hyperv1.Managed {
+		deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers,
+			util.BuildContainer(kasContainerWaitForEtcd(), buildKASContainerWaitForEtcd(images.CLI, deployment.Namespace)))
+	}
+
 	if len(images.Portieris) > 0 {
 		applyPortieriesConfig(&deployment.Spec.Template.Spec, images.Portieris)
 	}
@@ -209,7 +219,7 @@ func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment,
 				return err
 			}
 		default:
-			//nothing needed to be done
+			// nothing needed to be done
 		}
 	}
 	deploymentConfig.ApplyTo(deployment)
@@ -268,6 +278,28 @@ func buildKASContainerApplyBootstrap(image string) func(c *corev1.Container) {
 	}
 }
 
+func kasContainerWaitForEtcd() *corev1.Container {
+	return &corev1.Container{
+		Name: "wait-for-etcd",
+	}
+}
+
+func buildKASContainerWaitForEtcd(image string, namespace string) func(c *corev1.Container) {
+	return func(c *corev1.Container) {
+		c.Image = image
+		c.TerminationMessagePolicy = corev1.TerminationMessageReadFile
+		c.TerminationMessagePath = corev1.TerminationMessagePathDefault
+		c.ImagePullPolicy = corev1.PullIfNotPresent
+		c.Command = []string{
+			"/bin/bash",
+		}
+		c.Args = []string{
+			"-c",
+			waitForEtcdScript(namespace),
+		}
+	}
+}
+
 func kasContainerMain() *corev1.Container {
 	return &corev1.Container{
 		Name: "kube-apiserver",
@@ -593,6 +625,13 @@ done
 	return fmt.Sprintf(script, workDir)
 }
 
+func waitForEtcdScript(namespace string) string {
+	var script = `#!/bin/sh
+while ! nslookup etcd-client.%s.svc; do sleep 1; done
+`
+	return fmt.Sprintf(script, namespace)
+}
+
 func applyNamedCertificateMounts(certs []configv1.APIServerNamedServingCert, spec *corev1.PodSpec) {
 	var container *corev1.Container
 	for i := range spec.Containers {