Merge pull request kubernetes#128532 from neolit123/1.32-handle-custo…

…m-addreses-comp-readyz kubeadm: use advertise address for WaitForAllControlPlaneComponents
ii · Nov 6, 2024 · 3dcad5f · 3dcad5f
2 parents aafcf4e + 0cfcaa8
commit 3dcad5f
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 28 deletions.
diff --git a/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go b/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go
@@ -124,7 +124,8 @@ func runWaitControlPlanePhase(c workflow.RunData) error {
 
 	waiter.SetTimeout(data.Cfg().Timeouts.ControlPlaneComponentHealthCheck.Duration)
 	if features.Enabled(data.Cfg().ClusterConfiguration.FeatureGates, features.WaitForAllControlPlaneComponents) {
-		err = waiter.WaitForControlPlaneComponents(&data.Cfg().ClusterConfiguration)
+		err = waiter.WaitForControlPlaneComponents(&data.Cfg().ClusterConfiguration,
+			data.Cfg().LocalAPIEndpoint.AdvertiseAddress)
 	} else {
 		err = waiter.WaitForAPI()
 	}

diff --git a/cmd/kubeadm/app/cmd/phases/join/waitcontrolplane.go b/cmd/kubeadm/app/cmd/phases/join/waitcontrolplane.go
@@ -71,7 +71,8 @@ func runWaitControlPlanePhase(c workflow.RunData) error {
 	}
 
 	waiter.SetTimeout(data.Cfg().Timeouts.ControlPlaneComponentHealthCheck.Duration)
-	if err := waiter.WaitForControlPlaneComponents(&initCfg.ClusterConfiguration); err != nil {
+	if err := waiter.WaitForControlPlaneComponents(&initCfg.ClusterConfiguration,
+		data.Cfg().ControlPlane.LocalAPIEndpoint.AdvertiseAddress); err != nil {
 		return err
 	}
 

diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go
@@ -99,7 +99,7 @@ func NewFakeStaticPodWaiter(errsToReturn map[string]error) apiclient.Waiter {
 }
 
 // WaitForControlPlaneComponents just returns a dummy nil, to indicate that the program should just proceed
-func (w *fakeWaiter) WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration) error {
+func (w *fakeWaiter) WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, apiServerAddress string) error {
 	return nil
 }
 

diff --git a/cmd/kubeadm/app/util/apiclient/wait.go b/cmd/kubeadm/app/util/apiclient/wait.go
@@ -21,6 +21,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"time"
 
@@ -40,7 +41,7 @@ import (
 // Waiter is an interface for waiting for criteria in Kubernetes to happen
 type Waiter interface {
 	// WaitForControlPlaneComponents waits for all control plane components to be ready.
-	WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration) error
+	WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, apiServerAddress string) error
 	// WaitForAPI waits for the API Server's /healthz endpoint to become "ok"
 	// TODO: remove WaitForAPI once WaitForAllControlPlaneComponents goes GA:
 	// https://github.com/kubernetes/kubeadm/issues/2907
@@ -91,11 +92,17 @@ const (
 
 // getControlPlaneComponents takes a ClusterConfiguration and returns a slice of
 // control plane components and their health check URLs.
-func getControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration) []controlPlaneComponent {
+func getControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, defaultAddressAPIServer string) []controlPlaneComponent {
 	const (
 		portArg        = "secure-port"
-		addressArg     = "bind-address"
-		defaultAddress = "127.0.0.1"
+		bindAddressArg = "bind-address"
+		// By default, for kube-api-server, kubeadm does not apply a --bind-address flag.
+		// Check --advertise-address instead, which can override the defaultAddressAPIServer value.
+		advertiseAddressArg = "advertise-address"
+		// By default kubeadm deploys the kube-controller-manager and kube-scheduler
+		// with --bind-address=127.0.0.1. This should match get{Scheduler|ControllerManager}Command().
+		defaultAddressKCM       = "127.0.0.1"
+		defaultAddressScheduler = "127.0.0.1"
 	)
 
 	portAPIServer, idx := kubeadmapi.GetArgValue(cfg.APIServer.ExtraArgs, portArg, -1)
@@ -111,33 +118,39 @@ func getControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration) []controlPl
 		portScheduler = fmt.Sprintf("%d", constants.KubeSchedulerPort)
 	}
 
-	addressAPIServer, idx := kubeadmapi.GetArgValue(cfg.APIServer.ExtraArgs, addressArg, -1)
+	addressAPIServer, idx := kubeadmapi.GetArgValue(cfg.APIServer.ExtraArgs, advertiseAddressArg, -1)
 	if idx == -1 {
-		addressAPIServer = defaultAddress
+		addressAPIServer = defaultAddressAPIServer
 	}
-	addressKCM, idx := kubeadmapi.GetArgValue(cfg.ControllerManager.ExtraArgs, addressArg, -1)
+	addressKCM, idx := kubeadmapi.GetArgValue(cfg.ControllerManager.ExtraArgs, bindAddressArg, -1)
 	if idx == -1 {
-		addressKCM = defaultAddress
+		addressKCM = defaultAddressKCM
 	}
-	addressScheduler, idx := kubeadmapi.GetArgValue(cfg.Scheduler.ExtraArgs, addressArg, -1)
+	addressScheduler, idx := kubeadmapi.GetArgValue(cfg.Scheduler.ExtraArgs, bindAddressArg, -1)
 	if idx == -1 {
-		addressScheduler = defaultAddress
+		addressScheduler = defaultAddressScheduler
 	}
 
-	urlFormat := "https://%s:%s/%s"
+	getURL := func(address, port, endpoint string) string {
+		return fmt.Sprintf(
+			"https://%s/%s",
+			net.JoinHostPort(address, port),
+			endpoint,
+		)
+	}
 	return []controlPlaneComponent{
-		{name: "kube-apiserver", url: fmt.Sprintf(urlFormat, addressAPIServer, portAPIServer, endpointLivez)},
-		{name: "kube-controller-manager", url: fmt.Sprintf(urlFormat, addressKCM, portKCM, endpointHealthz)},
-		{name: "kube-scheduler", url: fmt.Sprintf(urlFormat, addressScheduler, portScheduler, endpointLivez)},
+		{name: "kube-apiserver", url: getURL(addressAPIServer, portAPIServer, endpointLivez)},
+		{name: "kube-controller-manager", url: getURL(addressKCM, portKCM, endpointHealthz)},
+		{name: "kube-scheduler", url: getURL(addressScheduler, portScheduler, endpointLivez)},
 	}
 }
 
 // WaitForControlPlaneComponents waits for all control plane components to report "ok".
-func (w *KubeWaiter) WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration) error {
+func (w *KubeWaiter) WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, apiSeverAddress string) error {
 	fmt.Printf("[control-plane-check] Waiting for healthy control plane components."+
 		" This can take up to %v\n", w.timeout)
 
-	components := getControlPlaneComponents(cfg)
+	components := getControlPlaneComponents(cfg, apiSeverAddress)
 
 	var errs []error
 	errChan := make(chan error, len(components))

diff --git a/cmd/kubeadm/app/util/apiclient/wait_test.go b/cmd/kubeadm/app/util/apiclient/wait_test.go
@@ -37,34 +37,34 @@ func TestGetControlPlaneComponents(t *testing.T) {
 					ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
 						ExtraArgs: []kubeadmapi.Arg{
 							{Name: "secure-port", Value: "1111"},
-							{Name: "bind-address", Value: "0.0.0.0"},
+							{Name: "advertise-address", Value: "fd00:1::"},
 						},
 					},
 				},
 				ControllerManager: kubeadmapi.ControlPlaneComponent{
 					ExtraArgs: []kubeadmapi.Arg{
 						{Name: "secure-port", Value: "2222"},
-						{Name: "bind-address", Value: "0.0.0.0"},
+						{Name: "bind-address", Value: "127.0.0.1"},
 					},
 				},
 				Scheduler: kubeadmapi.ControlPlaneComponent{
 					ExtraArgs: []kubeadmapi.Arg{
 						{Name: "secure-port", Value: "3333"},
-						{Name: "bind-address", Value: "0.0.0.0"},
+						{Name: "bind-address", Value: "127.0.0.1"},
 					},
 				},
 			},
 			expected: []controlPlaneComponent{
-				{name: "kube-apiserver", url: fmt.Sprintf("https://0.0.0.0:1111/%s", endpointLivez)},
-				{name: "kube-controller-manager", url: fmt.Sprintf("https://0.0.0.0:2222/%s", endpointHealthz)},
-				{name: "kube-scheduler", url: fmt.Sprintf("https://0.0.0.0:3333/%s", endpointLivez)},
+				{name: "kube-apiserver", url: fmt.Sprintf("https://[fd00:1::]:1111/%s", endpointLivez)},
+				{name: "kube-controller-manager", url: fmt.Sprintf("https://127.0.0.1:2222/%s", endpointHealthz)},
+				{name: "kube-scheduler", url: fmt.Sprintf("https://127.0.0.1:3333/%s", endpointLivez)},
 			},
 		},
 		{
 			name: "default ports and addresses",
 			cfg:  &kubeadmapi.ClusterConfiguration{},
 			expected: []controlPlaneComponent{
-				{name: "kube-apiserver", url: fmt.Sprintf("https://127.0.0.1:6443/%s", endpointLivez)},
+				{name: "kube-apiserver", url: fmt.Sprintf("https://192.168.0.1:6443/%s", endpointLivez)},
 				{name: "kube-controller-manager", url: fmt.Sprintf("https://127.0.0.1:10257/%s", endpointHealthz)},
 				{name: "kube-scheduler", url: fmt.Sprintf("https://127.0.0.1:10259/%s", endpointLivez)},
 			},
@@ -73,7 +73,7 @@ func TestGetControlPlaneComponents(t *testing.T) {
 
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
-			actual := getControlPlaneComponents(tc.cfg)
+			actual := getControlPlaneComponents(tc.cfg, "192.168.0.1")
 			if !reflect.DeepEqual(tc.expected, actual) {
 				t.Fatalf("expected result: %+v, got: %+v", tc.expected, actual)
 			}

diff --git a/cmd/kubeadm/app/util/dryrun/dryrun.go b/cmd/kubeadm/app/util/dryrun/dryrun.go
@@ -90,7 +90,7 @@ func NewWaiter() apiclient.Waiter {
 }
 
 // WaitForControlPlaneComponents just returns a dummy nil, to indicate that the program should just proceed
-func (w *Waiter) WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration) error {
+func (w *Waiter) WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, apiServerAddress string) error {
 	return nil
 }