kubernetes · kmala · Sep 21, 2023 · Nov 7, 2023 · Mar 12, 2024 · liggitt
diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list
@@ -138,6 +138,11 @@ API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudS
 API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudSharedConfiguration,NodeSyncPeriod
 API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudSharedConfiguration,RouteReconciliationPeriod
 API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudSharedConfiguration,UseServiceAccountCredentials
+API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,WebhookConfiguration,CaBundle
+API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,WebhookConfiguration,MutatingWebhookConfiguration
+API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,WebhookConfiguration,ValidatingWebhookConfiguration
+API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,WebhookConfiguration,WebhookAddress
+API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,WebhookConfiguration,WebhookPort
 API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,WebhookConfiguration,Webhooks
 API rule violation: names_match,k8s.io/controller-manager/config/v1alpha1,GenericControllerManagerConfiguration,Address
 API rule violation: names_match,k8s.io/controller-manager/config/v1alpha1,GenericControllerManagerConfiguration,ClientConnection

diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go
diff --git a/staging/src/k8s.io/cloud-provider/app/builder.go b/staging/src/k8s.io/cloud-provider/app/builder.go
@@ -140,7 +140,8 @@ func (cb *CommandBuilder) BuildCommand() *cobra.Command {
 			cliflag.PrintFlags(cmd.Flags())
 
 			config, err := cb.options.Config(ControllerNames(cb.controllerInitFuncConstructors), ControllersDisabledByDefault.List(),
-				cb.controllerAliases, WebhookNames(cb.webhookConfigs), WebhooksDisabledByDefault.List())
+				cb.controllerAliases, WebhookNamesByType(cb.webhookConfigs, ValidatingWebhook), WebhookNamesByType(cb.webhookConfigs, MutatingWebhook),
+				WebhooksDisabledByDefault.List())
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%v\n", err)
 				return err

diff --git a/staging/src/k8s.io/cloud-provider/app/controllermanager.go b/staging/src/k8s.io/cloud-provider/app/controllermanager.go
@@ -22,10 +22,15 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
 
+	v1 "k8s.io/api/admissionregistration/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/uuid"
@@ -42,6 +47,7 @@ import (
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	cloudprovider "k8s.io/cloud-provider"
 	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
+	cloudproviderconfig "k8s.io/cloud-provider/config"
 	"k8s.io/cloud-provider/names"
 	"k8s.io/cloud-provider/options"
 	cliflag "k8s.io/component-base/cli/flag"
@@ -96,7 +102,7 @@ the cloud specific control loops shipped with Kubernetes.`,
 				return err
 			}
 
-			c, err := s.Config(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List(), controllerAliases, AllWebhooks, DisabledByDefaultWebhooks)
+			c, err := s.Config(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List(), controllerAliases, AllWebhooks, AllWebhooks, DisabledByDefaultWebhooks)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%v\n", err)
 				return err
@@ -162,6 +168,97 @@ the cloud specific control loops shipped with Kubernetes.`,
 	return cmd
 }
 
+func createOrUpdateWebhookConfiguration(ctx context.Context, webhooks map[string]WebhookHandler, webhooksConfig cloudproviderconfig.WebhookConfiguration, clientBuilder clientbuilder.SimpleControllerClientBuilder) error {
+	errors := []error{}
+	if webhooksConfig.ValidatingWebhookConfiguration != nil {
+		for i, webhook := range webhooksConfig.ValidatingWebhookConfiguration.Webhooks {
+			webhookconfig, ok := webhooks[webhook.Name]
+			if !ok {
+				errors = append(errors, fmt.Errorf("webhook configuration not found for webhook %s", webhook.Name))
+				continue
+			}
+			url := fmt.Sprintf("https://%s:%d/%s", webhooksConfig.WebhookAddress, webhooksConfig.WebhookPort, strings.TrimPrefix(webhookconfig.Path, "/"))
+
+			webhooksConfig.ValidatingWebhookConfiguration.Webhooks[i].ClientConfig = v1.WebhookClientConfig{
+				URL:      &url,
+				CABundle: []byte(webhooksConfig.CaBundle),
+			}
+		}
+	}
+	if webhooksConfig.MutatingWebhookConfiguration != nil {
+		for i, webhook := range webhooksConfig.MutatingWebhookConfiguration.Webhooks {
+			webhookconfig, ok := webhooks[webhook.Name]
+			if !ok {
+				errors = append(errors, fmt.Errorf("webhook configuration not found for webhook %s", webhook.Name))
+				continue
+			}
+			url := fmt.Sprintf("https://%s:%d/%s", webhooksConfig.WebhookAddress, webhooksConfig.WebhookPort, strings.TrimPrefix(webhookconfig.Path, "/"))
+
+			webhooksConfig.MutatingWebhookConfiguration.Webhooks[i].ClientConfig = v1.WebhookClientConfig{
+				URL:      &url,
+				CABundle: []byte(webhooksConfig.CaBundle),
+			}
+		}
+	}
+	if len(errors) != 0 {
+		return utilerrors.NewAggregate(errors)
+	}
+
+	kubeClient := clientBuilder.ClientOrDie("ccm-webhook")
+
+	if webhooksConfig.ValidatingWebhookConfiguration != nil {
+		_, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(ctx, webhooksConfig.ValidatingWebhookConfiguration, metav1.CreateOptions{})
+		if err == nil {
+			klog.Infoln("validating webhook configuration successfully created")
+		} else {
+			if !apierrors.IsAlreadyExists(err) {
+				klog.ErrorS(err, "Unable to create validating webhook configuration with API server", "webhookconfiguration", klog.KObj(webhooksConfig.ValidatingWebhookConfiguration))
+				return fmt.Errorf("unable to create validating webhook configuration with API server %w", err)
+			}
+
+			currentConfiguration, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, webhooksConfig.ValidatingWebhookConfiguration.Name, metav1.GetOptions{})
+			if err != nil {
+				klog.ErrorS(err, "Unable to create validating webhook configuration with API server, error getting existing webhook configuration", "webhookconfiguration", webhooksConfig.ValidatingWebhookConfiguration.Name)
+				return fmt.Errorf("unable to get validating webhook configuration from API server %w", err)
+			}
+			currentConfiguration.Webhooks = webhooksConfig.ValidatingWebhookConfiguration.Webhooks
+
+			_, err = kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(ctx, currentConfiguration, metav1.UpdateOptions{})
+			if err != nil {
+				klog.ErrorS(err, "Unable to update validating webhook configuration with API server", "webhookconfiguration", klog.KObj(webhooksConfig.ValidatingWebhookConfiguration))
+				return fmt.Errorf("unable to update validating webhook configuration with API server %w", err)
+			}
+		}
+	}
+
+	if webhooksConfig.MutatingWebhookConfiguration != nil {
+		_, err := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(ctx, webhooksConfig.MutatingWebhookConfiguration, metav1.CreateOptions{})
+		if err == nil {
+			klog.Infoln("mutating webhook configuration successfully created")
+		} else {
+			if !apierrors.IsAlreadyExists(err) {
+				klog.ErrorS(err, "Unable to create mutating webhook configuration with API server", "webhookconfiguration", klog.KObj(webhooksConfig.MutatingWebhookConfiguration))
+				return fmt.Errorf("unable to create mutating webhook configuration with API server %w", err)
+			}
+
+			currentConfiguration, err := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, webhooksConfig.MutatingWebhookConfiguration.Name, metav1.GetOptions{})
+			if err != nil {
+				klog.ErrorS(err, "Unable to create mutating webhook configuration with API server, error fetching existing webhook configuration", "webhookconfiguration", webhooksConfig.MutatingWebhookConfiguration.Name)
+				return fmt.Errorf("unable to get mutating webhook configuration from API server %w", err)
+			}
+			currentConfiguration.Webhooks = webhooksConfig.MutatingWebhookConfiguration.Webhooks
+
+			_, err = kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(ctx, currentConfiguration, metav1.UpdateOptions{})
+			if err != nil {
+				klog.ErrorS(err, "Unable to update mutating webhook configuration with API server", "webhookconfiguration", klog.KObj(webhooksConfig.MutatingWebhookConfiguration))
+				return fmt.Errorf("unable to update mutating webhook configuration with API server %w", err)
+			}
+		}
+	}
+
+	return nil
+}
+
 // Run runs the ExternalCMServer.  This should never exit.
 func Run(c *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, controllerInitializers map[string]InitFunc, webhooks map[string]WebhookHandler,
 	stopCh <-chan struct{}) error {
@@ -220,6 +317,14 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface
 		if err != nil {
 			klog.Fatalf("error building controller context: %v", err)
 		}
+		if utilfeature.DefaultFeatureGate.Enabled(cmfeatures.CloudControllerManagerWebhook) {
+			if len(webhooks) > 0 {
+				klog.Info("creating/updating webhook configuration: ", webhooks)
+				if err := createOrUpdateWebhookConfiguration(ctx, webhooks, c.ComponentConfig.Webhook, clientBuilder); err != nil {
+					klog.Fatalf("error creating/updating webhook configuration: %v", err)
+				}
+			}
+		}
 		if err := startControllers(ctx, cloud, controllerContext, c, ctx.Done(), controllerInitializers, healthzHandler); err != nil {
 			klog.Fatalf("error running controllers: %v", err)
 		}

diff --git a/staging/src/k8s.io/cloud-provider/app/webhooks.go b/staging/src/k8s.io/cloud-provider/app/webhooks.go
@@ -53,14 +53,24 @@ func init() {
 // WebhooksDisabledByDefault is the webhooks disabled default when starting cloud-controller managers.
 var WebhooksDisabledByDefault = sets.NewString()
 
+// WebHookType specifies the type of webhook.
+type WebhookType string
+
+const (
+	ValidatingWebhook WebhookType = "ValidatingWebhook"
+	MutatingWebhook   WebhookType = "MutatingWebhook"
+)
+
 type WebhookConfig struct {
 	Path             string
+	Type             WebhookType
 	AdmissionHandler func(*admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error)
 }
 
 type WebhookHandler struct {
 	Name string
 	Path string
+	Type WebhookType
 	http.Handler
 	AdmissionHandler func(*admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error)
 	CompletedConfig  *config.CompletedConfig
@@ -78,6 +88,7 @@ func NewWebhookHandlers(webhookConfigs map[string]WebhookConfig, completedConfig
 		webhookHandlers[name] = WebhookHandler{
 			Name:             name,
 			Path:             config.Path,
+			Type:             config.Type,
 			AdmissionHandler: config.AdmissionHandler,
 			CompletedConfig:  completedConfig,
 			Cloud:            cloud,
@@ -91,6 +102,16 @@ func WebhookNames(webhooks map[string]WebhookConfig) []string {
 	return ret.List()
 }
 
+func WebhookNamesByType(webhooks map[string]WebhookConfig, webhookType WebhookType) []string {
+	var webhookNames []string
+	for name, config := range webhooks {
+		if config.Type == webhookType {
+			webhookNames = append(webhookNames, name)
+		}
+	}
+	return webhookNames
+}
+
 func newHandler(webhooks map[string]WebhookHandler) *mux.PathRecorderMux {
 	mux := mux.NewPathRecorderMux("controller-manager-webhook")
 

diff --git a/staging/src/k8s.io/cloud-provider/config/types.go b/staging/src/k8s.io/cloud-provider/config/types.go
@@ -17,6 +17,7 @@ limitations under the License.
 package config
 
 import (
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	nodeconfig "k8s.io/cloud-provider/controllers/node/config"
 	serviceconfig "k8s.io/cloud-provider/controllers/service/config"
@@ -100,4 +101,15 @@ type WebhookConfiguration struct {
 	// '-foo' means "disable 'foo'"
 	// first item for a particular name wins
 	Webhooks []string
+	// WebhookPort is the port that the controller-manager's webhook service runs on.
+	WebhookPort int32
+	// WebhookAddress is the IP address to serve on.
+	WebhookAddress string
+	// CaBundle is the ca certs to be used by apiserver for validating the webhook server certs.
+	CaBundle string
+	// ValidatingWebhookConfiguration is the config used for creating validation webhook config object.
+	ValidatingWebhookConfiguration *admissionregistrationv1.ValidatingWebhookConfiguration
+
+	// MutatingWebhookConfiguration is the config used for creating mutating webhook config object.
+	MutatingWebhookConfiguration *admissionregistrationv1.MutatingWebhookConfiguration
 }
diff --git a/staging/src/k8s.io/cloud-provider/config/v1alpha1/defaults.go b/staging/src/k8s.io/cloud-provider/config/v1alpha1/defaults.go
@@ -19,6 +19,7 @@ package v1alpha1
 import (
 	"time"
 
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	nodeconfigv1alpha1 "k8s.io/cloud-provider/controllers/node/config/v1alpha1"
@@ -69,3 +70,78 @@ func SetDefaults_KubeCloudSharedConfiguration(obj *KubeCloudSharedConfiguration)
 		obj.RouteReconciliationPeriod = metav1.Duration{Duration: 10 * time.Second}
 	}
 }
+
+// SetDefaults_ValidatingWebhook sets defaults for webhook validating. This function
+// is duplicated from "k8s.io/kubernetes/pkg/apis/admissionregistration/v1/defaults.go"
+// in order for in-tree cloud providers to not depend on internal packages.
+func SetDefaults_ValidatingWebhook(obj *admissionregistrationv1.ValidatingWebhook) {
+	if obj.FailurePolicy == nil {
+		policy := admissionregistrationv1.Fail
+		obj.FailurePolicy = &policy
+	}
+	if obj.MatchPolicy == nil {
+		policy := admissionregistrationv1.Equivalent
+		obj.MatchPolicy = &policy
+	}
+	if obj.NamespaceSelector == nil {
+		selector := metav1.LabelSelector{}
+		obj.NamespaceSelector = &selector
+	}
+	if obj.ObjectSelector == nil {
+		selector := metav1.LabelSelector{}
+		obj.ObjectSelector = &selector
+	}
+	if obj.TimeoutSeconds == nil {
+		obj.TimeoutSeconds = new(int32)
+		*obj.TimeoutSeconds = 10
+	}
+}
+
+// SetDefaults_MutatingWebhook sets defaults for webhook mutating This function
+// is duplicated from "k8s.io/kubernetes/pkg/apis/admissionregistration/v1/defaults.go"
+// in order for in-tree cloud providers to not depend on internal packages.
+func SetDefaults_MutatingWebhook(obj *admissionregistrationv1.MutatingWebhook) {
+	if obj.FailurePolicy == nil {
+		policy := admissionregistrationv1.Fail
+		obj.FailurePolicy = &policy
+	}
+	if obj.MatchPolicy == nil {
+		policy := admissionregistrationv1.Equivalent
+		obj.MatchPolicy = &policy
+	}
+	if obj.NamespaceSelector == nil {
+		selector := metav1.LabelSelector{}
+		obj.NamespaceSelector = &selector
+	}
+	if obj.ObjectSelector == nil {
+		selector := metav1.LabelSelector{}
+		obj.ObjectSelector = &selector
+	}
+	if obj.TimeoutSeconds == nil {
+		obj.TimeoutSeconds = new(int32)
+		*obj.TimeoutSeconds = 10
+	}
+	if obj.ReinvocationPolicy == nil {
+		never := admissionregistrationv1.NeverReinvocationPolicy
+		obj.ReinvocationPolicy = &never
+	}
+}
+
+// SetDefaults_Rule sets defaults for webhook rule This function
+// is duplicated from "k8s.io/kubernetes/pkg/apis/admissionregistration/v1/defaults.go"
+// in order for in-tree cloud providers to not depend on internal packages.
+func SetDefaults_Rule(obj *admissionregistrationv1.Rule) {
+	if obj.Scope == nil {
+		s := admissionregistrationv1.AllScopes
+		obj.Scope = &s
+	}
+}
+
+// SetDefaults_ServiceReference sets defaults for Webhook's ServiceReference This function
+// is duplicated from "k8s.io/kubernetes/pkg/apis/admissionregistration/v1/defaults.go"
+// in order for in-tree cloud providers to not depend on internal packages.
+func SetDefaults_ServiceReference(obj *admissionregistrationv1.ServiceReference) {
+	if obj.Port == nil {
+		obj.Port = utilpointer.Int32(443)
+	}
+}