Merge pull request kubernetes#128190 from HarshalNeelkamal/external-jwt

Add plugin and key-cache for ExternalJWTSigner integration
ii · Nov 7, 2024 · 6cc3570 · 6cc3570
2 parents c462d4c + 6fdacf0
commit 6cc3570
Show file tree

Hide file tree

Showing 62 changed files with 4,536 additions and 139 deletions.
diff --git a/cmd/kube-apiserver/app/options/completion.go b/cmd/kube-apiserver/app/options/completion.go
@@ -17,6 +17,7 @@ limitations under the License.
 package options
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"strings"
@@ -45,27 +46,27 @@ type CompletedOptions struct {
 
 // Complete set default ServerRunOptions.
 // Should be called after kube-apiserver flags parsed.
-func (opts *ServerRunOptions) Complete() (CompletedOptions, error) {
-	if opts == nil {
+func (s *ServerRunOptions) Complete(ctx context.Context) (CompletedOptions, error) {
+	if s == nil {
 		return CompletedOptions{completedOptions: &completedOptions{}}, nil
 	}
 
-	// process opts.ServiceClusterIPRange from list to Primary and Secondary
+	// process s.ServiceClusterIPRange from list to Primary and Secondary
 	// we process secondary only if provided by user
-	apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(opts.ServiceClusterIPRanges)
+	apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(s.ServiceClusterIPRanges)
 	if err != nil {
 		return CompletedOptions{}, err
 	}
-	controlplane, err := opts.Options.Complete([]string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
+	controlplane, err := s.Options.Complete(ctx, []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
 	if err != nil {
 		return CompletedOptions{}, err
 	}
 
 	completed := completedOptions{
 		CompletedOptions: controlplane,
-		CloudProvider:    opts.CloudProvider,
+		CloudProvider:    s.CloudProvider,
 
-		Extra: opts.Extra,
+		Extra: s.Extra,
 	}
 
 	completed.PrimaryServiceClusterIPRange = primaryServiceIPRange

diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go
@@ -67,6 +67,7 @@ func NewAPIServerCommand() *cobra.Command {
 	_, featureGate := featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
 		featuregate.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
 	s := options.NewServerRunOptions()
+	ctx := genericapiserver.SetupSignalContext()
 
 	cmd := &cobra.Command{
 		Use: "kube-apiserver",
@@ -97,7 +98,7 @@ cluster's shared state through which all other components interact.`,
 			cliflag.PrintFlags(fs)
 
 			// set default options
-			completedOptions, err := s.Complete()
+			completedOptions, err := s.Complete(ctx)
 			if err != nil {
 				return err
 			}
@@ -108,7 +109,7 @@ cluster's shared state through which all other components interact.`,
 			}
 			// add feature enablement metrics
 			featureGate.AddMetrics()
-			return Run(cmd.Context(), completedOptions)
+			return Run(ctx, completedOptions)
 		},
 		Args: func(cmd *cobra.Command, args []string) error {
 			for _, arg := range args {
@@ -119,7 +120,7 @@ cluster's shared state through which all other components interact.`,
 			return nil
 		},
 	}
-	cmd.SetContext(genericapiserver.SetupSignalContext())
+	cmd.SetContext(ctx)
 
 	fs := cmd.Flags()
 	namedFlagSets := s.Flags()

diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go
@@ -390,7 +390,7 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
 	s.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
 	s.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
 
-	completedOptions, err := s.Complete()
+	completedOptions, err := s.Complete(tCtx)
 	if err != nil {
 		return result, fmt.Errorf("failed to set default ServerRunOptions: %v", err)
 	}

diff --git a/go.mod b/go.mod
@@ -103,6 +103,7 @@ require (
 	k8s.io/csi-translation-lib v0.0.0
 	k8s.io/dynamic-resource-allocation v0.0.0
 	k8s.io/endpointslice v0.0.0
+	k8s.io/externaljwt v0.0.0
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/kms v0.0.0
 	k8s.io/kube-aggregator v0.0.0
@@ -239,6 +240,7 @@ replace (
 	k8s.io/csi-translation-lib => ./staging/src/k8s.io/csi-translation-lib
 	k8s.io/dynamic-resource-allocation => ./staging/src/k8s.io/dynamic-resource-allocation
 	k8s.io/endpointslice => ./staging/src/k8s.io/endpointslice
+	k8s.io/externaljwt => ./staging/src/k8s.io/externaljwt
 	k8s.io/kms => ./staging/src/k8s.io/kms
 	k8s.io/kube-aggregator => ./staging/src/k8s.io/kube-aggregator
 	k8s.io/kube-controller-manager => ./staging/src/k8s.io/kube-controller-manager

diff --git a/go.work b/go.work
@@ -23,6 +23,7 @@ use (
 	./staging/src/k8s.io/csi-translation-lib
 	./staging/src/k8s.io/dynamic-resource-allocation
 	./staging/src/k8s.io/endpointslice
+	./staging/src/k8s.io/externaljwt
 	./staging/src/k8s.io/kms
 	./staging/src/k8s.io/kube-aggregator
 	./staging/src/k8s.io/kube-controller-manager

diff --git a/hack/unwanted-dependencies.json b/hack/unwanted-dependencies.json
@@ -111,6 +111,7 @@
         "k8s.io/client-go",
         "k8s.io/code-generator",
         "k8s.io/cri-api",
+        "k8s.io/externaljwt",
         "k8s.io/kms",
         "k8s.io/kube-aggregator",
         "k8s.io/kubelet",

diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh
@@ -785,6 +785,8 @@ function codegen::protobindings() {
 
         "staging/src/k8s.io/kubelet/pkg/apis/pluginregistration"
         "pkg/kubelet/pluginmanager/pluginwatcher/example_plugin_apis"
+
+        "staging/src/k8s.io/externaljwt/apis"
     )
 
     kube::log::status "Generating protobuf bindings for ${#apis[@]} targets"

diff --git a/pkg/apis/authentication/validation/validation.go b/pkg/apis/authentication/validation/validation.go
@@ -19,19 +19,18 @@ limitations under the License.
 package validation
 
 import (
-	"time"
-
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/kubernetes/pkg/apis/authentication"
 )
 
+const MinTokenAgeSec = 10 * 60 // 10 minutes
+
 // ValidateTokenRequest validates a TokenRequest.
 func ValidateTokenRequest(tr *authentication.TokenRequest) field.ErrorList {
 	allErrs := field.ErrorList{}
 	specPath := field.NewPath("spec")
 
-	const min = 10 * time.Minute
-	if tr.Spec.ExpirationSeconds < int64(min.Seconds()) {
+	if tr.Spec.ExpirationSeconds < MinTokenAgeSec {
 		allErrs = append(allErrs, field.Invalid(specPath.Child("expirationSeconds"), tr.Spec.ExpirationSeconds, "may not specify a duration less than 10 minutes"))
 	}
 	if tr.Spec.ExpirationSeconds > 1<<32 {

diff --git a/pkg/controller/serviceaccount/tokens_controller.go b/pkg/controller/serviceaccount/tokens_controller.go
@@ -409,7 +409,9 @@ func (e *TokensController) generateTokenIfNeeded(logger klog.Logger, serviceAcco
 
 	// Generate the token
 	if needsToken {
-		token, err := e.token.GenerateToken(serviceaccount.LegacyClaims(*serviceAccount, *liveSecret))
+		c, pc := serviceaccount.LegacyClaims(*serviceAccount, *liveSecret)
+		// TODO: need to plumb context if using external signer ever becomes a posibility.
+		token, err := e.token.GenerateToken(context.TODO(), c, pc)
 		if err != nil {
 			return false, err
 		}

diff --git a/pkg/controller/serviceaccount/tokens_controller_test.go b/pkg/controller/serviceaccount/tokens_controller_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package serviceaccount
 
 import (
+	"context"
 	"reflect"
 	"testing"
 	"time"
@@ -40,7 +41,7 @@ type testGenerator struct {
 	Err   error
 }
 
-func (t *testGenerator) GenerateToken(sc *jwt.Claims, pc interface{}) (string, error) {
+func (t *testGenerator) GenerateToken(ctx context.Context, sc *jwt.Claims, pc interface{}) (string, error) {
 	return t.Token, t.Err
 }
 

diff --git a/pkg/controlplane/apiserver/apis.go b/pkg/controlplane/apiserver/apis.go
@@ -52,6 +52,7 @@ func (c *CompletedConfig) NewCoreGenericConfig() *corerest.GenericConfig {
 		LoopbackClientConfig:        c.Generic.LoopbackClientConfig,
 		ServiceAccountIssuer:        c.Extra.ServiceAccountIssuer,
 		ExtendExpiration:            c.Extra.ExtendExpiration,
+		IsTokenSignerExternal:       c.Extra.IsTokenSignerExternal,
 		ServiceAccountMaxExpiration: c.Extra.ServiceAccountMaxExpiration,
 		APIAudiences:                c.Generic.Authentication.APIAudiences,
 		Informers:                   c.Extra.VersionedInformers,

diff --git a/pkg/controlplane/apiserver/config.go b/pkg/controlplane/apiserver/config.go
@@ -92,6 +92,7 @@ type Extra struct {
 	ServiceAccountIssuer        serviceaccount.TokenGenerator
 	ServiceAccountMaxExpiration time.Duration
 	ExtendExpiration            bool
+	IsTokenSignerExternal       bool
 
 	// ServiceAccountIssuerDiscovery
 	ServiceAccountIssuerURL        string
@@ -300,6 +301,7 @@ func CreateConfig(
 			ServiceAccountIssuer:        opts.ServiceAccountIssuer,
 			ServiceAccountMaxExpiration: opts.ServiceAccountTokenMaxExpiration,
 			ExtendExpiration:            opts.Authentication.ServiceAccounts.ExtendExpiration,
+			IsTokenSignerExternal:       opts.Authentication.ServiceAccounts.IsTokenSignerExternal,
 
 			VersionedInformers: versionedInformers,
 		},

diff --git a/pkg/controlplane/apiserver/config_test.go b/pkg/controlplane/apiserver/config_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package apiserver
 
 import (
+	"context"
 	"net"
 	"testing"
 
@@ -45,7 +46,7 @@ func TestBuildGenericConfig(t *testing.T) {
 	s.BindPort = ln.Addr().(*net.TCPAddr).Port
 	opts.SecureServing = s
 
-	completedOptions, err := opts.Complete(nil, nil)
+	completedOptions, err := opts.Complete(context.TODO(), nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to complete apiserver options: %v", err)
 	}

diff --git a/pkg/controlplane/apiserver/options/options.go b/pkg/controlplane/apiserver/options/options.go
@@ -18,6 +18,7 @@ limitations under the License.
 package options
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"os"
@@ -36,9 +37,11 @@ import (
 	"k8s.io/klog/v2"
 	netutil "k8s.io/utils/net"
 
+	"k8s.io/kubernetes/pkg/apis/authentication/validation"
 	_ "k8s.io/kubernetes/pkg/features"
 	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 	"k8s.io/kubernetes/pkg/serviceaccount"
+	"k8s.io/kubernetes/pkg/serviceaccount/externaljwt/plugin"
 )
 
 // Options define the flags and validation for a generic controlplane. If the
@@ -85,6 +88,8 @@ type Options struct {
 	ShowHiddenMetricsForVersion string
 
 	SystemNamespaces []string
+
+	ServiceAccountSigningEndpoint string
 }
 
 // completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
@@ -191,9 +196,12 @@ func (s *Options) AddFlags(fss *cliflag.NamedFlagSets) {
 
 	fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+
 		"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.")
+
+	fs.StringVar(&s.ServiceAccountSigningEndpoint, "service-account-signing-endpoint", s.ServiceAccountSigningEndpoint, ""+
+		"Path to socket where a external JWT signer is listening. This flag is mutually exclusive with --service-account-signing-key-file and --service-account-key-file. Requires enabling feature gate (ExternalServiceAccountTokenSigner)")
 }
 
-func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
+func (o *Options) Complete(ctx context.Context, alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
 	if o == nil {
 		return CompletedOptions{completedOptions: &completedOptions{}}, nil
 	}
@@ -233,52 +241,91 @@ func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (Comple
 	// adjust authentication for completed authorization
 	completed.Authentication.ApplyAuthorization(completed.Authorization)
 
-	// verify and adjust ServiceAccountTokenMaxExpiration
+	err := o.completeServiceAccountOptions(ctx, &completed)
+	if err != nil {
+		return CompletedOptions{}, err
+	}
+
+	for key, value := range completed.APIEnablement.RuntimeConfig {
+		if key == "v1" || strings.HasPrefix(key, "v1/") ||
+			key == "api/v1" || strings.HasPrefix(key, "api/v1/") {
+			delete(completed.APIEnablement.RuntimeConfig, key)
+			completed.APIEnablement.RuntimeConfig["/v1"] = value
+		}
+		if key == "api/legacy" {
+			delete(completed.APIEnablement.RuntimeConfig, key)
+		}
+	}
+
+	return CompletedOptions{
+		completedOptions: &completed,
+	}, nil
+}
+
+func (o *Options) completeServiceAccountOptions(ctx context.Context, completed *completedOptions) error {
+	transitionWarningFmt := "service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)"
+	expExtensionWarningFmt := "service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s"
+	// verify service-account-max-token-expiration
 	if completed.Authentication.ServiceAccounts.MaxExpiration != 0 {
 		lowBound := time.Hour
 		upBound := time.Duration(1<<32) * time.Second
 		if completed.Authentication.ServiceAccounts.MaxExpiration < lowBound ||
 			completed.Authentication.ServiceAccounts.MaxExpiration > upBound {
-			return CompletedOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds")
-		}
-		if completed.Authentication.ServiceAccounts.ExtendExpiration {
-			if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second {
-				klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, completed.Authentication.ServiceAccounts.MaxExpiration)
-			}
-			if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second {
-				klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, completed.Authentication.ServiceAccounts.MaxExpiration)
-			}
+			return fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds")
 		}
 	}
-	completed.ServiceAccountTokenMaxExpiration = completed.Authentication.ServiceAccounts.MaxExpiration
 
 	if len(completed.Authentication.ServiceAccounts.Issuers) != 0 && completed.Authentication.ServiceAccounts.Issuers[0] != "" {
-		if completed.ServiceAccountSigningKeyFile != "" {
+		switch {
+		case completed.ServiceAccountSigningEndpoint != "" && completed.ServiceAccountSigningKeyFile != "":
+			return fmt.Errorf("service-account-signing-key-file and service-account-signing-endpoint are mutually exclusive and cannot be set at the same time")
+		case completed.ServiceAccountSigningKeyFile != "":
 			sk, err := keyutil.PrivateKeyFromFile(completed.ServiceAccountSigningKeyFile)
 			if err != nil {
-				return CompletedOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %w", err)
+				return fmt.Errorf("failed to parse service-account-issuer-key-file: %w", err)
 			}
 			completed.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(completed.Authentication.ServiceAccounts.Issuers[0], sk)
 			if err != nil {
-				return CompletedOptions{}, fmt.Errorf("failed to build token generator: %w", err)
+				return fmt.Errorf("failed to build token generator: %w", err)
+			}
+		case completed.ServiceAccountSigningEndpoint != "":
+			plugin, cache, err := plugin.New(ctx, completed.Authentication.ServiceAccounts.Issuers[0], completed.ServiceAccountSigningEndpoint, 60*time.Second, false)
+			if err != nil {
+				return fmt.Errorf("while setting up external-jwt-signer: %w", err)
 			}
+			timedContext, cancel := context.WithTimeout(ctx, 10*time.Second)
+			defer cancel()
+			metadata, err := plugin.GetServiceMetadata(timedContext)
+			if err != nil {
+				return fmt.Errorf("while setting up external-jwt-signer: %w", err)
+			}
+			if metadata.MaxTokenExpirationSeconds < validation.MinTokenAgeSec {
+				return fmt.Errorf("max token life supported by external-jwt-signer (%ds) is less than acceptable (min %ds)", metadata.MaxTokenExpirationSeconds, validation.MinTokenAgeSec)
+			}
+			if completed.Authentication.ServiceAccounts.MaxExpiration != 0 {
+				return fmt.Errorf("service-account-max-token-expiration and service-account-signing-endpoint are mutually exclusive and cannot be set at the same time")
+			}
+			transitionWarningFmt = "service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, token lifetime supported by external-jwt-signer must be longer than %d seconds (currently %s)"
+			expExtensionWarningFmt = "service-account-extend-token-expiration is true, tokens validity will be caped at the smaller of %d seconds and maximum token lifetime supported by external-jwt-signer (%s)"
+			completed.ServiceAccountIssuer = plugin
+			completed.Authentication.ServiceAccounts.ExternalPublicKeysGetter = cache
+			completed.Authentication.ServiceAccounts.MaxExpiration = time.Duration(metadata.MaxTokenExpirationSeconds) * time.Second
+			completed.Authentication.ServiceAccounts.IsTokenSignerExternal = true
 		}
 	}
 
-	for key, value := range completed.APIEnablement.RuntimeConfig {
-		if key == "v1" || strings.HasPrefix(key, "v1/") ||
-			key == "api/v1" || strings.HasPrefix(key, "api/v1/") {
-			delete(completed.APIEnablement.RuntimeConfig, key)
-			completed.APIEnablement.RuntimeConfig["/v1"] = value
+	// Set Max expiration and warn on conflicting configuration.
+	if completed.Authentication.ServiceAccounts.ExtendExpiration && completed.Authentication.ServiceAccounts.MaxExpiration != 0 {
+		if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second {
+			klog.Warningf(transitionWarningFmt, serviceaccount.WarnOnlyBoundTokenExpirationSeconds, completed.Authentication.ServiceAccounts.MaxExpiration)
 		}
-		if key == "api/legacy" {
-			delete(completed.APIEnablement.RuntimeConfig, key)
+		if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second {
+			klog.Warningf(expExtensionWarningFmt, serviceaccount.ExpirationExtensionSeconds, completed.Authentication.ServiceAccounts.MaxExpiration)
 		}
 	}
+	completed.ServiceAccountTokenMaxExpiration = completed.Authentication.ServiceAccounts.MaxExpiration
 
-	return CompletedOptions{
-		completedOptions: &completed,
-	}, nil
+	return nil
 }
 
 // ServiceIPRange checks if the serviceClusterIPRange flag is nil, raising a warning if so and