From 403301bfdf2c7312591077827abd2e72f445a53a Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Fri, 19 Jan 2024 16:07:00 -0800 Subject: [PATCH] apiserver: Add API emulation versioning. Co-authored-by: Siyuan Zhang Co-authored-by: Joe Betz Co-authored-by: Alex Zielenski Signed-off-by: Siyuan Zhang --- cmd/kube-apiserver/app/options/options.go | 8 +- .../app/options/options_test.go | 16 +- cmd/kube-apiserver/app/server.go | 17 +- cmd/kube-apiserver/app/testing/testserver.go | 21 +- .../app/options/options.go | 2 +- cmd/kube-scheduler/app/options/options.go | 2 +- pkg/controlplane/apiserver/apiextensions.go | 2 +- pkg/controlplane/apiserver/apis.go | 2 +- pkg/controlplane/apiserver/config.go | 2 + pkg/controlplane/apiserver/config_test.go | 8 +- pkg/controlplane/apiserver/options/options.go | 10 +- .../apiserver/options/options_test.go | 15 +- .../apiserver/options/validation.go | 1 + .../apiserver/options/validation_test.go | 5 +- pkg/controlplane/apiserver/peer.go | 2 +- pkg/controlplane/instance.go | 2 +- pkg/controlplane/instance_test.go | 3 + pkg/features/client_adapter_test.go | 5 +- pkg/features/kube_features.go | 1 + pkg/features/kube_features_test.go | 3 + pkg/features/versioned_kube_features.go | 34 + .../default_storage_factory_builder.go | 2 + pkg/registry/core/rest/storage_core.go | 2 +- pkg/registry/core/rest/storage_core_test.go | 3 +- pkg/registry/registrytest/etcd.go | 2 +- .../pkg/apiserver/customresource_handler.go | 5 +- .../pkg/cmd/server/options/options.go | 8 +- .../pkg/cmd/server/server.go | 15 +- .../pkg/cmd/server/testing/testserver.go | 14 +- .../integration/conversion/conversion_test.go | 2 +- .../test/integration/defaulting_test.go | 2 +- .../test/integration/fixtures/server.go | 2 +- .../test/integration/objectmeta_test.go | 2 +- .../test/integration/pruning_test.go | 2 +- .../apimachinery/pkg/util/version/version.go | 71 ++ .../pkg/util/version/version_test.go | 91 ++ .../apiserver/pkg/cel/environment/base.go | 21 +- .../apiserver/pkg/features/kube_features.go | 11 + .../apiserver/pkg/registry/generic/options.go | 8 +- .../pkg/registry/generic/registry/store.go | 2 +- .../src/k8s.io/apiserver/pkg/server/config.go | 34 +- .../apiserver/pkg/server/deleted_kinds.go | 65 +- .../pkg/server/deleted_kinds_test.go | 173 +-- .../apiserver/pkg/server/genericapiserver.go | 7 + .../pkg/server/genericapiserver_test.go | 4 +- .../pkg/server/options/api_enablement_test.go | 42 +- .../apiserver/pkg/server/options/etcd.go | 10 +- .../apiserver/pkg/server/options/etcd_test.go | 2 +- .../pkg/server/options/server_run_options.go | 32 +- .../server/options/server_run_options_test.go | 36 +- .../pkg/server/options/serving_test.go | 11 +- .../storage/resource_encoding_config.go | 110 +- .../pkg/server/storage/storage_factory.go | 19 +- .../server/storage/storage_factory_test.go | 246 +++- .../pkg/util/feature/feature_gate.go | 2 +- .../apiserver/pkg/util/version/registry.go | 142 +++ .../pkg/util/version/registry_test.go | 48 + .../apiserver/pkg/util/version/version.go | 195 +++ .../pkg/util/version/version_test.go | 180 +++ .../k8s.io/cloud-provider/options/options.go | 2 +- .../featuregate/feature_gate.go | 388 ++++-- .../featuregate/feature_gate_test.go | 1055 +++++++++++++++-- .../featuregate/testing/feature_gate.go | 6 +- .../component-base/logs/example/cmd/logger.go | 2 +- .../logs/example/k8s2slog/k8s2slog.go | 2 +- .../pkg/apiserver/apiserver.go | 2 +- .../kube-aggregator/pkg/cmd/server/start.go | 23 +- .../pkg/apis/wardle/v1alpha1/doc.go | 1 + .../pkg/apis/wardle/v1alpha1/types.go | 8 + .../zz_generated.prerelease-lifecycle.go | 94 ++ .../pkg/apiserver/apiserver.go | 12 +- .../sample-apiserver/pkg/cmd/server/start.go | 46 +- .../pkg/cmd/server/start_test.go | 66 ++ test/e2e/dra/test-driver/app/server.go | 2 +- test/e2e_node/services/apiserver.go | 6 +- test/integration/apiserver/apiserver_test.go | 296 +++++ test/integration/auth/rbac_test.go | 5 +- .../kmsv2_transformation_test.go | 2 +- .../transformation/transformation_test.go | 4 +- .../etcd/crd_overlap_storage_test.go | 10 +- .../integration/etcd/etcd_cross_group_test.go | 2 + .../etcd/etcd_storage_path_test.go | 2 + test/integration/etcd/server.go | 6 +- test/integration/examples/apiserver_test.go | 1 + test/integration/framework/test_server.go | 6 +- .../integration/servicecidr/allocator_test.go | 2 + 86 files changed, 3414 insertions(+), 421 deletions(-) create mode 100644 pkg/features/versioned_kube_features.go create mode 100644 staging/src/k8s.io/apiserver/pkg/util/version/registry.go create mode 100644 staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go create mode 100644 staging/src/k8s.io/apiserver/pkg/util/version/version.go create mode 100644 staging/src/k8s.io/apiserver/pkg/util/version/version_test.go create mode 100644 staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/zz_generated.prerelease-lifecycle.go create mode 100644 staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index afdb9e73ff33c..53a5b3c06eebd 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -26,6 +26,8 @@ import ( utilnet "k8s.io/apimachinery/pkg/util/net" cliflag "k8s.io/component-base/cli/flag" + utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/component-base/featuregate" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/cluster/ports" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" @@ -63,10 +65,10 @@ type Extra struct { MasterCount int } -// NewServerRunOptions creates a new ServerRunOptions object with default parameters -func NewServerRunOptions() *ServerRunOptions { +// NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run. +func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions { s := ServerRunOptions{ - Options: controlplaneapiserver.NewOptions(), + Options: controlplaneapiserver.NewOptions(featureGate, effectiveVersion), CloudProvider: kubeoptions.NewCloudProviderOptions(), Extra: Extra{ diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 2fa5da605a6c2..56cbcbebd0377 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -31,9 +31,11 @@ import ( apiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/etcd3" "k8s.io/apiserver/pkg/storage/storagebackend" + utilversion "k8s.io/apiserver/pkg/util/version" auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" "k8s.io/component-base/metrics" kapi "k8s.io/kubernetes/pkg/apis/core" @@ -46,10 +48,15 @@ import ( func TestAddFlags(t *testing.T) { fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) - s := NewServerRunOptions() + + featureGate := featuregate.NewFeatureGate() + effectiveVersion := utilversion.NewEffectiveVersion("1.32") + s := NewServerRunOptions(featureGate, effectiveVersion) for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } + featureGate.AddFlag(fs, "") + effectiveVersion.AddFlags(fs, "") args := []string{ "--enable-admission-plugins=AlwaysDeny", @@ -121,6 +128,7 @@ func TestAddFlags(t *testing.T) { "--storage-backend=etcd3", "--service-cluster-ip-range=192.168.128.0/17", "--lease-reuse-duration-seconds=100", + "--emulated-version=1.31", } fs.Parse(args) @@ -136,6 +144,8 @@ func TestAddFlags(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), MaxRequestBodyBytes: int64(3 * 1024 * 1024), + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, Admission: &kubeoptions.AdmissionOptions{ GenericAdmission: &apiserveroptions.AdmissionOptions{ @@ -337,4 +347,8 @@ func TestAddFlags(t *testing.T) { if !reflect.DeepEqual(expected, s) { t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{}))) } + + if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" { + t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31") + } } diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 81f826ac876ee..3489f2130ae32 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -36,6 +36,7 @@ import ( serverstorage "k8s.io/apiserver/pkg/server/storage" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/notfoundhandler" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/apiserver/pkg/util/webhook" clientgoinformers "k8s.io/client-go/informers" "k8s.io/client-go/rest" @@ -63,7 +64,10 @@ func init() { // NewAPIServerCommand creates a *cobra.Command object with default parameters func NewAPIServerCommand() *cobra.Command { - s := options.NewServerRunOptions() + effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + utilversion.ComponentGenericAPIServer, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + s := options.NewServerRunOptions(featureGate, effectiveVersion) + cmd := &cobra.Command{ Use: "kube-apiserver", Long: `The Kubernetes API server validates and configures data @@ -83,9 +87,13 @@ cluster's shared state through which all other components interact.`, verflag.PrintAndExitIfRequested() fs := cmd.Flags() + if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + return err + } + // Activate logging as soon as possible, after that // show flags with the final logging configuration. - if err := logsapi.ValidateAndApply(s.Logs, utilfeature.DefaultFeatureGate); err != nil { + if err := logsapi.ValidateAndApply(s.Logs, featureGate); err != nil { return err } cliflag.PrintFlags(fs) @@ -101,7 +109,7 @@ cluster's shared state through which all other components interact.`, return utilerrors.NewAggregate(errs) } // add feature enablement metrics - utilfeature.DefaultMutableFeatureGate.AddMetrics() + featureGate.AddMetrics() return Run(cmd.Context(), completedOptions) }, Args: func(cmd *cobra.Command, args []string) error { @@ -118,6 +126,9 @@ cluster's shared state through which all other components interact.`, fs := cmd.Flags() namedFlagSets := s.Flags() verflag.AddFlags(namedFlagSets.FlagSet("global")) + featureGate.AddFlag(namedFlagSets.FlagSet("global"), "") + effectiveVersion.AddFlags(namedFlagSets.FlagSet("global"), "") + globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags()) options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic")) for _, f := range namedFlagSets.FlagSets { diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index 3d00006e38a46..7eb7dce049629 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -48,6 +48,7 @@ import ( "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/apiserver/pkg/storageversion" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" clientgotransport "k8s.io/client-go/transport" @@ -98,6 +99,9 @@ type TestServerInstanceOptions struct { // We specify this as on option to pass a common proxyCA to multiple apiservers to simulate // an apiserver version skew scenario where all apiservers use the same proxyCA to verify client connections. ProxyCA *ProxyCA + // Set the BinaryVersion of server effective version. + // Default to 1.31 + BinaryVersion string } // TestServer return values supplied by kube-test-ApiServer @@ -177,10 +181,21 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, fs := pflag.NewFlagSet("test", pflag.PanicOnError) - s := options.NewServerRunOptions() + featureGate := utilfeature.DefaultMutableFeatureGate + binaryVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String() + if instanceOptions.BinaryVersion != "" { + binaryVersion = instanceOptions.BinaryVersion + } + effectiveVersion := utilversion.NewEffectiveVersion(binaryVersion) + _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true) + + s := options.NewServerRunOptions(featureGate, effectiveVersion) + for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } + featureGate.AddFlag(fs, "") + effectiveVersion.AddFlags(fs, "") s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() if err != nil { @@ -321,6 +336,10 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, return result, err } + if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + return result, err + } + saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key") if err != nil { t.Fatalf("create temp file failed: %v", err) diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 1c3e4edb1780c..d72920d0cd18b 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -273,7 +273,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy fs := fss.FlagSet("misc") fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") fs.StringVar(&s.Generic.ClientConnection.Kubeconfig, "kubeconfig", s.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).") - utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic")) + utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "") return fss } diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index e039b0d05c994..2b1233c435b6b 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -189,7 +189,7 @@ func (o *Options) initFlags() { o.Authorization.AddFlags(nfs.FlagSet("authorization")) o.Deprecated.AddFlags(nfs.FlagSet("deprecated")) options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election")) - utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate")) + utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"), "") o.Metrics.AddFlags(nfs.FlagSet("metrics")) logsapi.AddFlags(o.Logs, nfs.FlagSet("logs")) diff --git a/pkg/controlplane/apiserver/apiextensions.go b/pkg/controlplane/apiserver/apiextensions.go index d203d2757e33d..1a0b8910c297e 100644 --- a/pkg/controlplane/apiserver/apiextensions.go +++ b/pkg/controlplane/apiserver/apiextensions.go @@ -17,7 +17,6 @@ limitations under the License. package apiserver import ( - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" apiextensionsoptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" @@ -27,6 +26,7 @@ import ( "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/util/webhook" "k8s.io/client-go/informers" + v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "k8s.io/kubernetes/pkg/controlplane/apiserver/options" ) diff --git a/pkg/controlplane/apiserver/apis.go b/pkg/controlplane/apiserver/apis.go index 1ccfe78746601..9ebb7e8f4fe1d 100644 --- a/pkg/controlplane/apiserver/apis.go +++ b/pkg/controlplane/apiserver/apis.go @@ -88,7 +88,7 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error nonLegacy := []*genericapiserver.APIGroupInfo{} // used later in the loop to filter the served resource by those that have expired. - resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*s.GenericAPIServer.Version) + resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion()) if err != nil { return err } diff --git a/pkg/controlplane/apiserver/config.go b/pkg/controlplane/apiserver/config.go index b413fac34c6df..7933f076ec618 100644 --- a/pkg/controlplane/apiserver/config.go +++ b/pkg/controlplane/apiserver/config.go @@ -185,7 +185,9 @@ func BuildGenericConfig( } storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig() + storageFactoryConfig.CurrentVersion = genericConfig.EffectiveVersion storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig + storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(genericConfig.EffectiveVersion) storageFactory, lastErr = storageFactoryConfig.Complete(s.Etcd).New() if lastErr != nil { return diff --git a/pkg/controlplane/apiserver/config_test.go b/pkg/controlplane/apiserver/config_test.go index 80bff6f50a8f7..c5ff6a79b830e 100644 --- a/pkg/controlplane/apiserver/config_test.go +++ b/pkg/controlplane/apiserver/config_test.go @@ -24,6 +24,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" apiserveroptions "k8s.io/apiserver/pkg/server/options" + utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/component-base/featuregate" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/controlplane/apiserver/options" @@ -32,7 +34,9 @@ import ( ) func TestBuildGenericConfig(t *testing.T) { - opts := options.NewOptions() + featureGate := featuregate.NewFeatureGate() + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + opts := options.NewOptions(featureGate, effectiveVersion) s := (&apiserveroptions.SecureServingOptions{ BindAddress: netutils.ParseIPSloppy("127.0.0.1"), }).WithLoopback() @@ -66,7 +70,7 @@ func TestBuildGenericConfig(t *testing.T) { t.Errorf("There are different StorageObjectCountTracker in genericConfig and storageFactory") } - restOptions, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}) + restOptions, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/controlplane/apiserver/options/options.go b/pkg/controlplane/apiserver/options/options.go index 98396cd8450d9..bdaa56ffede86 100644 --- a/pkg/controlplane/apiserver/options/options.go +++ b/pkg/controlplane/apiserver/options/options.go @@ -28,8 +28,10 @@ import ( peerreconcilers "k8s.io/apiserver/pkg/reconcilers" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/storagebackend" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/util/keyutil" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/component-base/metrics" @@ -98,9 +100,9 @@ type CompletedOptions struct { } // NewOptions creates a new ServerRunOptions object with default parameters -func NewOptions() *Options { +func NewOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *Options { s := Options{ - GenericServerRunOptions: genericoptions.NewServerRunOptions(), + GenericServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)), SecureServing: kubeoptions.NewSecureServingOptions(), Audit: genericoptions.NewAuditOptions(), @@ -202,6 +204,10 @@ func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (Comple Options: *o, } + if err := completed.GenericServerRunOptions.Complete(); err != nil { + return CompletedOptions{}, err + } + // set defaults if err := completed.GenericServerRunOptions.DefaultAdvertiseAddress(completed.SecureServing.SecureServingOptions); err != nil { return CompletedOptions{}, err diff --git a/pkg/controlplane/apiserver/options/options_test.go b/pkg/controlplane/apiserver/options/options_test.go index 88fe5b961ddfa..8660231d2d0c3 100644 --- a/pkg/controlplane/apiserver/options/options_test.go +++ b/pkg/controlplane/apiserver/options/options_test.go @@ -30,9 +30,11 @@ import ( apiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/etcd3" "k8s.io/apiserver/pkg/storage/storagebackend" + utilversion "k8s.io/apiserver/pkg/util/version" auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" "k8s.io/component-base/metrics" netutils "k8s.io/utils/net" @@ -42,12 +44,16 @@ import ( func TestAddFlags(t *testing.T) { fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) - s := NewOptions() + featureGate := featuregate.NewFeatureGate() + effectiveVersion := utilversion.NewEffectiveVersion("1.32") + s := NewOptions(featureGate, effectiveVersion) var fss cliflag.NamedFlagSets s.AddFlags(&fss) for _, f := range fss.FlagSets { fs.AddFlagSet(f) } + featureGate.AddFlag(fs, "") + effectiveVersion.AddFlags(fs, "") args := []string{ "--enable-admission-plugins=AlwaysDeny", @@ -108,6 +114,7 @@ func TestAddFlags(t *testing.T) { "--request-timeout=2m", "--storage-backend=etcd3", "--lease-reuse-duration-seconds=100", + "--emulated-version=1.31", } fs.Parse(args) @@ -122,6 +129,8 @@ func TestAddFlags(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), MaxRequestBodyBytes: int64(3 * 1024 * 1024), + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, Admission: &kubeoptions.AdmissionOptions{ GenericAdmission: &apiserveroptions.AdmissionOptions{ @@ -292,4 +301,8 @@ func TestAddFlags(t *testing.T) { if !reflect.DeepEqual(expected, s) { t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{}))) } + + if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" { + t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31") + } } diff --git a/pkg/controlplane/apiserver/options/validation.go b/pkg/controlplane/apiserver/options/validation.go index 1cae7323ab1f7..cca855be43fa8 100644 --- a/pkg/controlplane/apiserver/options/validation.go +++ b/pkg/controlplane/apiserver/options/validation.go @@ -100,6 +100,7 @@ func validateUnknownVersionInteroperabilityProxyFlags(options *Options) []error func (s *Options) Validate() []error { var errs []error + errs = append(errs, s.GenericServerRunOptions.Validate()...) errs = append(errs, s.Etcd.Validate()...) errs = append(errs, validateAPIPriorityAndFairness(s)...) errs = append(errs, s.SecureServing.Validate()...) diff --git a/pkg/controlplane/apiserver/options/validation_test.go b/pkg/controlplane/apiserver/options/validation_test.go index de7456c43a707..9dddc971a42df 100644 --- a/pkg/controlplane/apiserver/options/validation_test.go +++ b/pkg/controlplane/apiserver/options/validation_test.go @@ -23,6 +23,7 @@ import ( kubeapiserveradmission "k8s.io/apiserver/pkg/admission" genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/component-base/featuregate" basemetrics "k8s.io/component-base/metrics" "k8s.io/kubernetes/pkg/features" @@ -200,7 +201,7 @@ func TestValidateOptions(t *testing.T) { name: "validate master count equal 0", expectErrors: true, options: &Options{ - GenericServerRunOptions: &genericoptions.ServerRunOptions{}, + GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")}, Etcd: &genericoptions.EtcdOptions{}, SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, Audit: &genericoptions.AuditOptions{}, @@ -227,7 +228,7 @@ func TestValidateOptions(t *testing.T) { name: "validate token request enable not attempted", expectErrors: true, options: &Options{ - GenericServerRunOptions: &genericoptions.ServerRunOptions{}, + GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")}, Etcd: &genericoptions.EtcdOptions{}, SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, Audit: &genericoptions.AuditOptions{}, diff --git a/pkg/controlplane/apiserver/peer.go b/pkg/controlplane/apiserver/peer.go index 8d04f520c669a..e15b8980c8d38 100644 --- a/pkg/controlplane/apiserver/peer.go +++ b/pkg/controlplane/apiserver/peer.go @@ -82,7 +82,7 @@ func BuildPeerProxy(versionedInformer clientgoinformers.SharedInformerFactory, s // The peer endpoint leases are used to find network locations of apiservers for peer proxy func CreatePeerEndpointLeaseReconciler(c genericapiserver.Config, storageFactory serverstorage.StorageFactory) (reconcilers.PeerEndpointLeaseReconciler, error) { ttl := DefaultPeerEndpointReconcilerTTL - config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo")) + config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo"), &api.Endpoints{}) if err != nil { return nil, fmt.Errorf("error creating storage factory config: %w", err) } diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go index d51f15ec982be..5f1b732de5b0c 100644 --- a/pkg/controlplane/instance.go +++ b/pkg/controlplane/instance.go @@ -214,7 +214,7 @@ func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler { endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient) ttl := c.Extra.MasterEndpointReconcileTTL - config, err := c.ControlPlane.StorageFactory.NewConfig(api.Resource("apiServerIPInfo")) + config, err := c.ControlPlane.StorageFactory.NewConfig(api.Resource("apiServerIPInfo"), &api.Endpoints{}) if err != nil { klog.Fatalf("Error creating storage factory config: %v", err) } diff --git a/pkg/controlplane/instance_test.go b/pkg/controlplane/instance_test.go index d4c415101bc0f..32d1811d82c07 100644 --- a/pkg/controlplane/instance_test.go +++ b/pkg/controlplane/instance_test.go @@ -55,6 +55,7 @@ import ( serverstorage "k8s.io/apiserver/pkg/server/storage" etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" "k8s.io/apiserver/pkg/util/openapi" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/discovery" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -103,7 +104,9 @@ func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertio }, } + config.ControlPlane.Generic.EffectiveVersion = utilversion.DefaultKubeEffectiveVersion() storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig() + storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(config.ControlPlane.Generic.EffectiveVersion) storageConfig.StorageObjectCountTracker = config.ControlPlane.Generic.StorageObjectCountTracker resourceEncoding := resourceconfig.MergeResourceEncodingConfigs(storageFactoryConfig.DefaultResourceEncoding, storageFactoryConfig.ResourceEncodingOverrides) storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, "application/vnd.kubernetes.protobuf", storageFactoryConfig.Serializer, resourceEncoding, DefaultAPIResourceConfigSource(), nil) diff --git a/pkg/features/client_adapter_test.go b/pkg/features/client_adapter_test.go index 13ec29094ecd3..cae067ca99cfa 100644 --- a/pkg/features/client_adapter_test.go +++ b/pkg/features/client_adapter_test.go @@ -19,6 +19,8 @@ package features import ( "testing" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" clientfeatures "k8s.io/client-go/features" "k8s.io/component-base/featuregate" ) @@ -78,8 +80,7 @@ func TestClientAdapterAdd(t *testing.T) { t.Errorf("expected feature %q not found", name) continue } - - if actual != expected { + if diff := cmp.Diff(actual, expected, cmpopts.IgnoreFields(featuregate.FeatureSpec{}, "Version")); diff != "" { t.Errorf("expected feature %q spec %#v, got spec %#v", name, expected, actual) } } diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index a83aa2a27f3b8..f84825e8e9f0d 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -946,6 +946,7 @@ const ( func init() { runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)) + runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates)) // Register all client-go features with kube's feature gate instance and make all client-go // feature checks use kube's instance. The effect is that for kube binaries, client-go diff --git a/pkg/features/kube_features_test.go b/pkg/features/kube_features_test.go index 81bfbc421d2c2..be847a1f0408a 100644 --- a/pkg/features/kube_features_test.go +++ b/pkg/features/kube_features_test.go @@ -62,6 +62,9 @@ func TestAllRegisteredFeaturesExpected(t *testing.T) { if err := knownFeatureGates.Add(defaultKubernetesFeatureGates); err != nil { t.Fatal(err) } + if err := knownFeatureGates.AddVersioned(defaultVersionedKubernetesFeatureGates); err != nil { + t.Fatal(err) + } knownFeatures := knownFeatureGates.GetAll() for registeredFeature := range registeredFeatures { diff --git a/pkg/features/versioned_kube_features.go b/pkg/features/versioned_kube_features.go new file mode 100644 index 0000000000000..7ef53861822d9 --- /dev/null +++ b/pkg/features/versioned_kube_features.go @@ -0,0 +1,34 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "k8s.io/component-base/featuregate" +) + +// defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs. +// To add a new feature, define a key for it and add it here. The features will be +// available throughout Kubernetes binaries. +// +// Entries are separated from each other with blank lines to avoid sweeping gofmt changes +// when adding or removing one entry. +var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{ + // Example: + // genericfeatures.EmulationVersion: { + // {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, + // }, +} diff --git a/pkg/kubeapiserver/default_storage_factory_builder.go b/pkg/kubeapiserver/default_storage_factory_builder.go index f73289ba761e5..0622862d5964a 100644 --- a/pkg/kubeapiserver/default_storage_factory_builder.go +++ b/pkg/kubeapiserver/default_storage_factory_builder.go @@ -25,6 +25,7 @@ import ( "k8s.io/apiserver/pkg/server/resourceconfig" serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/util/version" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/apps" "k8s.io/kubernetes/pkg/apis/certificates" @@ -93,6 +94,7 @@ type StorageFactoryConfig struct { Serializer runtime.StorageSerializer ResourceEncodingOverrides []schema.GroupVersionResource EtcdServersOverrides []string + CurrentVersion version.EffectiveVersion } // Complete completes the StorageFactoryConfig with provided etcdOptions returning completedStorageFactoryConfig. diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go index f808615ecd176..4e57732e8f369 100644 --- a/pkg/registry/core/rest/storage_core.go +++ b/pkg/registry/core/rest/storage_core.go @@ -321,7 +321,7 @@ func (p *legacyProvider) NewRESTStorage(apiResourceConfigSource serverstorage.AP func (c *Config) newServiceIPAllocators() (registries rangeRegistries, primaryClusterIPAllocator ipallocator.Interface, clusterIPAllocators map[api.IPFamily]ipallocator.Interface, nodePortAllocator *portallocator.PortAllocator, err error) { clusterIPAllocators = map[api.IPFamily]ipallocator.Interface{} - serviceStorageConfig, err := c.StorageFactory.NewConfig(api.Resource("services")) + serviceStorageConfig, err := c.StorageFactory.NewConfig(api.Resource("services"), &api.Service{}) if err != nil { return rangeRegistries{}, nil, nil, nil, err } diff --git a/pkg/registry/core/rest/storage_core_test.go b/pkg/registry/core/rest/storage_core_test.go index 411efa211dce9..4c74d0341627e 100644 --- a/pkg/registry/core/rest/storage_core_test.go +++ b/pkg/registry/core/rest/storage_core_test.go @@ -19,6 +19,7 @@ package rest import ( "testing" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/server/storage" "k8s.io/apiserver/pkg/storage/storagebackend" @@ -40,7 +41,7 @@ func TestGetServersToValidate(t *testing.T) { type fakeStorageFactory struct{} -func (f fakeStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) { +func (f fakeStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { return nil, nil } diff --git a/pkg/registry/registrytest/etcd.go b/pkg/registry/registrytest/etcd.go index 990d6b3d2772f..226c35c1e7f31 100644 --- a/pkg/registry/registrytest/etcd.go +++ b/pkg/registry/registrytest/etcd.go @@ -44,7 +44,7 @@ func NewEtcdStorageForResource(t *testing.T, resource schema.GroupResource) (*st if err != nil { t.Fatalf("Error while making storage factory: %v", err) } - resourceConfig, err := factory.NewConfig(resource) + resourceConfig, err := factory.NewConfig(resource, nil) if err != nil { t.Fatalf("Error while finding storage destination: %v", err) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index 9c0ee416f5239..e0440f7a185bf 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -1178,8 +1178,9 @@ type crdConversionRESTOptionsGetter struct { preserveUnknownFields bool } -func (t crdConversionRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) { - ret, err := t.RESTOptionsGetter.GetRESTOptions(resource) +func (t crdConversionRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) { + // Explicitly ignore example, we override storageconfig below + ret, err := t.RESTOptionsGetter.GetRESTOptions(resource, nil) if err == nil { d := schemaCoercingDecoder{delegate: ret.StorageConfig.Codec, validator: unstructuredSchemaCoercer{ // drop invalid fields while decoding old CRs (before we haven't had any ObjectMeta validation) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go index 3d42acf8b3d45..07fdcb940f5e0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go @@ -39,9 +39,11 @@ import ( flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" "k8s.io/apiserver/pkg/util/openapi" "k8s.io/apiserver/pkg/util/proxy" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/apiserver/pkg/util/webhook" scheme "k8s.io/client-go/kubernetes/scheme" corev1 "k8s.io/client-go/listers/core/v1" + "k8s.io/component-base/featuregate" netutils "k8s.io/utils/net" ) @@ -58,9 +60,9 @@ type CustomResourceDefinitionsServerOptions struct { } // NewCustomResourceDefinitionsServerOptions creates default options of an apiextensions-apiserver. -func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer) *CustomResourceDefinitionsServerOptions { +func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer, featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *CustomResourceDefinitionsServerOptions { o := &CustomResourceDefinitionsServerOptions{ - ServerRunOptions: genericoptions.NewServerRunOptions(), + ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), RecommendedOptions: genericoptions.NewRecommendedOptions( defaultEtcdPathPrefix, apiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion), @@ -92,7 +94,7 @@ func (o CustomResourceDefinitionsServerOptions) Validate() error { // Complete fills in missing options. func (o *CustomResourceDefinitionsServerOptions) Complete() error { - return nil + return o.ServerRunOptions.Complete() } // Config returns an apiextensions-apiserver configuration. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go index 0509a6aa6dfdf..b59b12f6da931 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go @@ -24,15 +24,26 @@ import ( "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" genericapiserver "k8s.io/apiserver/pkg/server" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" ) func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command { - o := options.NewCustomResourceDefinitionsServerOptions(out, errOut) + // effectiveVersion is used to set what apis and feature gates the generic api server is compatible with. + // You can also have the flag setting the effectiveVersion of the apiextensions apiserver, and + // having a mapping from the apiextensions apiserver version to generic apiserver version. + effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + o := options.NewCustomResourceDefinitionsServerOptions(out, errOut, featureGate, effectiveVersion) cmd := &cobra.Command{ Short: "Launch an API extensions API server", Long: "Launch an API extensions API server", RunE: func(c *cobra.Command, args []string) error { + if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + return err + } + if err := o.Complete(); err != nil { return err } @@ -48,6 +59,8 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command cmd.SetContext(ctx) fs := cmd.Flags() + featureGate.AddFlag(fs, "") + effectiveVersion.AddFlags(fs, "") o.AddFlags(fs) return cmd } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go index 3f2e7812bf4e7..b1e5b9dd8d601 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go @@ -35,7 +35,9 @@ import ( openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/storage/storagebackend" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/openapi" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" logsapi "k8s.io/component-base/logs/api/v1" @@ -120,7 +122,13 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin fs := pflag.NewFlagSet("test", pflag.PanicOnError) - s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr) + featureGate := utilfeature.DefaultMutableFeatureGate + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true) + s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr, featureGate, effectiveVersion) + + featureGate.AddFlag(fs, "") + effectiveVersion.AddFlags(fs, "") s.AddFlags(fs) s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() @@ -143,6 +151,10 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin fs.Parse(customFlags) + if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + return result, err + } + if err := s.Complete(); err != nil { return result, fmt.Errorf("failed to set default options: %v", err) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go index 01a15e02c48e6..71e14f2cbd5fb 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go @@ -183,7 +183,7 @@ func testWebhookConverter(t *testing.T, watchCache bool) { crd := multiVersionFixture.DeepCopy() RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil) - restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}) + restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}, nil) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go index dcfdf2f752571..eff48b4f847d0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go @@ -659,7 +659,7 @@ func TestCustomResourceDefaultingOfMetaFields(t *testing.T) { // get persisted object RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil) - restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}) + restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}, nil) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go index 8be6a88e54cc9..60a98f335a9b1 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go @@ -156,7 +156,7 @@ func StartDefaultServerWithClientsAndEtcd(t servertesting.Logger, extraFlags ... } RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, resourceTransformers, nil) - restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "hopefully-ignored-group", Resource: "hopefully-ignored-resources"}) + restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "hopefully-ignored-group", Resource: "hopefully-ignored-resources"}, nil) if err != nil { return nil, nil, nil, nil, "", err } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go index 9ac7387d7abab..67e85bcc125d5 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go @@ -133,7 +133,7 @@ func TestInvalidObjectMetaInStorage(t *testing.T) { } RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil) - restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: noxuDefinition.Spec.Group, Resource: noxuDefinition.Spec.Names.Plural}) + restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: noxuDefinition.Spec.Group, Resource: noxuDefinition.Spec.Names.Plural}, nil) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go index 003a9b55ac58f..18042c3a9737e 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go @@ -326,7 +326,7 @@ func TestPruningFromStorage(t *testing.T) { t.Fatal(err) } - restOptions, err := completedConfig.GenericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}) + restOptions, err := completedConfig.GenericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}, nil) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apimachinery/pkg/util/version/version.go b/staging/src/k8s.io/apimachinery/pkg/util/version/version.go index 2292ba13765f4..d327f00b3bec8 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/version/version.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/version/version.go @@ -145,6 +145,43 @@ func MustParseGeneric(str string) *Version { return v } +// Parse tries to do ParseSemantic first to keep more information. +// If ParseSemantic fails, it would just do ParseGeneric. +func Parse(str string) (*Version, error) { + v, err := parse(str, true) + if err != nil { + return parse(str, false) + } + return v, err +} + +// MustParse is like Parse except that it panics on error +func MustParse(str string) *Version { + v, err := Parse(str) + if err != nil { + panic(err) + } + return v +} + +// ParseMajorMinor parses a "generic" version string and returns a version with the major and minor version. +func ParseMajorMinor(str string) (*Version, error) { + v, err := ParseGeneric(str) + if err != nil { + return nil, err + } + return MajorMinor(v.Major(), v.Minor()), nil +} + +// MustParseMajorMinor is like ParseMajorMinor except that it panics on error +func MustParseMajorMinor(str string) *Version { + v, err := ParseMajorMinor(str) + if err != nil { + panic(err) + } + return v +} + // ParseSemantic parses a version string that exactly obeys the syntax and semantics of // the "Semantic Versioning" specification (http://semver.org/) (although it ignores // leading and trailing whitespace, and allows the version to be preceded by "v"). For @@ -215,6 +252,21 @@ func (v *Version) WithMinor(minor uint) *Version { return &result } +// SubtractMinor returns the version diff minor versions back, with the same major and no patch. +// If diff >= current minor, the minor would be 0. +func (v *Version) SubtractMinor(diff uint) *Version { + var minor uint + if diff < v.Minor() { + minor = v.Minor() - diff + } + return MajorMinor(v.Major(), minor) +} + +// AddMinor returns the version diff minor versions forward, with the same major and no patch. +func (v *Version) AddMinor(diff uint) *Version { + return MajorMinor(v.Major(), v.Minor()+diff) +} + // WithPatch returns copy of the version object with requested patch number func (v *Version) WithPatch(patch uint) *Version { result := *v @@ -224,6 +276,9 @@ func (v *Version) WithPatch(patch uint) *Version { // WithPreRelease returns copy of the version object with requested prerelease func (v *Version) WithPreRelease(preRelease string) *Version { + if len(preRelease) == 0 { + return v + } result := *v result.components = []uint{v.Major(), v.Minor(), v.Patch()} result.preRelease = preRelease @@ -345,6 +400,17 @@ func onlyZeros(array []uint) bool { return true } +// EqualTo tests if a version is equal to a given version. +func (v *Version) EqualTo(other *Version) bool { + if v == nil { + return other == nil + } + if other == nil { + return false + } + return v.compareInternal(other) == 0 +} + // AtLeast tests if a version is at least equal to a given minimum version. If both // Versions are Semantic Versions, this will use the Semantic Version comparison // algorithm. Otherwise, it will compare only the numeric components, with non-present @@ -360,6 +426,11 @@ func (v *Version) LessThan(other *Version) bool { return v.compareInternal(other) == -1 } +// GreaterThan tests if a version is greater than a given version. +func (v *Version) GreaterThan(other *Version) bool { + return v.compareInternal(other) == 1 +} + // Compare compares v against a version string (which will be parsed as either Semantic // or non-Semantic depending on v). On success it returns -1 if v is less than other, 1 if // it is greater than other, or 0 if they are equal. diff --git a/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go b/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go index 4d36bf3c12120..c8625fb809116 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go @@ -452,3 +452,94 @@ func TestHighestSupportedVersion(t *testing.T) { } } } + +func TestSubtractMinor(t *testing.T) { + var tests = []struct { + version string + diff uint + expectedComponents []uint + }{ + { + version: "1.0.2", + diff: 3, + expectedComponents: []uint{1, 0}, + }, + { + version: "1.3.2-alpha+001", + diff: 2, + expectedComponents: []uint{1, 1}, + }, + { + version: "1.3.2-alpha+001", + diff: 3, + expectedComponents: []uint{1, 0}, + }, + { + version: "1.20", + diff: 5, + expectedComponents: []uint{1, 15}, + }, + } + + for _, test := range tests { + version, _ := ParseGeneric(test.version) + if !reflect.DeepEqual(test.expectedComponents, version.SubtractMinor(test.diff).Components()) { + t.Error("parse returned un'expected components") + } + } +} + +func TestParse(t *testing.T) { + + var tests = []struct { + version string + expectErr bool + expectedComponents []uint + expectedPreRelease string + expectedBuildMetadata string + }{ + { + version: "1.0.2", + expectedComponents: []uint{1, 0, 2}, + }, + { + version: "1.0.2-alpha+001", + expectedComponents: []uint{1, 0, 2}, + expectedPreRelease: "alpha", + expectedBuildMetadata: "001", + }, + { + version: "1.2", + expectedComponents: []uint{1, 2}, + }, + { + version: "1.0.2-beta+exp.sha.5114f85", + expectedComponents: []uint{1, 0, 2}, + expectedPreRelease: "beta", + expectedBuildMetadata: "exp.sha.5114f85", + }, + { + version: "a.b.c", + expectErr: true, + }, + } + + for _, test := range tests { + version, err := Parse(test.version) + if test.expectErr { + if err == nil { + t.Fatalf("got no err, expected err") + } + continue + } + if !reflect.DeepEqual(test.expectedComponents, version.Components()) { + t.Error("parse returned un'expected components") + } + if test.expectedPreRelease != version.PreRelease() { + t.Errorf("parse returned version.PreRelease %s, expected %s", test.expectedPreRelease, version.PreRelease()) + } + if test.expectedBuildMetadata != version.BuildMetadata() { + t.Errorf("parse returned version.BuildMetadata %s, expected %s", test.expectedBuildMetadata, version.BuildMetadata()) + } + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go b/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go index 9153869995bfa..837cbacddb00b 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go @@ -30,20 +30,27 @@ import ( "k8s.io/apimachinery/pkg/util/version" celconfig "k8s.io/apiserver/pkg/apis/cel" "k8s.io/apiserver/pkg/cel/library" + utilversion "k8s.io/apiserver/pkg/util/version" ) // DefaultCompatibilityVersion returns a default compatibility version for use with EnvSet // that guarantees compatibility with CEL features/libraries/parameters understood by -// an n-1 version +// the api server min compatibility version // -// This default will be set to no more than n-1 the current Kubernetes major.minor version. +// This default will be set to no more than the current Kubernetes major.minor version. // -// Note that a default version number less than n-1 indicates a wider range of version -// compatibility than strictly required for rollback. A wide range of compatibility is -// desirable because it means that CEL expressions are portable across a wider range -// of Kubernetes versions. +// Note that a default version number less than n-1 the current Kubernetes major.minor version +// indicates a wider range of version compatibility than strictly required for rollback. +// A wide range of compatibility is desirable because it means that CEL expressions are portable +// across a wider range of Kubernetes versions. +// A default version number equal to the current Kubernetes major.minor version +// indicates fast forward CEL features that can be used when rollback is no longer needed. func DefaultCompatibilityVersion() *version.Version { - return version.MajorMinor(1, 30) + effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer) + if effectiveVer == nil { + effectiveVer = utilversion.DefaultKubeEffectiveVersion() + } + return effectiveVer.MinCompatibilityVersion() } var baseOpts = append(baseOptsWithoutStrictCost, StrictCostOpt) diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index 80dc25cc6b105..8448e36ec6afc 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -322,6 +322,17 @@ const ( func init() { runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)) + runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates)) +} + +// defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs. +// To add a new feature, define a key for it above and add it here. The features will be +// available throughout Kubernetes binaries. +var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{ + // Example: + // EmulationVersion: { + // {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, + // }, } // defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys. diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/options.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/options.go index d675a258f5e46..44d07c0e241d5 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/options.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/options.go @@ -19,6 +19,7 @@ package generic import ( "time" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend" @@ -39,12 +40,15 @@ type RESTOptions struct { } // Implement RESTOptionsGetter so that RESTOptions can directly be used when available (i.e. tests) -func (opts RESTOptions) GetRESTOptions(schema.GroupResource) (RESTOptions, error) { +func (opts RESTOptions) GetRESTOptions(schema.GroupResource, runtime.Object) (RESTOptions, error) { return opts, nil } type RESTOptionsGetter interface { - GetRESTOptions(resource schema.GroupResource) (RESTOptions, error) + // GetRESTOptions returns the RESTOptions for the given resource and example object. + // The example object is used to determine the storage version for the resource. + // If the example object is nil, the storage version will be determined by the resource's default storage version. + GetRESTOptions(resource schema.GroupResource, example runtime.Object) (RESTOptions, error) } // StoreOptions is set of configuration options used to complete generic registries. diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go index a8e01708a30d3..98efac93339dc 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go @@ -1518,7 +1518,7 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error { return err } - opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource) + opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource, e.NewFunc()) if err != nil { return err } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 0502448d6ebfc..b266cb2e49411 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -42,8 +42,9 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup" - "k8s.io/apimachinery/pkg/version" + apimachineryversion "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/authentication/authenticator" @@ -70,8 +71,10 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/informers" restclient "k8s.io/client-go/rest" + "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" "k8s.io/component-base/metrics/features" "k8s.io/component-base/metrics/prometheus/slis" @@ -148,7 +151,12 @@ type Config struct { PostStartHooks map[string]PostStartHookConfigEntry // Version will enable the /version endpoint if non-nil - Version *version.Info + Version *apimachineryversion.Info + // EffectiveVersion determines which apis and features are available + // based on when the api/feature lifecyle. + EffectiveVersion utilversion.EffectiveVersion + // FeatureGate is a way to plumb feature gate through if you have them. + FeatureGate featuregate.FeatureGate // AuditBackend is where audit events are sent to. AuditBackend audit.Backend // AuditPolicyRuleEvaluator makes the decision of whether and how to audit log a request. @@ -585,7 +593,7 @@ func (c *Config) AddPostStartHookOrDie(name string, hook PostStartHookFunc) { } } -func completeOpenAPI(config *openapicommon.Config, version *version.Info) { +func completeOpenAPI(config *openapicommon.Config, version *version.Version) { if config == nil { return } @@ -624,7 +632,7 @@ func completeOpenAPI(config *openapicommon.Config, version *version.Info) { } } -func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Info) { +func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Version) { if config == nil { return } @@ -676,6 +684,9 @@ func (c *Config) ShutdownInitiatedNotify() <-chan struct{} { // Complete fills in any fields not set that are required to have valid data and can be derived // from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver. func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig { + if c.FeatureGate == nil { + c.FeatureGate = utilfeature.DefaultFeatureGate + } if len(c.ExternalAddress) == 0 && c.PublicAddress != nil { c.ExternalAddress = c.PublicAddress.String() } @@ -691,9 +702,12 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo } c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port)) } - - completeOpenAPI(c.OpenAPIConfig, c.Version) - completeOpenAPIV3(c.OpenAPIV3Config, c.Version) + var ver *version.Version + if c.EffectiveVersion != nil { + ver = c.EffectiveVersion.EmulationVersion() + } + completeOpenAPI(c.OpenAPIConfig, ver) + completeOpenAPIV3(c.OpenAPIV3Config, ver) if c.DiscoveryAddresses == nil { c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress} @@ -711,7 +725,7 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo } else { c.EquivalentResourceRegistry = runtime.NewEquivalentResourceRegistryWithIdentity(func(groupResource schema.GroupResource) string { // use the storage prefix as the key if possible - if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource); err == nil { + if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource, nil); err == nil { return opts.ResourcePrefix } // otherwise return "" to use the default key (parent GV name) @@ -819,7 +833,9 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G APIServerID: c.APIServerID, StorageVersionManager: c.StorageVersionManager, - Version: c.Version, + EffectiveVersion: c.EffectiveVersion, + Version: c.Version, + FeatureGate: c.FeatureGate, muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{}, } diff --git a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go index f14ed9213d133..841c1f5729cb5 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go +++ b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go @@ -17,6 +17,7 @@ limitations under the License. package server import ( + "fmt" "os" "strconv" "strings" @@ -25,16 +26,15 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" - apimachineryversion "k8s.io/apimachinery/pkg/version" + apimachineryversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/klog/v2" ) // resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API type resourceExpirationEvaluator struct { - currentMajor int - currentMinor int - isAlpha bool + currentVersion *apimachineryversion.Version + isAlpha bool // This is usually set for testing for which tests need to be removed. This prevent insta-failing CI. // Set KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA to see what will be removed when we tag beta strictRemovedHandlingInAlpha bool @@ -53,30 +53,17 @@ type ResourceExpirationEvaluator interface { ShouldServeForVersion(majorRemoved, minorRemoved int) bool } -func NewResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (ResourceExpirationEvaluator, error) { +func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version) (ResourceExpirationEvaluator, error) { + if currentVersion == nil { + return nil, fmt.Errorf("empty NewResourceExpirationEvaluator currentVersion") + } + klog.V(1).Infof("NewResourceExpirationEvaluator with currentVersion: %s.", currentVersion) ret := &resourceExpirationEvaluator{ strictRemovedHandlingInAlpha: false, } - if len(currentVersion.Major) > 0 { - currentMajor64, err := strconv.ParseInt(currentVersion.Major, 10, 32) - if err != nil { - return nil, err - } - ret.currentMajor = int(currentMajor64) - } - if len(currentVersion.Minor) > 0 { - // split the "normal" + and - for semver stuff - minorString := strings.Split(currentVersion.Minor, "+")[0] - minorString = strings.Split(minorString, "-")[0] - minorString = strings.Split(minorString, ".")[0] - currentMinor64, err := strconv.ParseInt(minorString, 10, 32) - if err != nil { - return nil, err - } - ret.currentMinor = int(currentMinor64) - } - - ret.isAlpha = strings.Contains(currentVersion.GitVersion, "alpha") + // Only keeps the major and minor versions from input version. + ret.currentVersion = apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor()) + ret.isAlpha = strings.Contains(currentVersion.PreRelease(), "alpha") if envString, ok := os.LookupEnv("KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA"); !ok { // do nothing @@ -112,6 +99,16 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio return false } + introduced, ok := versionedPtr.(introducedInterface) + // skip the introduced check for test where currentVersion is 0.0 + if ok && (e.currentVersion.Major() > 0 || e.currentVersion.Minor() > 0) { + majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced() + verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced)) + if e.currentVersion.LessThan(verIntroduced) { + return false + } + } + removed, ok := versionedPtr.(removedInterface) if !ok { return true @@ -121,16 +118,11 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio } func (e *resourceExpirationEvaluator) ShouldServeForVersion(majorRemoved, minorRemoved int) bool { - if e.currentMajor < majorRemoved { + removedVer := apimachineryversion.MajorMinor(uint(majorRemoved), uint(minorRemoved)) + if removedVer.GreaterThan(e.currentVersion) { return true } - if e.currentMajor > majorRemoved { - return false - } - if e.currentMinor < minorRemoved { - return true - } - if e.currentMinor > minorRemoved { + if removedVer.LessThan(e.currentVersion) { return false } // at this point major and minor are equal, so this API should be removed when the current release GAs. @@ -152,6 +144,11 @@ type removedInterface interface { APILifecycleRemoved() (major, minor int) } +// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +type introducedInterface interface { + APILifecycleIntroduced() (major, minor int) +} + // removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted. // versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage. func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) { @@ -171,6 +168,8 @@ func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versi } klog.V(1).Infof("Removing resource %v.%v.%v because it is time to stop serving it per APILifecycle.", resourceName, apiVersion, groupName) + storage := versionToResource[resourceName] + storage.Destroy() delete(versionToResource, resourceName) } versionedResourcesStorageMap[apiVersion] = versionToResource diff --git a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go index c033990b4057f..84403fd15870f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go @@ -25,57 +25,41 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/dump" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/version" + apimachineryversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/registry/rest" ) func Test_newResourceExpirationEvaluator(t *testing.T) { tests := []struct { name string - currentVersion version.Info + currentVersion string expected resourceExpirationEvaluator expectedErr string }{ { - name: "beta", - currentVersion: version.Info{ - Major: "1", - Minor: "20+", - GitVersion: "v1.20.0-beta.0.62+a5d22854a2ac21", - }, - expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20}, + name: "beta", + currentVersion: "v1.20.0-beta.0.62+a5d22854a2ac21", + expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)}, }, { - name: "alpha", - currentVersion: version.Info{ - Major: "1", - Minor: "20+", - GitVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21", - }, - expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20, isAlpha: true}, + name: "alpha", + currentVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21", + expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20), isAlpha: true}, }, { - name: "maintenance", - currentVersion: version.Info{ - Major: "1", - Minor: "20+", - GitVersion: "v1.20.1", - }, - expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20}, + name: "maintenance", + currentVersion: "v1.20.1", + expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)}, }, { - name: "bad", - currentVersion: version.Info{ - Major: "1", - Minor: "20something+", - GitVersion: "v1.20.1", - }, - expectedErr: `strconv.ParseInt: parsing "20something": invalid syntax`, + name: "no v prefix", + currentVersion: "1.20.1", + expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actual, actualErr := NewResourceExpirationEvaluator(tt.currentVersion) + actual, actualErr := NewResourceExpirationEvaluator(apimachineryversion.MustParse(tt.currentVersion)) checkErr(t, actualErr, tt.expectedErr) if actualErr != nil { @@ -90,12 +74,12 @@ func Test_newResourceExpirationEvaluator(t *testing.T) { } } -func storageRemovedIn(major, minor int) removedInStorage { - return removedInStorage{major: major, minor: minor} +func storageRemovedIn(major, minor int) *removedInStorage { + return &removedInStorage{major: major, minor: minor} } -func storageNeverRemoved() removedInStorage { - return removedInStorage{neverRemoved: true} +func storageNeverRemoved() *removedInStorage { + return &removedInStorage{neverRemoved: true} } type removedInStorage struct { @@ -103,23 +87,23 @@ type removedInStorage struct { neverRemoved bool } -func (r removedInStorage) New() runtime.Object { +func (r *removedInStorage) New() runtime.Object { if r.neverRemoved { - return neverRemovedObj{} + return &defaultObj{} } - return removedInObj{major: r.major, minor: r.minor} + return &removedInObj{major: r.major, minor: r.minor} } -func (r removedInStorage) Destroy() { +func (r *removedInStorage) Destroy() { } -type neverRemovedObj struct { +type defaultObj struct { } -func (r neverRemovedObj) GetObjectKind() schema.ObjectKind { +func (r *defaultObj) GetObjectKind() schema.ObjectKind { panic("don't do this") } -func (r neverRemovedObj) DeepCopyObject() runtime.Object { +func (r *defaultObj) DeepCopyObject() runtime.Object { panic("don't do this either") } @@ -127,13 +111,45 @@ type removedInObj struct { major, minor int } -func (r removedInObj) GetObjectKind() schema.ObjectKind { +func (r *removedInObj) GetObjectKind() schema.ObjectKind { + panic("don't do this") +} +func (r *removedInObj) DeepCopyObject() runtime.Object { + panic("don't do this either") +} +func (r *removedInObj) APILifecycleRemoved() (major, minor int) { + return r.major, r.minor +} + +func storageIntroducedIn(major, minor int) *introducedInStorage { + return &introducedInStorage{major: major, minor: minor} +} + +type introducedInStorage struct { + major, minor int +} + +func (r *introducedInStorage) New() runtime.Object { + if r.major == 0 && r.minor == 0 { + return &defaultObj{} + } + return &IntroducedInObj{major: r.major, minor: r.minor} +} + +func (r *introducedInStorage) Destroy() { +} + +type IntroducedInObj struct { + major, minor int +} + +func (r *IntroducedInObj) GetObjectKind() schema.ObjectKind { panic("don't do this") } -func (r removedInObj) DeepCopyObject() runtime.Object { +func (r *IntroducedInObj) DeepCopyObject() runtime.Object { panic("don't do this either") } -func (r removedInObj) APILifecycleRemoved() (major, minor int) { +func (r *IntroducedInObj) APILifecycleIntroduced() (major, minor int) { return r.major, r.minor } @@ -147,8 +163,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-curr", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, restStorage: storageRemovedIn(1, 20), expected: false, @@ -156,8 +171,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-curr-but-deferred", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), serveRemovedAPIsOneMoreRelease: true, }, restStorage: storageRemovedIn(1, 20), @@ -166,9 +180,8 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-curr-but-alpha", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, - isAlpha: true, + currentVersion: apimachineryversion.MajorMinor(1, 20), + isAlpha: true, }, restStorage: storageRemovedIn(1, 20), expected: true, @@ -176,8 +189,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-curr-but-alpha-but-strict", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), isAlpha: true, strictRemovedHandlingInAlpha: true, }, @@ -187,8 +199,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-prev-deferral-does-not-help", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 21, + currentVersion: apimachineryversion.MajorMinor(1, 21), serveRemovedAPIsOneMoreRelease: true, }, restStorage: storageRemovedIn(1, 20), @@ -197,8 +208,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-prev-major", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 2, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(2, 20), serveRemovedAPIsOneMoreRelease: true, }, restStorage: storageRemovedIn(1, 20), @@ -207,8 +217,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-future", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, restStorage: storageRemovedIn(1, 21), expected: true, @@ -216,12 +225,43 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "never-removed", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, restStorage: storageNeverRemoved(), expected: true, }, + { + name: "introduced-in-curr", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + }, + restStorage: storageIntroducedIn(1, 20), + expected: true, + }, + { + name: "introduced-in-prev-major", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + }, + restStorage: storageIntroducedIn(1, 19), + expected: true, + }, + { + name: "introduced-in-future", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + }, + restStorage: storageIntroducedIn(1, 21), + expected: false, + }, + { + name: "missing-introduced", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + }, + restStorage: storageIntroducedIn(0, 0), + expected: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -269,8 +309,7 @@ func Test_removeDeletedKinds(t *testing.T) { { name: "remove-one-of-two", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, versionedResourcesStorageMap: map[string]map[string]rest.Storage{ "v1": { @@ -287,8 +326,7 @@ func Test_removeDeletedKinds(t *testing.T) { { name: "remove-nested-not-expired", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, versionedResourcesStorageMap: map[string]map[string]rest.Storage{ "v1": { @@ -306,8 +344,7 @@ func Test_removeDeletedKinds(t *testing.T) { { name: "remove-all-of-version", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, versionedResourcesStorageMap: map[string]map[string]rest.Storage{ "v1": { diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index e0dcbf75849ce..c47fc5b23f064 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -53,7 +53,9 @@ import ( "k8s.io/apiserver/pkg/server/routes" "k8s.io/apiserver/pkg/storageversion" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" restclient "k8s.io/client-go/rest" + "k8s.io/component-base/featuregate" "k8s.io/klog/v2" openapibuilder3 "k8s.io/kube-openapi/pkg/builder3" openapicommon "k8s.io/kube-openapi/pkg/common" @@ -238,6 +240,11 @@ type GenericAPIServer struct { // Version will enable the /version endpoint if non-nil Version *version.Info + // EffectiveVersion determines which apis and features are available + // based on when the api/feature lifecyle. + EffectiveVersion utilversion.EffectiveVersion + // FeatureGate is a way to plumb feature gate through if you have them. + FeatureGate featuregate.FeatureGate // lifecycleSignals provides access to the various signals that happen during the life cycle of the apiserver. lifecycleSignals lifecycleSignals diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go index 3549b238c9f77..c2bfef4dda498 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go @@ -48,6 +48,7 @@ import ( openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" "k8s.io/apiserver/pkg/registry/rest" genericfilters "k8s.io/apiserver/pkg/server/filters" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/apiserver/pkg/warning" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" @@ -460,6 +461,7 @@ func TestNotRestRoutesHaveAuth(t *testing.T) { kubeVersion := fakeVersion() config.Version = &kubeVersion + config.EffectiveVersion = utilversion.NewEffectiveVersion(kubeVersion.String()) s, err := config.Complete(nil).New("test", NewEmptyDelegate()) if err != nil { @@ -586,7 +588,7 @@ func fakeVersion() version.Info { return version.Info{ Major: "42", Minor: "42", - GitVersion: "42", + GitVersion: "42.42", GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689", GitTreeState: "Dirty", BuildDate: time.Now().String(), diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go index b37931eb28cc9..7d60244004d69 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go @@ -32,46 +32,42 @@ func (f fakeGroupRegistry) IsGroupRegistered(group string) bool { func TestAPIEnablementOptionsValidate(t *testing.T) { testCases := []struct { - name string - testOptions *APIEnablementOptions - expectErr string + name string + testOptions *APIEnablementOptions + runtimeConfig cliflag.ConfigurationMap + expectErr string }{ { name: "test when options is nil", }, { - name: "test when invalid key with only api/all=false", - testOptions: &APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{"api/all": "false"}, - }, - expectErr: "invalid key with only api/all=false", + name: "test when invalid key with only api/all=false", + runtimeConfig: cliflag.ConfigurationMap{"api/all": "false"}, + expectErr: "invalid key with only api/all=false", }, { - name: "test when ConfigurationMap key is invalid", - testOptions: &APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{"apiall": "false"}, - }, - expectErr: "runtime-config invalid key", + name: "test when ConfigurationMap key is invalid", + runtimeConfig: cliflag.ConfigurationMap{"apiall": "false"}, + expectErr: "runtime-config invalid key", }, { - name: "test when unknown api groups", - testOptions: &APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{"api/v1": "true"}, - }, - expectErr: "unknown api groups", + name: "test when unknown api groups", + runtimeConfig: cliflag.ConfigurationMap{"api/v1": "true"}, + expectErr: "unknown api groups", }, { - name: "test when valid api groups", - testOptions: &APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"}, - }, + name: "test when valid api groups", + runtimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"}, }, } testGroupRegistry := fakeGroupRegistry{} for _, testcase := range testCases { t.Run(testcase.name, func(t *testing.T) { - errs := testcase.testOptions.Validate(testGroupRegistry) + testOptions := &APIEnablementOptions{ + RuntimeConfig: testcase.runtimeConfig, + } + errs := testOptions.Validate(testGroupRegistry) if len(testcase.expectErr) != 0 && !strings.Contains(utilerrors.NewAggregate(errs).Error(), testcase.expectErr) { t.Errorf("got err: %v, expected err: %s", errs, testcase.expectErr) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go index 10f9775efcc19..af7696c40ae68 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go @@ -383,8 +383,8 @@ type StorageFactoryRestOptionsFactory struct { StorageFactory serverstorage.StorageFactory } -func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) { - storageConfig, err := f.StorageFactory.NewConfig(resource) +func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) { + storageConfig, err := f.StorageFactory.NewConfig(resource, example) if err != nil { return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error()) } @@ -469,7 +469,7 @@ type SimpleStorageFactory struct { StorageConfig storagebackend.Config } -func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) { +func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { return s.StorageConfig.ForResource(resource), nil } @@ -493,8 +493,8 @@ type transformerStorageFactory struct { resourceTransformers storagevalue.ResourceTransformers } -func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) { - config, err := t.delegate.NewConfig(resource) +func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { + config, err := t.delegate.NewConfig(resource, example) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go index 06d0cae0a6ce5..ea4464b24f80b 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go @@ -437,7 +437,7 @@ func TestRestOptionsStorageObjectCountTracker(t *testing.T) { if err := etcdOptions.ApplyTo(serverConfig); err != nil { t.Fatalf("Failed to apply etcd options error: %v", err) } - restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}) + restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go index 1373d8a4d730b..88dc3802f7fc5 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go @@ -26,7 +26,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apiserver/pkg/server" - utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/component-base/featuregate" "github.com/spf13/pflag" ) @@ -89,9 +90,13 @@ type ServerRunOptions struct { // This grace period is orthogonal to other grace periods, and // it is not overridden by any other grace period. ShutdownWatchTerminationGracePeriod time.Duration + + // FeatureGate are the featuregate to install on the CLI + FeatureGate featuregate.FeatureGate + EffectiveVersion utilversion.EffectiveVersion } -func NewServerRunOptions() *ServerRunOptions { +func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions { defaults := server.NewConfig(serializer.CodecFactory{}) return &ServerRunOptions{ MaxRequestsInFlight: defaults.MaxRequestsInFlight, @@ -104,6 +109,8 @@ func NewServerRunOptions() *ServerRunOptions { JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes, MaxRequestBodyBytes: defaults.MaxRequestBodyBytes, ShutdownSendRetryAfter: false, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, } } @@ -124,6 +131,8 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error { c.PublicAddress = s.AdvertiseAddress c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod + c.EffectiveVersion = s.EffectiveVersion + c.FeatureGate = s.FeatureGate return nil } @@ -196,6 +205,14 @@ func (s *ServerRunOptions) Validate() []error { if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil { errors = append(errors, err) } + if s.FeatureGate != nil { + if errs := s.FeatureGate.Validate(); len(errs) != 0 { + errors = append(errors, errs...) + } + } + if errs := s.EffectiveVersion.Validate(); len(errs) != 0 { + errors = append(errors, errs...) + } return errors } @@ -336,6 +353,15 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+ "This option, if set, represents the maximum amount of grace period the apiserver will wait "+ "for active watch request(s) to drain during the graceful server shutdown window.") +} - utilfeature.DefaultMutableFeatureGate.AddFlag(fs) +// Complete fills missing fields with defaults. +func (s *ServerRunOptions) Complete() error { + if s.FeatureGate == nil { + return fmt.Errorf("nil FeatureGate in ServerRunOptions") + } + if s.EffectiveVersion == nil { + return fmt.Errorf("nil EffectiveVersion in ServerRunOptions") + } + return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go index d1f67fe071902..0c62a4d689339 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go @@ -23,10 +23,14 @@ import ( "time" utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" netutils "k8s.io/utils/net" ) func TestServerRunOptionsValidate(t *testing.T) { + featureGate := utilfeature.DefaultFeatureGate.DeepCopy() + effectiveVersion := utilversion.NewEffectiveVersion("1.30") testCases := []struct { name string testOptions *ServerRunOptions @@ -43,6 +47,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--max-requests-inflight can not be negative value", }, @@ -57,6 +63,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--max-mutating-requests-inflight can not be negative value", }, @@ -71,6 +79,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--request-timeout can not be negative value", }, @@ -85,6 +95,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: -1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--min-request-timeout can not be negative value", }, @@ -99,6 +111,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: -10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value", }, @@ -113,6 +127,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: -10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value", }, @@ -128,6 +144,8 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, LivezGracePeriod: -time.Second, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--livez-grace-period can not be a negative value", }, @@ -143,6 +161,8 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, ShutdownDelayDuration: -time.Second, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--shutdown-delay-duration can not be negative value", }, @@ -158,6 +178,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information", }, @@ -173,6 +195,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, }, } @@ -192,6 +216,8 @@ func TestServerRunOptionsValidate(t *testing.T) { } func TestValidateCorsAllowedOriginList(t *testing.T) { + featureGate := utilfeature.DefaultFeatureGate.DeepCopy() + effectiveVersion := utilversion.NewEffectiveVersion("1.30") tests := []struct { regexp [][]string errShouldContain string @@ -239,7 +265,7 @@ func TestValidateCorsAllowedOriginList(t *testing.T) { for _, test := range tests { for _, regexp := range test.regexp { t.Run(fmt.Sprintf("regexp/%s", regexp), func(t *testing.T) { - options := NewServerRunOptions() + options := NewServerRunOptions(featureGate, effectiveVersion) if errs := options.Validate(); len(errs) != 0 { t.Fatalf("wrong test setup: %#v", errs) } @@ -263,6 +289,8 @@ func TestValidateCorsAllowedOriginList(t *testing.T) { } func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { + featureGate := utilfeature.DefaultFeatureGate.DeepCopy() + effectiveVersion := utilversion.NewEffectiveVersion("1.30") tests := []struct { name string optionsFn func() *ServerRunOptions @@ -271,13 +299,13 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { { name: "default should be valid", optionsFn: func() *ServerRunOptions { - return NewServerRunOptions() + return NewServerRunOptions(featureGate, effectiveVersion) }, }, { name: "negative not allowed", optionsFn: func() *ServerRunOptions { - o := NewServerRunOptions() + o := NewServerRunOptions(featureGate, effectiveVersion) o.ShutdownWatchTerminationGracePeriod = -time.Second return o }, @@ -304,7 +332,7 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { } t.Run("default should be zero", func(t *testing.T) { - options := NewServerRunOptions() + options := NewServerRunOptions(featureGate, effectiveVersion) if options.ShutdownWatchTerminationGracePeriod != time.Duration(0) { t.Errorf("expected default of ShutdownWatchTerminationGracePeriod to be zero, but got: %s", options.ShutdownWatchTerminationGracePeriod) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go index a08ce2b3bd3ad..40feeaf62c1ee 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go @@ -43,6 +43,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/server" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/discovery" restclient "k8s.io/client-go/rest" cliflag "k8s.io/component-base/cli/flag" @@ -276,9 +277,9 @@ func TestServerRunWithSNI(t *testing.T) { // launch server config := setUp(t) - v := fakeVersion() config.Version = &v + config.EffectiveVersion = utilversion.NewEffectiveVersion(v.String()) config.EnableIndex = true secureOptions := (&SecureServingOptions{ @@ -463,11 +464,9 @@ func certSignature(cert tls.Certificate) (string, error) { func fakeVersion() version.Info { return version.Info{ - Major: "42", - Minor: "42", - GitVersion: "42", - GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689", - GitTreeState: "Dirty", + Major: "42", + Minor: "42", + GitVersion: "42.42", } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go index efb22fbc8d6ca..5d5cfccc7fddf 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go @@ -21,6 +21,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + apimachineryversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apiserver/pkg/util/version" ) type ResourceEncodingConfig interface { @@ -33,10 +35,15 @@ type ResourceEncodingConfig interface { InMemoryEncodingFor(schema.GroupResource) (schema.GroupVersion, error) } +type CompatibilityResourceEncodingConfig interface { + BackwardCompatibileStorageEncodingFor(schema.GroupResource, runtime.Object) (schema.GroupVersion, error) +} + type DefaultResourceEncodingConfig struct { // resources records the overriding encoding configs for individual resources. - resources map[schema.GroupResource]*OverridingResourceEncoding - scheme *runtime.Scheme + resources map[schema.GroupResource]*OverridingResourceEncoding + scheme *runtime.Scheme + effectiveVersion version.EffectiveVersion } type OverridingResourceEncoding struct { @@ -47,7 +54,7 @@ type OverridingResourceEncoding struct { var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{} func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig { - return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme} + return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: version.DefaultKubeEffectiveVersion()} } func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored schema.GroupResource, externalEncodingVersion, internalVersion schema.GroupVersion) { @@ -57,6 +64,10 @@ func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored } } +func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion version.EffectiveVersion) { + o.effectiveVersion = effectiveVersion +} + func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) { if !o.scheme.IsGroupRegistered(resource.Group) { return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group) @@ -71,6 +82,24 @@ func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.Group return o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], nil } +func (o *DefaultResourceEncodingConfig) BackwardCompatibileStorageEncodingFor(resource schema.GroupResource, example runtime.Object) (schema.GroupVersion, error) { + if !o.scheme.IsGroupRegistered(resource.Group) { + return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group) + } + + // Always respect overrides + resourceOverride, resourceExists := o.resources[resource] + if resourceExists { + return resourceOverride.ExternalResourceEncoding, nil + } + + return emulatedStorageVersion( + o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], + example, + o.effectiveVersion, + o.scheme) +} + func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) { if !o.scheme.IsGroupRegistered(resource.Group) { return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group) @@ -82,3 +111,78 @@ func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.Grou } return schema.GroupVersion{Group: resource.Group, Version: runtime.APIVersionInternal}, nil } + +// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +type introducedInterface interface { + APILifecycleIntroduced() (major, minor int) +} + +func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion version.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) { + if example == nil || effectiveVersion == nil { + return binaryVersionOfResource, nil + } + + // Look up example in scheme to find all objects of the same Group-Kind + // Use the highest priority version for that group-kind whose lifecycle window + // includes the current emulation version. + // If no version is found, use the binary version + // (in this case the API should be disabled anyway) + gvks, _, err := scheme.ObjectKinds(example) + if err != nil { + return schema.GroupVersion{}, err + } else if len(gvks) == 0 { + // Probably shouldn't happen if err is non-nil + return schema.GroupVersion{}, fmt.Errorf("object %T has no GVKs registered in scheme", example) + } + + // VersionsForGroupKind returns versions in priority order + versions := scheme.VersionsForGroupKind(schema.GroupKind{Group: gvks[0].Group, Kind: gvks[0].Kind}) + + compatibilityVersion := effectiveVersion.MinCompatibilityVersion() + + for _, gv := range versions { + if gv.Version == runtime.APIVersionInternal { + continue + } + + gvk := schema.GroupVersionKind{ + Group: gv.Group, + Version: gv.Version, + Kind: gvks[0].Kind, + } + + exampleOfGVK, err := scheme.New(gvk) + if err != nil { + return schema.GroupVersion{}, err + } + + // If it was introduced after current compatibility version, don't use it + if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) { + // API resource lifecycles should be relative to k8s api version + majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced() + introducedVer := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced)) + if introducedVer.GreaterThan(compatibilityVersion) { + continue + } + } + + // versions is returned in priority order, so just use first result + return gvk.GroupVersion(), nil + } + + // Getting here means we're serving a version that is unknown to the + // min-compatibility-version server. + // + // This is only expected to happen when serving an alpha API type due + // to missing pre-release lifecycle information + // (which doesn't happen by default), or when emulation-version and + // min-compatibility-version are several versions apart so a beta or GA API + // was being served which didn't exist at all in min-compatibility-version. + // + // In the alpha case - we do not support compatibility versioning of + // alpha types and recommend users do not mix the two. + // In the skip-level case - The version of apiserver we are retaining + // compatibility with has no knowledge of the type, + // so storing it in another type is no issue. + return binaryVersionOfResource, nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go index 0dc50cea61d4b..ad01c5a5d8940 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go @@ -42,7 +42,7 @@ type Backend struct { type StorageFactory interface { // New finds the storage destination for the given group and resource. It will // return an error if the group has no storage destination configured. - NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) + NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) // ResourcePrefix returns the overridden resource prefix for the GroupResource // This allows for cohabitation of resources with different native types and provides @@ -226,7 +226,7 @@ func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.Gro // New finds the storage destination for the given group and resource. It will // return an error if the group has no storage destination configured. -func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) { +func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { chosenStorageResource := s.getStorageGroupResource(groupResource) // operate on copy @@ -244,14 +244,23 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (* } var err error - codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource) - if err != nil { - return nil, err + if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok { + codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(groupResource, example) + if err != nil { + return nil, err + } + } else { + codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource) + if err != nil { + return nil, err + } } + codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource) if err != nil { return nil, err } + codecConfig.Config = storageConfig storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig) diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go index c52049ca0ec09..a12e865f05954 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go @@ -26,10 +26,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" + apimachineryversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/apis/example" exampleinstall "k8s.io/apiserver/pkg/apis/example/install" examplev1 "k8s.io/apiserver/pkg/apis/example/v1" "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/util/version" ) var ( @@ -118,7 +120,7 @@ func TestConfigurableStorageFactory(t *testing.T) { f.SetEtcdLocation(example.Resource("*"), []string{"/server2"}) f.SetEtcdPrefix(example.Resource("test"), "/prefix_for_test") - config, err := f.NewConfig(example.Resource("test")) + config, err := f.NewConfig(example.Resource("test"), nil) if err != nil { t.Fatal(err) } @@ -163,7 +165,7 @@ func TestUpdateEtcdOverrides(t *testing.T) { storageFactory.SetEtcdLocation(test.resource, test.servers) var err error - config, err := storageFactory.NewConfig(test.resource) + config, err := storageFactory.NewConfig(test.resource, nil) if err != nil { t.Errorf("%d: unexpected error %v", i, err) continue @@ -173,7 +175,7 @@ func TestUpdateEtcdOverrides(t *testing.T) { continue } - config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"}) + config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"}, nil) if err != nil { t.Errorf("%d: unexpected error %v", i, err) continue @@ -244,3 +246,241 @@ func TestConfigs(t *testing.T) { } } } + +var introducedLifecycles = map[reflect.Type]*apimachineryversion.Version{} +var removedLifecycles = map[reflect.Type]*apimachineryversion.Version{} + +type fakeLifecycler[T, V any] struct { + metav1.TypeMeta + metav1.ObjectMeta +} + +type removedLifecycler[T, V any] struct { + fakeLifecycler[T, V] +} + +func (f *fakeLifecycler[T, V]) GetObjectKind() schema.ObjectKind { return f } +func (f *fakeLifecycler[T, V]) DeepCopyObject() runtime.Object { return f } +func (f *fakeLifecycler[T, V]) APILifecycleIntroduced() (major, minor int) { + if introduced, ok := introducedLifecycles[reflect.TypeOf(f)]; ok { + return int(introduced.Major()), int(introduced.Minor()) + } + panic("no lifecycle version set") +} +func (f *removedLifecycler[T, V]) APILifecycleRemoved() (major, minor int) { + if removed, ok := removedLifecycles[reflect.TypeOf(f)]; ok { + return int(removed.Major()), int(removed.Minor()) + } + panic("no lifecycle version set") +} + +func registerFakeLifecycle[T, V any](sch *runtime.Scheme, group, introduced, removed string) { + f := fakeLifecycler[T, V]{} + + introducedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(introduced) + + var res runtime.Object + if removed != "" { + removedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(removed) + res = &removedLifecycler[T, V]{fakeLifecycler: f} + } else { + res = &f + } + + var v V + var t T + sch.AddKnownTypeWithName( + schema.GroupVersionKind{ + Group: group, + Version: strings.ToLower(reflect.TypeOf(v).Name()), + Kind: reflect.TypeOf(t).Name(), + }, + res, + ) + + // Also ensure internal version is registered + // If it is registertd multiple times, it will ignore subsequent registrations + internalInstance := &fakeLifecycler[T, struct{}]{} + sch.AddKnownTypeWithName( + schema.GroupVersionKind{ + Group: group, + Version: runtime.APIVersionInternal, + Kind: reflect.TypeOf(t).Name(), + }, + internalInstance, + ) +} + +func TestStorageFactoryCompatibilityVersion(t *testing.T) { + // Creates a scheme with stub types for unit test + sch := runtime.NewScheme() + codecs := serializer.NewCodecFactory(sch) + + type Internal = struct{} + type V1beta1 struct{} + type V1beta2 struct{} + type V1beta3 struct{} + type V1 struct{} + + type Pod struct{} + type FlowSchema struct{} + type ValidatingAdmisisonPolicy struct{} + type CronJob struct{} + + // Order dictates priority order + registerFakeLifecycle[FlowSchema, V1](sch, "flowcontrol.apiserver.k8s.io", "1.29.0", "") + registerFakeLifecycle[FlowSchema, V1beta3](sch, "flowcontrol.apiserver.k8s.io", "1.26.0", "1.32.0") + registerFakeLifecycle[FlowSchema, V1beta2](sch, "flowcontrol.apiserver.k8s.io", "1.23.0", "1.29.0") + registerFakeLifecycle[FlowSchema, V1beta1](sch, "flowcontrol.apiserver.k8s.io", "1.20.0", "1.26.0") + registerFakeLifecycle[CronJob, V1](sch, "batch", "1.21.0", "") + registerFakeLifecycle[CronJob, V1beta1](sch, "batch", "1.8.0", "1.21.0") + registerFakeLifecycle[ValidatingAdmisisonPolicy, V1](sch, "admissionregistration.k8s.io", "1.30.0", "") + registerFakeLifecycle[ValidatingAdmisisonPolicy, V1beta1](sch, "admissionregistration.k8s.io", "1.28.0", "1.34.0") + registerFakeLifecycle[Pod, V1](sch, "", "1.31.0", "") + + // FlowSchema + // - v1beta1: 1.20.0 - 1.23.0 + // - v1beta2: 1.23.0 - 1.26.0 + // - v1beta3: 1.26.0 - 1.30.0 + // - v1: 1.29.0+ + // CronJob + // - v1beta1: 1.8.0 - 1.21.0 + // - v1: 1.21.0+ + // ValidatingAdmissionPolicy + // - v1beta1: 1.28.0 - 1.31.0 + // - v1: 1.30.0+ + + testcases := []struct { + effectiveVersion string + example runtime.Object + expectedVersion schema.GroupVersion + }{ + { + // Basic case. Beta version for long time + effectiveVersion: "1.14.0", + example: &fakeLifecycler[CronJob, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"}, + }, + { + // Basic case. Beta version for long time + effectiveVersion: "1.20.0", + example: &fakeLifecycler[CronJob, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"}, + }, + { + // Basic case. GA version for long time + effectiveVersion: "1.28.0", + example: &fakeLifecycler[CronJob, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1"}, + }, + { + // Basic core/v1 + effectiveVersion: "1.31.0", + example: &fakeLifecycler[Pod, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "", Version: "v1"}, + }, + { + // Corner case: 1.1.0 has no flowcontrol. Options are to error + // out or to use the latest version. This test assumes the latter. + effectiveVersion: "1.1.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"}, + }, + { + effectiveVersion: "1.21.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"}, + }, + { + // v2Beta1 introduced this version, but minCompatibility should + // force v1beta1 + effectiveVersion: "1.23.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"}, + }, + { + effectiveVersion: "1.24.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"}, + }, + { + effectiveVersion: "1.26.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"}, + }, + { + effectiveVersion: "1.27.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"}, + }, + { + // GA API introduced 1.29 but must keep storing in v1beta3 for downgrades + effectiveVersion: "1.29.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"}, + }, + { + // Version after GA api is introduced + effectiveVersion: "1.30.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"}, + }, + { + effectiveVersion: "1.30.0", + example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"}, + }, + { + effectiveVersion: "1.31.0", + example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1"}, + }, + { + effectiveVersion: "1.29.0", + example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"}, + }, + } + + for _, tc := range testcases { + gvks, _, err := sch.ObjectKinds(tc.example) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + gvk := gvks[0] + t.Run(gvk.GroupKind().String()+"@"+tc.effectiveVersion, func(t *testing.T) { + config := NewDefaultResourceEncodingConfig(sch) + config.SetEffectiveVersion(version.NewEffectiveVersion(tc.effectiveVersion)) + f := NewDefaultStorageFactory( + storagebackend.Config{}, + "", + codecs, + config, + NewResourceConfig(), + nil) + + cfg, err := f.NewConfig(schema.GroupResource{ + Group: gvk.Group, + Resource: gvk.Kind, // doesnt really matter here + }, tc.example) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + gvks, _, err := sch.ObjectKinds(tc.example) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + expectEncodeVersioner := runtime.NewMultiGroupVersioner(tc.expectedVersion, + schema.GroupKind{ + Group: gvks[0].Group, + }, schema.GroupKind{ + Group: gvks[0].Group, + }) + if cfg.EncodeVersioner.Identifier() != expectEncodeVersioner.Identifier() { + t.Errorf("expected %v, got %v", expectEncodeVersioner, cfg.EncodeVersioner) + } + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go b/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go index 7dd3df589e374..00a9e099ba7d9 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go +++ b/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go @@ -25,7 +25,7 @@ var ( // Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this. // Tests that need to modify feature gates for the duration of their test should use: // featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., ) - DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate() + DefaultMutableFeatureGate featuregate.MutableVersionedFeatureGate = featuregate.NewFeatureGate() // DefaultFeatureGate is a shared global FeatureGate. // Top-level commands/options setup that needs to modify this feature gate should use DefaultMutableFeatureGate. diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go new file mode 100644 index 0000000000000..f8b4ef5cd4100 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go @@ -0,0 +1,142 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "fmt" + "sync" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/component-base/featuregate" +) + +var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry() + +const ( + ComponentGenericAPIServer = "k8s.io/apiserver" +) + +// ComponentGlobals stores the global variables for a component for easy access. +type ComponentGlobals struct { + effectiveVersion MutableEffectiveVersion + featureGate featuregate.MutableVersionedFeatureGate +} + +type ComponentGlobalsRegistry interface { + // EffectiveVersionFor returns the EffectiveVersion registered under the component. + // Returns nil if the component is not registered. + EffectiveVersionFor(component string) EffectiveVersion + // FeatureGateFor returns the FeatureGate registered under the component. + // Returns nil if the component is not registered. + FeatureGateFor(component string) featuregate.FeatureGate + // Register registers the EffectiveVersion and FeatureGate for a component. + // Overrides existing ComponentGlobals if it is already in the registry if override is true, + // otherwise returns error if the component is already registered. + Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error + // ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry. + // Otherwise, the provided variables would be registered under the component, and the same variables would be returned. + ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) + // SetAllComponents sets the emulation version for other global variables for all components registered. + SetAllComponents() error + // SetAllComponents calls the Validate() function for all the global variables for all components registered. + ValidateAllComponents() []error +} + +type componentGlobalsRegistry struct { + componentGlobals map[string]ComponentGlobals + mutex sync.RWMutex +} + +func NewComponentGlobalsRegistry() ComponentGlobalsRegistry { + return &componentGlobalsRegistry{componentGlobals: map[string]ComponentGlobals{}} +} + +func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { + r.mutex.RLock() + defer r.mutex.RUnlock() + globals, ok := r.componentGlobals[component] + if !ok { + return nil + } + return globals.effectiveVersion +} + +func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate.FeatureGate { + r.mutex.RLock() + defer r.mutex.RUnlock() + globals, ok := r.componentGlobals[component] + if !ok { + return nil + } + return globals.featureGate +} + +func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { + if _, ok := r.componentGlobals[component]; ok && !override { + return fmt.Errorf("component globals of %s already registered", component) + } + if featureGate != nil { + featureGate.DeferErrorsToValidation(true) + } + c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate} + r.componentGlobals[component] = c + return nil +} + +func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { + r.mutex.Lock() + defer r.mutex.Unlock() + return r.unsafeRegister(component, effectiveVersion, featureGate, override) +} + +func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) { + r.mutex.Lock() + defer r.mutex.Unlock() + globals, ok := r.componentGlobals[component] + if ok { + return globals.effectiveVersion, globals.featureGate + } + utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate, false)) + return effectiveVersion, featureGate +} + +func (r *componentGlobalsRegistry) SetAllComponents() error { + r.mutex.Lock() + defer r.mutex.Unlock() + for _, globals := range r.componentGlobals { + if globals.featureGate == nil { + continue + } + if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil { + return err + } + } + return nil +} + +func (r *componentGlobalsRegistry) ValidateAllComponents() []error { + var errs []error + r.mutex.Lock() + defer r.mutex.Unlock() + for _, globals := range r.componentGlobals { + errs = append(errs, globals.effectiveVersion.Validate()...) + if globals.featureGate != nil { + errs = append(errs, globals.featureGate.Validate()...) + } + } + return errs +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go new file mode 100644 index 0000000000000..2e19552703f4a --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go @@ -0,0 +1,48 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "testing" +) + +func TestEffectiveVersionRegistry(t *testing.T) { + r := NewComponentGlobalsRegistry() + testComponent := "test" + ver1 := NewEffectiveVersion("1.31") + ver2 := NewEffectiveVersion("1.28") + + if r.EffectiveVersionFor(testComponent) != nil { + t.Fatalf("expected nil EffectiveVersion initially") + } + if err := r.Register(testComponent, ver1, nil, false); err != nil { + t.Fatalf("expected no error to register new component, but got err: %v", err) + } + if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) { + t.Fatalf("expected EffectiveVersionFor to return the version registered") + } + // overwrite + if err := r.Register(testComponent, ver2, nil, false); err == nil { + t.Fatalf("expected error to register existing component when override is false") + } + if err := r.Register(testComponent, ver2, nil, true); err != nil { + t.Fatalf("expected no error to overriding existing component, but got err: %v", err) + } + if !r.EffectiveVersionFor(testComponent).EqualTo(ver2) { + t.Fatalf("expected EffectiveVersionFor to return the version overridden") + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version.go b/staging/src/k8s.io/apiserver/pkg/util/version/version.go new file mode 100644 index 0000000000000..fce927c6c613e --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version.go @@ -0,0 +1,195 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "fmt" + "strings" + "sync/atomic" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/version" + baseversion "k8s.io/component-base/version" +) + +type EffectiveVersion interface { + BinaryVersion() *version.Version + EmulationVersion() *version.Version + MinCompatibilityVersion() *version.Version + EqualTo(other EffectiveVersion) bool + String() string + Validate() []error +} + +type MutableEffectiveVersion interface { + EffectiveVersion + Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) + SetEmulationVersion(emulationVersion *version.Version) + SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) + // AddFlags adds the "{prefix}-emulated-version" to the flagset. + AddFlags(fs *pflag.FlagSet, prefix string) +} + +type VersionVar struct { + val atomic.Pointer[version.Version] +} + +// Set sets the flag value +func (v *VersionVar) Set(s string) error { + components := strings.Split(s, ".") + if len(components) != 2 { + return fmt.Errorf("version %s is not in the format of major.minor", s) + } + ver, err := version.ParseGeneric(s) + if err != nil { + return err + } + v.val.Store(ver) + return nil +} + +// String returns the flag value +func (v *VersionVar) String() string { + ver := v.val.Load() + return ver.String() +} + +// Type gets the flag type +func (v *VersionVar) Type() string { + return "version" +} + +type effectiveVersion struct { + binaryVersion atomic.Pointer[version.Version] + // If the emulationVersion is set by the users, it could only contain major and minor versions. + // In tests, emulationVersion could be the same as the binary version, or set directly, + // which can have "alpha" as pre-release to continue serving expired apis while we clean up the test. + emulationVersion VersionVar + // minCompatibilityVersion could only contain major and minor versions. + minCompatibilityVersion VersionVar +} + +func (m *effectiveVersion) BinaryVersion() *version.Version { + return m.binaryVersion.Load() +} + +func (m *effectiveVersion) EmulationVersion() *version.Version { + // Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test. + // The pre-release should not be accessible to the users. + return m.emulationVersion.val.Load().WithPreRelease(m.BinaryVersion().PreRelease()) +} + +func (m *effectiveVersion) MinCompatibilityVersion() *version.Version { + return m.minCompatibilityVersion.val.Load() +} + +func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool { + return m.BinaryVersion().EqualTo(other.BinaryVersion()) && m.EmulationVersion().EqualTo(other.EmulationVersion()) && m.MinCompatibilityVersion().EqualTo(other.MinCompatibilityVersion()) +} + +func (m *effectiveVersion) String() string { + if m == nil { + return "" + } + return fmt.Sprintf("{BinaryVersion: %s, EmulationVersion: %s, MinCompatibilityVersion: %s}", + m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String()) +} + +func (m *effectiveVersion) Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) { + m.binaryVersion.Store(binaryVersion) + m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor())) + m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor())) +} + +func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) { + m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor())) +} + +func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) { + m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor())) +} + +func (m *effectiveVersion) Validate() []error { + var errs []error + // Validate only checks the major and minor versions. + binaryVersion := m.binaryVersion.Load().WithPatch(0) + emulationVersion := m.emulationVersion.val.Load() + minCompatibilityVersion := m.minCompatibilityVersion.val.Load() + + // emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}. + maxEmuVer := binaryVersion + minEmuVer := binaryVersion.SubtractMinor(1) + // TODO: remove in 1.32 + // emulationVersion is introduced in 1.31, so it cannot be lower than that. + // binaryVersion could be lower than 1.31 in tests. So we are only checking 1.31. + if binaryVersion.EqualTo(version.MajorMinor(1, 31)) { + minEmuVer = version.MajorMinor(1, 31) + } + if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) { + errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String())) + } + // minCompatibilityVersion can only be 1.{binaryMinor-1} for alpha. + maxCompVer := binaryVersion.SubtractMinor(1) + minCompVer := binaryVersion.SubtractMinor(1) + if minCompatibilityVersion.GreaterThan(maxCompVer) || minCompatibilityVersion.LessThan(minCompVer) { + errs = append(errs, fmt.Errorf("minCompatibilityVersion version %s is not between [%s, %s]", minCompatibilityVersion.String(), minCompVer.String(), maxCompVer.String())) + } + return errs +} + +// AddFlags adds the "{prefix}-emulated-version" to the flagset. +func (m *effectiveVersion) AddFlags(fs *pflag.FlagSet, prefix string) { + if m == nil { + return + } + if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") { + prefix += "-" + } + fs.Var(&m.emulationVersion, prefix+"emulated-version", ""+ + "The version the K8s component emulates its capabilities (APIs, features, ...) of.\n"+ + "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ + "Any capabilities present in the binary version that were introduced after the emulated version will be unavailable and any capabilities removed after the emulated version will be available.\n"+ + "This flag applies only to component capabilities, and does not disable bug fixes and performance improvements present in the binary version.\n"+ + "Defaults to the binary version. The value should be between 1.{binaryMinorVersion-1} and 1.{binaryMinorVersion}.\n"+ + "Format could only be major.minor") +} + +func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion { + effective := &effectiveVersion{} + binaryVersion := version.MustParse(binaryVer) + compatVersion := binaryVersion.SubtractMinor(1) + effective.Set(binaryVersion, binaryVersion, compatVersion) + return effective +} + +// DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the +// current build information. +func DefaultBuildEffectiveVersion() MutableEffectiveVersion { + verInfo := baseversion.Get() + ver := NewEffectiveVersion(verInfo.String()) + if ver.BinaryVersion().Major() == 0 && ver.BinaryVersion().Minor() == 0 { + ver = DefaultKubeEffectiveVersion() + } + return ver +} + +// DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the +// latest K8s release. +// Should update for each minor release! +func DefaultKubeEffectiveVersion() MutableEffectiveVersion { + return NewEffectiveVersion("1.31") +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go new file mode 100644 index 0000000000000..8c61e92c1de0e --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go @@ -0,0 +1,180 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "fmt" + "strings" + "testing" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/version" +) + +func TestValidate(t *testing.T) { + tests := []struct { + name string + binaryVersion string + emulationVersion string + minCompatibilityVersion string + expectErrors bool + }{ + { + name: "patch version diff ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.32.1", + minCompatibilityVersion: "v1.31.5", + }, + { + name: "emulation version one minor lower than binary ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.31.0", + }, + { + name: "binary version 1.31, emulation version lower than 1.31", + binaryVersion: "v1.31.2", + emulationVersion: "v1.30.0", + minCompatibilityVersion: "v1.30.0", + expectErrors: true, + }, + { + name: "binary version 1.31, emulation version 1.31", + binaryVersion: "v1.31.2", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.30.0", + }, + { + name: "binary version lower than 1.31", + binaryVersion: "v1.30.2", + emulationVersion: "v1.29.0", + minCompatibilityVersion: "v1.29.0", + }, + { + name: "emulation version two minor lower than binary not ok", + binaryVersion: "v1.33.2", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.32.0", + expectErrors: true, + }, + { + name: "emulation version one minor higher than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.33.0", + minCompatibilityVersion: "v1.31.0", + expectErrors: true, + }, + { + name: "emulation version two minor higher than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.34.0", + minCompatibilityVersion: "v1.31.0", + expectErrors: true, + }, + { + name: "compatibility version same as binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.32.0", + minCompatibilityVersion: "v1.32.0", + expectErrors: true, + }, + { + name: "compatibility version two minor lower than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.32.0", + minCompatibilityVersion: "v1.30.0", + expectErrors: true, + }, + { + name: "compatibility version one minor higher than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.32.0", + minCompatibilityVersion: "v1.33.0", + expectErrors: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + binaryVersion := version.MustParseGeneric(test.binaryVersion) + effective := &effectiveVersion{} + emulationVersion := version.MustParseGeneric(test.emulationVersion) + minCompatibilityVersion := version.MustParseGeneric(test.minCompatibilityVersion) + effective.Set(binaryVersion, emulationVersion, minCompatibilityVersion) + + errs := effective.Validate() + if len(errs) > 0 && !test.expectErrors { + t.Errorf("expected no errors, errors found %+v", errs) + } + + if len(errs) == 0 && test.expectErrors { + t.Errorf("expected errors, no errors found") + } + }) + } +} + +func TestEffectiveVersionsFlag(t *testing.T) { + tests := []struct { + name string + emulationVerson string + expectedEmulationVersion *version.Version + parseError string + }{ + { + name: "major.minor ok", + emulationVerson: "1.30", + expectedEmulationVersion: version.MajorMinor(1, 30), + }, + { + name: "v prefix ok", + emulationVerson: "v1.30", + expectedEmulationVersion: version.MajorMinor(1, 30), + }, + { + name: "semantic version not ok", + emulationVerson: "1.30.1", + parseError: "version 1.30.1 is not in the format of major.minor", + }, + { + name: "invalid version", + emulationVerson: "1.foo", + parseError: "illegal version string", + }, + } + for i, test := range tests { + t.Run(test.name, func(t *testing.T) { + fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) + effective := NewEffectiveVersion("1.30") + effective.AddFlags(fs, "test") + + err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVerson)}) + if test.parseError != "" { + if !strings.Contains(err.Error(), test.parseError) { + t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) + } + return + } + if err != nil { + t.Fatalf("%d: Parse() Expected nil, Got %v", i, err) + } + if !effective.EmulationVersion().EqualTo(test.expectedEmulationVersion) { + t.Errorf("%d: EmulationVersion Expected %s, Got %s", i, test.expectedEmulationVersion.String(), effective.EmulationVersion().String()) + } + }) + } +} diff --git a/staging/src/k8s.io/cloud-provider/options/options.go b/staging/src/k8s.io/cloud-provider/options/options.go index b96948106ca07..9d55650aa19ba 100644 --- a/staging/src/k8s.io/cloud-provider/options/options.go +++ b/staging/src/k8s.io/cloud-provider/options/options.go @@ -164,7 +164,7 @@ func (o *CloudControllerManagerOptions) Flags(allControllers []string, disabledB fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") fs.StringVar(&o.Generic.ClientConnection.Kubeconfig, "kubeconfig", o.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).") fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.") - utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic")) + utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "") return fss } diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate.go b/staging/src/k8s.io/component-base/featuregate/feature_gate.go index 1e441289ea54e..c6b4d7627f53a 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate.go @@ -19,6 +19,7 @@ package featuregate import ( "context" "fmt" + "reflect" "sort" "strconv" "strings" @@ -27,8 +28,11 @@ import ( "github.com/spf13/pflag" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/naming" + "k8s.io/apimachinery/pkg/util/version" featuremetrics "k8s.io/component-base/metrics/prometheus/feature" + baseversion "k8s.io/component-base/version" "k8s.io/klog/v2" ) @@ -52,13 +56,13 @@ const ( var ( // The generic features. - defaultFeatures = map[Feature]FeatureSpec{ - allAlphaGate: {Default: false, PreRelease: Alpha}, - allBetaGate: {Default: false, PreRelease: Beta}, + defaultFeatures = map[Feature]VersionedSpecs{ + allAlphaGate: {{Default: false, PreRelease: Alpha, Version: version.MajorMinor(0, 0)}}, + allBetaGate: {{Default: false, PreRelease: Beta, Version: version.MajorMinor(0, 0)}}, } // Special handling for a few gates. - specialFeatures = map[Feature]func(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool){ + specialFeatures = map[Feature]func(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version){ allAlphaGate: setUnsetAlphaGates, allBetaGate: setUnsetBetaGates, } @@ -69,13 +73,28 @@ type FeatureSpec struct { Default bool // LockToDefault indicates that the feature is locked to its default and cannot be changed LockToDefault bool - // PreRelease indicates the maturity level of the feature + // PreRelease indicates the current maturity level of the feature PreRelease prerelease + // Version indicates the earliest version from which this FeatureSpec is valid. + // If multiple FeatureSpecs exist for a Feature, the one with the highest version that is less + // than or equal to the effective version of the component is used. + Version *version.Version } +type VersionedSpecs []FeatureSpec + +func (g VersionedSpecs) Len() int { return len(g) } +func (g VersionedSpecs) Less(i, j int) bool { + return g[i].Version.LessThan(g[j].Version) +} +func (g VersionedSpecs) Swap(i, j int) { g[i], g[j] = g[j], g[i] } + +type PromotionVersionMapping map[prerelease]string + type prerelease string const ( + PreAlpha = prerelease("PRE-ALPHA") // Values for PreRelease. Alpha = prerelease("ALPHA") Beta = prerelease("BETA") @@ -94,7 +113,10 @@ type FeatureGate interface { // DeepCopy returns a deep copy of the FeatureGate object, such that gates can be // set on the copy without mutating the original. This is useful for validating // config against potential feature gate changes before committing those changes. - DeepCopy() MutableFeatureGate + DeepCopy() MutableVersionedFeatureGate + // Validate checks if the flag gates are valid at the emulated version. + // Should always be called after Set when DeferErrorsToValidation is set to true. + Validate() []error } // MutableFeatureGate parses and stores flag gates for known features from @@ -103,7 +125,7 @@ type MutableFeatureGate interface { FeatureGate // AddFlag adds a flag for setting global feature gates to the specified FlagSet. - AddFlag(fs *pflag.FlagSet) + AddFlag(fs *pflag.FlagSet, prefix string) // Set parses and stores flag gates for known features // from a string like feature1=true,feature2=false,... Set(value string) error @@ -128,25 +150,70 @@ type MutableFeatureGate interface { OverrideDefault(name Feature, override bool) error } +// MutableVersionedFeatureGate parses and stores flag gates for known features from +// a string like feature1=true,feature2=false,... +// MutableVersionedFeatureGate sets options based on the emulated version of the featured gate. +type MutableVersionedFeatureGate interface { + MutableFeatureGate + // EmulationVersion returns the version the feature gate is set to emulate. + // If set, the feature gate would enable/disable features based on + // feature availability and pre-release at the emulated version instead of the binary version. + EmulationVersion() *version.Version + // SetEmulationVersion overrides the emulationVersion of the feature gate. + // Otherwise, the emulationVersion will be the same as the binary version. + // If set, the feature defaults and availability will be as if the binary is at the emulated version. + SetEmulationVersion(emulationVersion *version.Version) error + // DeferErrorsToValidation defers the errors of Set function to the Validate() function if true. + // This is used when the user wants to set the feature gate flag before the emulationVersion is finalized. + // Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. + DeferErrorsToValidation(val bool) + // GetAll returns a copy of the map of known feature names to versioned feature specs. + GetAllVersioned() map[Feature]VersionedSpecs + // AddVersioned adds versioned feature specs to the featureGate. + AddVersioned(features map[Feature]VersionedSpecs) error +} + +// MutableVersionedFeatureGateForTests is a feature gate interface that should only be used in tests. +type MutableVersionedFeatureGateForTests interface { + MutableVersionedFeatureGate + // Reset sets the enabled and enabledRaw to the input map. + Reset(m map[string]bool) + // EnabledRawMap returns the raw enable map from the feature gate. + EnabledRawMap() map[string]bool +} + // featureGate implements FeatureGate as well as pflag.Value for flag parsing. type featureGate struct { featureGateName string - special map[Feature]func(map[Feature]FeatureSpec, map[Feature]bool, bool) + special map[Feature]func(map[Feature]VersionedSpecs, map[Feature]bool, bool, *version.Version) - // lock guards writes to known, enabled, and reads/writes of closed + // lock guards writes to all below fields. lock sync.Mutex // known holds a map[Feature]FeatureSpec known atomic.Value // enabled holds a map[Feature]bool enabled atomic.Value + // enabledRaw holds a raw map[string]bool of the parsed flag. + // It keeps the original values of "special" features like "all alpha gates", + // while enabled keeps the values of all resolved features. + enabledRaw atomic.Value // closed is set to true when AddFlag is called, and prevents subsequent calls to Add closed bool + // deferErrorsToValidation could be set to true to defer checking flag setting error, + // because the emulationVersion may not be the final emulationVersion when the flag is set. + // Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. + deferErrorsToValidation bool + emulationVersion atomic.Pointer[version.Version] } -func setUnsetAlphaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) { +func setUnsetAlphaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) { for k, v := range known { - if v.PreRelease == Alpha { + if k == "AllAlpha" || k == "AllBeta" { + continue + } + currentVersion := getCurrentVersion(v, cVer) + if currentVersion.PreRelease == Alpha { if _, found := enabled[k]; !found { enabled[k] = val } @@ -154,9 +221,13 @@ func setUnsetAlphaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, } } -func setUnsetBetaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) { +func setUnsetBetaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) { for k, v := range known { - if v.PreRelease == Beta { + if k == "AllAlpha" || k == "AllBeta" { + continue + } + currentVersion := getCurrentVersion(v, cVer) + if currentVersion.PreRelease == Beta { if _, found := enabled[k]; !found { enabled[k] = val } @@ -171,8 +242,10 @@ var _ pflag.Value = &featureGate{} // call chains, so they'd be unhelpful as names. var internalPackages = []string{"k8s.io/component-base/featuregate/feature_gate.go"} -func NewFeatureGate() *featureGate { - known := map[Feature]FeatureSpec{} +// NewVersionedFeatureGate creates a feature gate with the emulation version set to the provided version. +// SetEmulationVersion can be called after to change emulation version to a desired value. +func NewVersionedFeatureGate(emulationVersion *version.Version) *featureGate { + known := map[Feature]VersionedSpecs{} for k, v := range defaultFeatures { known[k] = v } @@ -183,10 +256,18 @@ func NewFeatureGate() *featureGate { } f.known.Store(known) f.enabled.Store(map[Feature]bool{}) - + f.enabledRaw.Store(map[string]bool{}) + f.emulationVersion.Store(emulationVersion) + klog.V(1).Infof("new feature gate with emulationVersion=%s", f.emulationVersion.Load().String()) return f } +// NewFeatureGate creates a feature gate with the current binary version. +func NewFeatureGate() *featureGate { + binaryVersison := version.MustParse(baseversion.Get().String()) + return NewVersionedFeatureGate(binaryVersison) +} + // Set parses a string of the form "key1=value1,key2=value2,..." into a // map[string]bool of known keys or returns an error. func (f *featureGate) Set(value string) error { @@ -207,52 +288,100 @@ func (f *featureGate) Set(value string) error { } m[k] = boolValue } - return f.SetFromMap(m) + err := f.SetFromMap(m) + // ignores SetFromMap error, because the emulationVersion may not be the final emulationVersion when the flag is set. + // Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. + if f.deferErrorsToValidation { + return nil + } + return err } -// SetFromMap stores flag gates for known features from a map[string]bool or returns an error -func (f *featureGate) SetFromMap(m map[string]bool) error { - f.lock.Lock() - defer f.lock.Unlock() +// Validate checks if the flag gates are valid at the emulated version. +// Should always be called after Set when DeferErrorsToValidation is set to true. +func (f *featureGate) Validate() []error { + m, ok := f.enabledRaw.Load().(map[string]bool) + if !ok { + return []error{fmt.Errorf("cannot cast enabledRaw to map[string]bool")} + } + enabled := map[Feature]bool{} + return f.unsafeSetFromMap(enabled, m) +} +// unsafeSetFromMap stores flag gates for known features from a map[string]bool into an enabled map. +func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bool) []error { + var errs []error // Copy existing state - known := map[Feature]FeatureSpec{} - for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + known := map[Feature]VersionedSpecs{} + for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { + sort.Sort(v) known[k] = v } - enabled := map[Feature]bool{} - for k, v := range f.enabled.Load().(map[Feature]bool) { - enabled[k] = v - } for k, v := range m { - k := Feature(k) - featureSpec, ok := known[k] + key := Feature(k) + versionedSpecs, ok := known[key] if !ok { - return fmt.Errorf("unrecognized feature gate: %s", k) + // early return if encounters an unknown feature. + errs = append(errs, fmt.Errorf("unrecognized feature gate: %s", k)) + return errs } - if featureSpec.LockToDefault && featureSpec.Default != v { - return fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, featureSpec.Default) + currentVersion := f.getCurrentVersion(versionedSpecs) + if currentVersion.LockToDefault && currentVersion.Default != v { + errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, currentVersion.Default)) + continue } - enabled[k] = v // Handle "special" features like "all alpha gates" - if fn, found := f.special[k]; found { - fn(known, enabled, v) + if fn, found := f.special[key]; found { + fn(known, enabled, v, f.emulationVersion.Load()) + enabled[key] = v + continue + } + if currentVersion.PreRelease == PreAlpha { + errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is PreAlpha at emulated version %s", k, v, f.EmulationVersion().String())) + continue } + enabled[key] = v - if featureSpec.PreRelease == Deprecated { + if currentVersion.PreRelease == Deprecated { klog.Warningf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v) - } else if featureSpec.PreRelease == GA { + } else if currentVersion.PreRelease == GA { klog.Warningf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v) } } + return errs +} - // Persist changes - f.known.Store(known) - f.enabled.Store(enabled) +// SetFromMap stores flag gates for known features from a map[string]bool or returns an error +func (f *featureGate) SetFromMap(m map[string]bool) error { + f.lock.Lock() + defer f.lock.Unlock() - klog.V(1).Infof("feature gates: %v", f.enabled) - return nil + // Copy existing state + enabled := map[Feature]bool{} + for k, v := range f.enabled.Load().(map[Feature]bool) { + enabled[k] = v + } + enabledRaw := map[string]bool{} + for k, v := range f.enabledRaw.Load().(map[string]bool) { + enabledRaw[k] = v + } + + // Update enabledRaw first. + // SetFromMap might be called when emulationVersion is not finalized yet, and we do not know the final state of enabled. + // But the flags still need to be saved. + for k, v := range m { + enabledRaw[k] = v + } + f.enabledRaw.Store(enabledRaw) + + errs := f.unsafeSetFromMap(enabled, enabledRaw) + if len(errs) == 0 { + // Persist changes + f.enabled.Store(enabled) + klog.V(1).Infof("feature gates: %v", f.enabled) + } + return utilerrors.NewAggregate(errs) } // String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...". @@ -271,6 +400,17 @@ func (f *featureGate) Type() string { // Add adds features to the featureGate. func (f *featureGate) Add(features map[Feature]FeatureSpec) error { + vs := map[Feature]VersionedSpecs{} + for name, spec := range features { + // if no version is provided for the FeatureSpec, it is defaulted to version 0.0 so that it can be enabled/disabled regardless of emulation version. + spec.Version = version.MajorMinor(0, 0) + vs[name] = VersionedSpecs{spec} + } + return f.AddVersioned(vs) +} + +// AddVersioned adds versioned feature specs to the featureGate. +func (f *featureGate) AddVersioned(features map[Feature]VersionedSpecs) error { f.lock.Lock() defer f.lock.Unlock() @@ -279,20 +419,21 @@ func (f *featureGate) Add(features map[Feature]FeatureSpec) error { } // Copy existing state - known := map[Feature]FeatureSpec{} - for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + known := map[Feature]VersionedSpecs{} + for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { known[k] = v } - for name, spec := range features { + for name, specs := range features { + sort.Sort(specs) if existingSpec, found := known[name]; found { - if existingSpec == spec { + sort.Sort(existingSpec) + if reflect.DeepEqual(existingSpec, specs) { continue } return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec) } - - known[name] = spec + known[name] = specs } // Persist updated state @@ -309,17 +450,22 @@ func (f *featureGate) OverrideDefault(name Feature, override bool) error { return fmt.Errorf("cannot override default for feature %q: gates already added to a flag set", name) } - known := map[Feature]FeatureSpec{} - for name, spec := range f.known.Load().(map[Feature]FeatureSpec) { - known[name] = spec + known := map[Feature]VersionedSpecs{} + for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { + sort.Sort(v) + known[k] = v } - spec, ok := known[name] - switch { - case !ok: + specs, ok := known[name] + if !ok { return fmt.Errorf("cannot override default: feature %q is not registered", name) + } + spec := f.getCurrentVersion(specs) + switch { case spec.LockToDefault: return fmt.Errorf("cannot override default: feature %q default is locked to %t", name, spec.Default) + case spec.PreRelease == PreAlpha: + return fmt.Errorf("cannot override default: feature %q is not available before emulation version %s", name, f.EmulationVersion().String()) case spec.PreRelease == Deprecated: klog.Warningf("Overriding default of deprecated feature gate %s=%t. It will be removed in a future release.", name, override) case spec.PreRelease == GA: @@ -327,35 +473,112 @@ func (f *featureGate) OverrideDefault(name Feature, override bool) error { } spec.Default = override - known[name] = spec + known[name] = specs f.known.Store(known) return nil } -// GetAll returns a copy of the map of known feature names to feature specs. +// GetAll returns a copy of the map of known feature names to feature specs for the current emulationVersion. func (f *featureGate) GetAll() map[Feature]FeatureSpec { retval := map[Feature]FeatureSpec{} - for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + for k, v := range f.GetAllVersioned() { + spec := f.getCurrentVersion(v) + if spec.PreRelease == PreAlpha { + // The feature is not available at the emulation version. + continue + } + retval[k] = *f.getCurrentVersion(v) + } + return retval +} + +// GetAllVersioned returns a copy of the map of known feature names to versioned feature specs. +func (f *featureGate) GetAllVersioned() map[Feature]VersionedSpecs { + retval := map[Feature]VersionedSpecs{} + for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { retval[k] = v } return retval } +// DeferErrorsToValidation could be used to defer checking flag setting error, +// because the emulationVersion may not be the final emulationVersion when the flag is set. +// Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. +func (f *featureGate) DeferErrorsToValidation(val bool) { + f.lock.Lock() + defer f.lock.Unlock() + f.deferErrorsToValidation = val +} + +func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) error { + f.lock.Lock() + defer f.lock.Unlock() + klog.V(1).Infof("set feature gate emulationVersion to %s", emulationVersion.String()) + f.emulationVersion.Store(emulationVersion) + + // Copy existing state + enabledRaw := map[string]bool{} + for k, v := range f.enabledRaw.Load().(map[string]bool) { + enabledRaw[k] = v + } + // enabled map should be reset whenever emulationVersion is changed. + enabled := map[Feature]bool{} + errs := f.unsafeSetFromMap(enabled, enabledRaw) + if len(errs) == 0 { + // Persist changes + f.enabled.Store(enabled) + } + return utilerrors.NewAggregate(errs) +} + +func (f *featureGate) EmulationVersion() *version.Version { + return f.emulationVersion.Load() +} + +// FeatureSpec returns the FeatureSpec at the EmulationVersion if the key exists, an error otherwise. +func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) { + if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok { + currentVersion := f.getCurrentVersion(v) + return *currentVersion, nil + } + return FeatureSpec{}, fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName) +} + // Enabled returns true if the key is enabled. If the key is not known, this call will panic. func (f *featureGate) Enabled(key Feature) bool { + // fallback to default behavior, since we don't have emulation version set if v, ok := f.enabled.Load().(map[Feature]bool)[key]; ok { return v } - if v, ok := f.known.Load().(map[Feature]FeatureSpec)[key]; ok { - return v.Default + if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok { + return f.getCurrentVersion(v).Default } panic(fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName)) } +func (f *featureGate) getCurrentVersion(v VersionedSpecs) *FeatureSpec { + return getCurrentVersion(v, f.EmulationVersion()) +} + +func getCurrentVersion(v VersionedSpecs, emulationVersion *version.Version) *FeatureSpec { + i := len(v) - 1 + for ; i >= 0; i-- { + if v[i].Version.GreaterThan(emulationVersion) { + continue + } + return &v[i] + } + return &FeatureSpec{ + Default: false, + PreRelease: PreAlpha, + Version: version.MajorMinor(0, 0), + } +} + // AddFlag adds a flag for setting global feature gates to the specified FlagSet. -func (f *featureGate) AddFlag(fs *pflag.FlagSet) { +func (f *featureGate) AddFlag(fs *pflag.FlagSet, prefix string) { f.lock.Lock() // TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead? // Not all components expose a feature gates flag using this AddFlag method, and @@ -365,7 +588,10 @@ func (f *featureGate) AddFlag(fs *pflag.FlagSet) { f.lock.Unlock() known := f.KnownFeatures() - fs.Var(f, flagName, ""+ + if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") { + prefix += "-" + } + fs.Var(f, prefix+flagName, ""+ "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ "Options are:\n"+strings.Join(known, "\n")) } @@ -377,14 +603,19 @@ func (f *featureGate) AddMetrics() { } // KnownFeatures returns a slice of strings describing the FeatureGate's known features. -// Deprecated and GA features are hidden from the list. +// preAlpha, Deprecated and GA features are hidden from the list. func (f *featureGate) KnownFeatures() []string { var known []string - for k, v := range f.known.Load().(map[Feature]FeatureSpec) { - if v.PreRelease == GA || v.PreRelease == Deprecated { + for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { + if k == "AllAlpha" || k == "AllBeta" { + known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v[0].PreRelease, v[0].Default)) + continue + } + currentV := f.getCurrentVersion(v) + if currentV.PreRelease == GA || currentV.PreRelease == Deprecated || currentV.PreRelease == PreAlpha { continue } - known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v.PreRelease, v.Default)) + known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, currentV.PreRelease, currentV.Default)) } sort.Strings(known) return known @@ -393,16 +624,20 @@ func (f *featureGate) KnownFeatures() []string { // DeepCopy returns a deep copy of the FeatureGate object, such that gates can be // set on the copy without mutating the original. This is useful for validating // config against potential feature gate changes before committing those changes. -func (f *featureGate) DeepCopy() MutableFeatureGate { +func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { // Copy existing state. - known := map[Feature]FeatureSpec{} - for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + known := map[Feature]VersionedSpecs{} + for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { known[k] = v } enabled := map[Feature]bool{} for k, v := range f.enabled.Load().(map[Feature]bool) { enabled[k] = v } + enabledRaw := map[string]bool{} + for k, v := range f.enabledRaw.Load().(map[string]bool) { + enabledRaw[k] = v + } // Construct a new featureGate around the copied state. // Note that specialFeatures is treated as immutable by convention, @@ -411,9 +646,22 @@ func (f *featureGate) DeepCopy() MutableFeatureGate { special: specialFeatures, closed: f.closed, } - + fg.emulationVersion.Store(f.EmulationVersion()) fg.known.Store(known) fg.enabled.Store(enabled) - + fg.enabledRaw.Store(enabledRaw) return fg } + +// Reset sets the enabled and enabledRaw to the input map. +func (f *featureGate) Reset(m map[string]bool) { + enabled := map[Feature]bool{} + enabledRaw := map[string]bool{} + f.enabled.Store(enabled) + f.enabledRaw.Store(enabledRaw) + _ = f.SetFromMap(m) +} + +func (f *featureGate) EnabledRawMap() map[string]bool { + return f.enabledRaw.Load().(map[string]bool) +} diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go index 063a5a363685e..b46d5483fb51b 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go @@ -18,12 +18,17 @@ package featuregate import ( "fmt" + "reflect" + "sort" "strings" "testing" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/component-base/metrics/legacyregistry" featuremetrics "k8s.io/component-base/metrics/prometheus/feature" "k8s.io/component-base/metrics/testutil" @@ -33,6 +38,8 @@ func TestFeatureGateFlag(t *testing.T) { // gates for testing const testAlphaGate Feature = "TestAlpha" const testBetaGate Feature = "TestBeta" + const testDeprecatedGate Feature = "TestDeprecated" + const testLockedFalseGate Feature = "TestLockedFalse" tests := []struct { arg string @@ -42,93 +49,126 @@ func TestFeatureGateFlag(t *testing.T) { { arg: "", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testDeprecatedGate: false, + testLockedFalseGate: false, + }, + }, + { + arg: "TestDeprecated=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testDeprecatedGate: true, + testLockedFalseGate: false, + }, + }, + { + arg: "TestLockedFalse=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, + parseError: "cannot set feature gate TestLockedFalse to true, feature is locked to false", }, { arg: "fooBarBaz=true", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, parseError: "unrecognized feature gate: fooBarBaz", }, { arg: "AllAlpha=false", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "AllAlpha=true", expect: map[Feature]bool{ - allAlphaGate: true, - allBetaGate: false, - testAlphaGate: true, - testBetaGate: false, + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: true, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "AllAlpha=banana", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, parseError: "invalid value of AllAlpha", }, { arg: "AllAlpha=false,TestAlpha=true", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: true, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: true, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "TestAlpha=true,AllAlpha=false", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: true, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: true, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "AllAlpha=true,TestAlpha=false", expect: map[Feature]bool{ - allAlphaGate: true, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "TestAlpha=false,AllAlpha=true", expect: map[Feature]bool{ - allAlphaGate: true, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "TestBeta=true,AllAlpha=false", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: true, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, }, }, @@ -144,65 +184,72 @@ func TestFeatureGateFlag(t *testing.T) { { arg: "AllBeta=true", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: true, - testAlphaGate: false, - testBetaGate: true, + allAlphaGate: false, + allBetaGate: true, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, }, }, { arg: "AllBeta=banana", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, parseError: "invalid value of AllBeta", }, { arg: "AllBeta=false,TestBeta=true", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: true, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, }, }, { arg: "TestBeta=true,AllBeta=false", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: true, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, }, }, { arg: "AllBeta=true,TestBeta=false", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: true, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: true, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "TestBeta=false,AllBeta=true", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: true, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: true, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "TestAlpha=true,AllBeta=false", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: true, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: true, + testBetaGate: false, + testLockedFalseGate: false, }, }, } @@ -210,13 +257,15 @@ func TestFeatureGateFlag(t *testing.T) { t.Run(test.arg, func(t *testing.T) { fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) f := NewFeatureGate() - f.Add(map[Feature]FeatureSpec{ - testAlphaGate: {Default: false, PreRelease: Alpha}, - testBetaGate: {Default: false, PreRelease: Beta}, + err := f.Add(map[Feature]FeatureSpec{ + testAlphaGate: {Default: false, PreRelease: Alpha}, + testBetaGate: {Default: false, PreRelease: Beta}, + testDeprecatedGate: {Default: false, PreRelease: Deprecated}, + testLockedFalseGate: {Default: false, PreRelease: GA, LockToDefault: true}, }) - f.AddFlag(fs) - - err := fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) + require.NoError(t, err) + f.AddFlag(fs, "") + err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) if test.parseError != "" { if !strings.Contains(err.Error(), test.parseError) { t.Errorf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) @@ -239,12 +288,16 @@ func TestFeatureGateOverride(t *testing.T) { // Don't parse the flag, assert defaults are used. var f *featureGate = NewFeatureGate() - f.Add(map[Feature]FeatureSpec{ + err := f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: false, PreRelease: Beta}, }) + require.NoError(t, err) f.Set("TestAlpha=true,TestBeta=true") + if errs := f.Validate(); len(errs) > 0 { + t.Fatalf("Validate() Expected no error, Got %v", errs) + } if f.Enabled(testAlphaGate) != true { t.Errorf("Expected true") } @@ -253,6 +306,9 @@ func TestFeatureGateOverride(t *testing.T) { } f.Set("TestAlpha=false") + if errs := f.Validate(); len(errs) > 0 { + t.Fatalf("Validate() Expected no error, Got %v", errs) + } if f.Enabled(testAlphaGate) != false { t.Errorf("Expected false") } @@ -268,10 +324,11 @@ func TestFeatureGateFlagDefaults(t *testing.T) { // Don't parse the flag, assert defaults are used. var f *featureGate = NewFeatureGate() - f.Add(map[Feature]FeatureSpec{ + err := f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: true, PreRelease: Beta}, }) + require.NoError(t, err) if f.Enabled(testAlphaGate) != false { t.Errorf("Expected false") @@ -292,13 +349,13 @@ func TestFeatureGateKnownFeatures(t *testing.T) { // Don't parse the flag, assert defaults are used. var f *featureGate = NewFeatureGate() - f.Add(map[Feature]FeatureSpec{ + err := f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: true, PreRelease: Beta}, testGAGate: {Default: true, PreRelease: GA}, testDeprecatedGate: {Default: false, PreRelease: Deprecated}, }) - + require.NoError(t, err) known := strings.Join(f.KnownFeatures(), " ") assert.Contains(t, known, testAlphaGate) @@ -399,13 +456,14 @@ func TestFeatureGateSetFromMap(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("SetFromMap %s", test.name), func(t *testing.T) { f := NewFeatureGate() - f.Add(map[Feature]FeatureSpec{ + err := f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: false, PreRelease: Beta}, testLockedTrueGate: {Default: true, PreRelease: GA, LockToDefault: true}, testLockedFalseGate: {Default: false, PreRelease: GA, LockToDefault: true}, }) - err := f.SetFromMap(test.setmap) + require.NoError(t, err) + err = f.SetFromMap(test.setmap) if test.setmapError != "" { if err == nil { t.Errorf("expected error, got none") @@ -450,8 +508,8 @@ func TestFeatureGateMetrics(t *testing.T) { testBetaGate: {Default: true, PreRelease: Beta}, testBetaDisabled: {Default: true, PreRelease: Alpha}, } - f.Add(fMap) - f.SetFromMap(map[string]bool{"TestAlphaEnabled": true, "TestBetaDisabled": false}) + require.NoError(t, f.Add(fMap)) + require.NoError(t, f.SetFromMap(map[string]bool{"TestAlphaEnabled": true, "TestBetaDisabled": false})) f.AddMetrics() if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedOutput), testedMetrics...); err != nil { t.Fatal(err) @@ -499,8 +557,8 @@ func TestFeatureGateString(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("SetFromMap %s", test.expect), func(t *testing.T) { f := NewFeatureGate() - f.Add(featuremap) - f.SetFromMap(test.setmap) + require.NoError(t, f.Add(featuremap)) + require.NoError(t, f.SetFromMap(test.setmap)) result := f.String() if result != test.expect { t.Errorf("%d: SetFromMap(%#v) Expected %s, Got %s", i, test.setmap, test.expect, result) @@ -518,12 +576,8 @@ func TestFeatureGateOverrideDefault(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := f.OverrideDefault("TestFeature1", false); err != nil { - t.Fatal(err) - } - if err := f.OverrideDefault("TestFeature2", true); err != nil { - t.Fatal(err) - } + require.NoError(t, f.OverrideDefault("TestFeature1", false)) + require.NoError(t, f.OverrideDefault("TestFeature2", true)) if f.Enabled("TestFeature1") { t.Error("expected TestFeature1 to have effective default of false") } @@ -534,12 +588,8 @@ func TestFeatureGateOverrideDefault(t *testing.T) { t.Run("overrides are preserved across deep copies", func(t *testing.T) { f := NewFeatureGate() - if err := f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: false}}); err != nil { - t.Fatal(err) - } - if err := f.OverrideDefault("TestFeature", true); err != nil { - t.Fatal(err) - } + require.NoError(t, f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: false}})) + require.NoError(t, f.OverrideDefault("TestFeature", true)) fcopy := f.DeepCopy() if !fcopy.Enabled("TestFeature") { t.Error("default override was not preserved by deep copy") @@ -554,9 +604,7 @@ func TestFeatureGateOverrideDefault(t *testing.T) { }}); err != nil { t.Fatal(err) } - if err := f.OverrideDefault("TestFeature", true); err != nil { - t.Fatal(err) - } + require.NoError(t, f.OverrideDefault("TestFeature", true)) var found bool for _, s := range f.KnownFeatures() { if !strings.Contains(s, "TestFeature") { @@ -592,15 +640,9 @@ func TestFeatureGateOverrideDefault(t *testing.T) { t.Run("does not supersede explicitly-set value", func(t *testing.T) { f := NewFeatureGate() - if err := f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: true}}); err != nil { - t.Fatal(err) - } - if err := f.OverrideDefault("TestFeature", false); err != nil { - t.Fatal(err) - } - if err := f.SetFromMap(map[string]bool{"TestFeature": true}); err != nil { - t.Fatal(err) - } + require.NoError(t, f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: true}})) + require.NoError(t, f.OverrideDefault("TestFeature", false)) + require.NoError(t, f.SetFromMap(map[string]bool{"TestFeature": true})) if !f.Enabled("TestFeature") { t.Error("expected feature to be effectively enabled despite default override") } @@ -616,9 +658,7 @@ func TestFeatureGateOverrideDefault(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := f.OverrideDefault("TestFeature", false); err != nil { - t.Fatal(err) - } + require.NoError(t, f.OverrideDefault("TestFeature", false)) if err := f.Add(map[Feature]FeatureSpec{ "TestFeature": { Default: true, @@ -639,10 +679,801 @@ func TestFeatureGateOverrideDefault(t *testing.T) { t.Run("returns error if already added to flag set", func(t *testing.T) { f := NewFeatureGate() fs := pflag.NewFlagSet("test", pflag.ContinueOnError) - f.AddFlag(fs) + f.AddFlag(fs, "") if err := f.OverrideDefault("TestFeature", true); err == nil { t.Error("expected a non-nil error to be returned") } }) } + +func TestVersionedFeatureGateFlag(t *testing.T) { + // gates for testing + const testGAGate Feature = "TestGA" + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + const testLockedFalseGate Feature = "TestLockedFalse" + const testAlphaGateNoVersion Feature = "TestAlphaNoVersion" + const testBetaGateNoVersion Feature = "TestBetaNoVersion" + + tests := []struct { + arg string + expect map[Feature]bool + parseError string + }{ + { + arg: "", + expect: map[Feature]bool{ + testGAGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "TestLockedFalse=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: true, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "fooBarBaz=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + parseError: "unrecognized feature gate: fooBarBaz", + }, + { + arg: "AllAlpha=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "AllAlpha=true", + expect: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: false, + testGAGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: true, + testBetaGateNoVersion: false, + }, + }, + { + arg: "AllAlpha=banana", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + parseError: "invalid value of AllAlpha", + }, + { + arg: "AllAlpha=false,TestAlpha=true,TestAlphaNoVersion=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: true, + testBetaGateNoVersion: false, + }, + parseError: "cannot set feature gate TestAlpha to true, feature is PreAlpha at emulated version 1.28", + }, + { + arg: "AllAlpha=false,TestAlphaNoVersion=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: true, + testBetaGateNoVersion: false, + }, + }, + { + arg: "TestAlpha=true,TestAlphaNoVersion=true,AllAlpha=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: true, + testBetaGateNoVersion: false, + }, + parseError: "cannot set feature gate TestAlpha to true, feature is PreAlpha at emulated version 1.28", + }, + { + arg: "AllAlpha=true,TestAlpha=false,TestAlphaNoVersion=false", + expect: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + parseError: "cannot set feature gate TestAlpha to false, feature is PreAlpha at emulated version 1.28", + }, + { + arg: "AllAlpha=true,TestAlphaNoVersion=false", + expect: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "TestAlpha=false,TestAlphaNoVersion=false,AllAlpha=true", + expect: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + parseError: "cannot set feature gate TestAlpha to false, feature is PreAlpha at emulated version 1.28", + }, + { + arg: "TestBeta=true,TestBetaNoVersion=true,TestGA=true,AllAlpha=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: true, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: true, + }, + }, + + { + arg: "AllBeta=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "AllBeta=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: true, + testGAGate: true, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: true, + }, + }, + { + arg: "AllBeta=banana", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + parseError: "invalid value of AllBeta", + }, + { + arg: "AllBeta=false,TestBeta=true,TestBetaNoVersion=true,TestGA=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: true, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: true, + }, + }, + { + arg: "TestBeta=true,TestBetaNoVersion=true,AllBeta=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: true, + }, + }, + { + arg: "AllBeta=true,TestBetaNoVersion=false,TestBeta=false,TestGA=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: true, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "TestBeta=false,TestBetaNoVersion=false,AllBeta=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: true, + testGAGate: true, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "TestAlpha=true,AllBeta=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: true, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + parseError: "cannot set feature gate TestAlpha to true, feature is PreAlpha at emulated version 1.28", + }, + } + for i, test := range tests { + t.Run(test.arg, func(t *testing.T) { + fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) + f := NewVersionedFeatureGate(version.MustParse("1.29")) + f.DeferErrorsToValidation(true) + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testGAGate: { + {Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: Alpha}, + }, + testAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, + testBetaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + }, + testLockedFalseGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: GA, LockToDefault: true}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: GA}, + }, + }) + require.NoError(t, err) + err = f.Add(map[Feature]FeatureSpec{ + testAlphaGateNoVersion: {Default: false, PreRelease: Alpha}, + testBetaGateNoVersion: {Default: false, PreRelease: Beta}, + }) + require.NoError(t, err) + f.AddFlag(fs, "") + + var errs []error + err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) + if err != nil { + errs = append(errs, err) + } else { + errs = append(errs, f.SetEmulationVersion(version.MustParse("1.28"))) + } + err = utilerrors.NewAggregate(errs) + if test.parseError != "" { + if !strings.Contains(err.Error(), test.parseError) { + t.Errorf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) + } + return + } else if err != nil { + t.Errorf("%d: Parse() Expected nil, Got %v", i, err) + } + for k, v := range test.expect { + if actual := f.enabled.Load().(map[Feature]bool)[k]; actual != v { + t.Errorf("%d: expected %s=%v, Got %v", i, k, v, actual) + } + } + }) + } +} + +func TestVersionedFeatureGateOverride(t *testing.T) { + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + + // Don't parse the flag, assert defaults are used. + f := NewVersionedFeatureGate(version.MustParse("1.29")) + + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, + testBetaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + }, + }) + require.NoError(t, err) + if f.Enabled(testAlphaGate) != false { + t.Errorf("Expected false") + } + if f.Enabled(testBetaGate) != false { + t.Errorf("Expected false") + } + if errs := f.Validate(); len(errs) > 0 { + t.Errorf("Expected no errors when emulation version is equal to binary version.") + } + + require.NoError(t, f.Set("TestAlpha=true,TestBeta=true")) + if f.Enabled(testAlphaGate) != true { + t.Errorf("Expected false") + } + if f.Enabled(testBetaGate) != true { + t.Errorf("Expected true") + } + + require.NoError(t, f.Set("TestAlpha=false")) + if f.Enabled(testAlphaGate) != false { + t.Errorf("Expected false") + } + if f.Enabled(testBetaGate) != true { + t.Errorf("Expected true") + } + if errs := f.Validate(); len(errs) > 0 { + t.Errorf("Expected no errors when emulation version is equal to binary version.") + } + + if err := f.SetEmulationVersion(version.MustParse("1.28")); err == nil { + t.Errorf("Expected errors when emulation version is 1.28.") + } +} + +func TestVersionedFeatureGateFlagDefaults(t *testing.T) { + // gates for testing + const testGAGate Feature = "TestGA" + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + + // Don't parse the flag, assert defaults are used. + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testGAGate: { + {Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + {Version: version.MustParse("1.27"), Default: true, PreRelease: Beta}, + {Version: version.MustParse("1.25"), Default: true, PreRelease: Alpha}, + }, + testAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, + testBetaGate: { + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha}, + }, + }) + require.NoError(t, err) + + if f.Enabled(testAlphaGate) != false { + t.Errorf("Expected false") + } + if fs, _ := f.FeatureSpec(testAlphaGate); fs.PreRelease != PreAlpha || fs.Version.String() != "0.0" { + t.Errorf("Expected (PreAlpha, 0.0)") + } + if f.Enabled(testBetaGate) != false { + t.Errorf("Expected false") + } + if fs, _ := f.FeatureSpec(testBetaGate); fs.PreRelease != Beta || fs.Version.String() != "1.28" { + t.Errorf("Expected (Beta, 1.28)") + } + if f.Enabled(testGAGate) != true { + t.Errorf("Expected true") + } + if fs, _ := f.FeatureSpec(testGAGate); fs.PreRelease != Beta || fs.Version.String() != "1.27" { + t.Errorf("Expected (Beta, 1.27)") + } + if _, err := f.FeatureSpec("NonExist"); err == nil { + t.Errorf("Expected Error") + } + allFeatures := f.GetAll() + expectedAllFeatures := []Feature{testGAGate, testBetaGate, allAlphaGate, allBetaGate} + if len(allFeatures) != 4 { + t.Errorf("Expected 4 features from GetAll(), got %d", len(allFeatures)) + } + for _, feature := range expectedAllFeatures { + if _, ok := allFeatures[feature]; !ok { + t.Errorf("Expected feature %s to be in GetAll()", feature) + } + } +} + +func TestVersionedFeatureGateKnownFeatures(t *testing.T) { + // gates for testing + const ( + testPreAlphaGate Feature = "TestPreAlpha" + testAlphaGate Feature = "TestAlpha" + testBetaGate Feature = "TestBeta" + testGAGate Feature = "TestGA" + testDeprecatedGate Feature = "TestDeprecated" + testGAGateNoVersion Feature = "TestGANoVersion" + testAlphaGateNoVersion Feature = "TestAlphaNoVersion" + testBetaGateNoVersion Feature = "TestBetaNoVersion" + testDeprecatedGateNoVersion Feature = "TestDeprecatedNoVersion" + ) + + // Don't parse the flag, assert defaults are used. + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testGAGate: { + {Version: version.MustParse("1.27"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: true, PreRelease: GA}, + }, + testPreAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, + testAlphaGate: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + }, + testBetaGate: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + }, + testDeprecatedGate: { + {Version: version.MustParse("1.28"), Default: true, PreRelease: Deprecated}, + {Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha}, + }, + }) + require.NoError(t, err) + err = f.Add(map[Feature]FeatureSpec{ + testAlphaGateNoVersion: {Default: false, PreRelease: Alpha}, + testBetaGateNoVersion: {Default: false, PreRelease: Beta}, + testGAGateNoVersion: {Default: false, PreRelease: GA}, + testDeprecatedGateNoVersion: {Default: false, PreRelease: Deprecated}, + }) + require.NoError(t, err) + + known := strings.Join(f.KnownFeatures(), " ") + + assert.NotContains(t, known, testPreAlphaGate) + assert.Contains(t, known, testAlphaGate) + assert.Contains(t, known, testBetaGate) + assert.NotContains(t, known, testGAGate) + assert.NotContains(t, known, testDeprecatedGate) + assert.Contains(t, known, testAlphaGateNoVersion) + assert.Contains(t, known, testBetaGateNoVersion) + assert.NotContains(t, known, testGAGateNoVersion) + assert.NotContains(t, known, testDeprecatedGateNoVersion) +} + +func TestVersionedFeatureGateMetrics(t *testing.T) { + // gates for testing + featuremetrics.ResetFeatureInfoMetric() + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + const testAlphaEnabled Feature = "TestAlphaEnabled" + const testBetaDisabled Feature = "TestBetaDisabled" + testedMetrics := []string{"kubernetes_feature_enabled"} + expectedOutput := ` + # HELP kubernetes_feature_enabled [BETA] This metric records the data about the stage and enablement of a k8s feature. + # TYPE kubernetes_feature_enabled gauge + kubernetes_feature_enabled{name="TestAlpha",stage="ALPHA"} 0 + kubernetes_feature_enabled{name="TestBeta",stage="BETA"} 1 + kubernetes_feature_enabled{name="TestAlphaEnabled",stage="ALPHA"} 1 + kubernetes_feature_enabled{name="AllAlpha",stage="ALPHA"} 0 + kubernetes_feature_enabled{name="AllBeta",stage="BETA"} 0 + kubernetes_feature_enabled{name="TestBetaDisabled",stage="BETA"} 0 +` + + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testAlphaGate: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, + }, + testAlphaEnabled: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, + }, + testBetaGate: { + {Version: version.MustParse("1.28"), Default: true, PreRelease: Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: Alpha}, + }, + testBetaDisabled: { + {Version: version.MustParse("1.28"), Default: true, PreRelease: Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: Alpha}, + }, + }) + require.NoError(t, err) + + require.NoError(t, f.SetFromMap(map[string]bool{"TestAlphaEnabled": true, "TestBetaDisabled": false})) + f.AddMetrics() + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedOutput), testedMetrics...); err != nil { + t.Fatal(err) + } +} + +func TestVersionedFeatureGateOverrideDefault(t *testing.T) { + t.Run("overrides take effect", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature1": { + {Version: version.MustParse("1.28"), Default: true}, + }, + "TestFeature2": { + {Version: version.MustParse("1.26"), Default: false}, + {Version: version.MustParse("1.29"), Default: true}, + }, + }); err != nil { + t.Fatal(err) + } + require.NoError(t, f.OverrideDefault("TestFeature1", false)) + require.NoError(t, f.OverrideDefault("TestFeature2", true)) + if f.Enabled("TestFeature1") { + t.Error("expected TestFeature1 to have effective default of false") + } + if !f.Enabled("TestFeature2") { + t.Error("expected TestFeature2 to have effective default of true") + } + }) + + t.Run("overrides are preserved across deep copies", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature": { + {Version: version.MustParse("1.28"), Default: false}, + {Version: version.MustParse("1.29"), Default: true}, + }, + }); err != nil { + t.Fatal(err) + } + require.NoError(t, f.OverrideDefault("TestFeature", true)) + fcopy := f.DeepCopy() + if !fcopy.Enabled("TestFeature") { + t.Error("default override was not preserved by deep copy") + } + }) + + t.Run("reflected in known features", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature": { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, + }, + }); err != nil { + t.Fatal(err) + } + require.NoError(t, f.OverrideDefault("TestFeature", true)) + var found bool + for _, s := range f.KnownFeatures() { + if !strings.Contains(s, "TestFeature") { + continue + } + found = true + if !strings.Contains(s, "default=true") { + t.Errorf("expected override of default to be reflected in known feature description %q", s) + } + } + if !found { + t.Error("found no entry for TestFeature in known features") + } + }) + + t.Run("may not change default for specs with locked defaults", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "LockedFeature": { + {Version: version.MustParse("1.28"), Default: true, LockToDefault: true}, + }, + }); err != nil { + t.Fatal(err) + } + if f.OverrideDefault("LockedFeature", false) == nil { + t.Error("expected error when attempting to override the default for a feature with a locked default") + } + if f.OverrideDefault("LockedFeature", true) == nil { + t.Error("expected error when attempting to override the default for a feature with a locked default") + } + }) + + t.Run("can change default for specs without locked defaults for emulation version", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "LockedFeature": { + {Version: version.MustParse("1.28"), Default: true}, + {Version: version.MustParse("1.29"), Default: true, LockToDefault: true}, + }, + }); err != nil { + t.Fatal(err) + } + require.NoError(t, f.OverrideDefault("LockedFeature", false)) + if f.Enabled("LockedFeature") { + t.Error("expected LockedFeature to have effective default of false") + } + }) + + t.Run("does not supersede explicitly-set value", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature": { + {Version: version.MustParse("1.28"), Default: true}, + }, + }); err != nil { + t.Fatal(err) + } + require.NoError(t, f.OverrideDefault("TestFeature", false)) + require.NoError(t, f.SetFromMap(map[string]bool{"TestFeature": true})) + if !f.Enabled("TestFeature") { + t.Error("expected feature to be effectively enabled despite default override") + } + }) + + t.Run("prevents re-registration of feature spec after overriding default", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature": { + {Version: version.MustParse("1.28"), Default: true, PreRelease: Alpha}, + }, + }); err != nil { + t.Fatal(err) + } + require.NoError(t, f.OverrideDefault("TestFeature", false)) + if err := f.Add(map[Feature]FeatureSpec{ + "TestFeature": { + Default: true, + PreRelease: Alpha, + }, + }); err == nil { + t.Error("expected re-registration to return a non-nil error after overriding its default") + } + }) + + t.Run("does not allow override for a feature added after emulation version", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature": { + {Version: version.MustParse("1.29"), Default: false}, + }, + }); err != nil { + t.Fatal(err) + } + if err := f.OverrideDefault("TestFeature", true); err == nil { + t.Error("expected an error to be returned in attempt to override default for a feature added after emulation version") + } + }) + + t.Run("does not allow override for an unknown feature", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.OverrideDefault("TestFeature", true); err == nil { + t.Error("expected an error to be returned in attempt to override default for unregistered feature") + } + }) + + t.Run("returns error if already added to flag set", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + f.AddFlag(fs, "") + + if err := f.OverrideDefault("TestFeature", true); err == nil { + t.Error("expected a non-nil error to be returned") + } + }) +} + +func TestGetCurrentVersion(t *testing.T) { + specs := VersionedSpecs{{Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, + } + sort.Sort(specs) + tests := []struct { + cVersion string + expect FeatureSpec + }{ + { + cVersion: "1.30", + expect: FeatureSpec{Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + }, + { + cVersion: "1.29", + expect: FeatureSpec{Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + }, + { + cVersion: "1.28", + expect: FeatureSpec{Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + }, + { + cVersion: "1.27", + expect: FeatureSpec{Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, + }, + { + cVersion: "1.25", + expect: FeatureSpec{Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, + }, + { + cVersion: "1.24", + expect: FeatureSpec{Version: version.MajorMinor(0, 0), Default: false, PreRelease: PreAlpha}, + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("getCurrentVersion for emulationVersion %s", test.cVersion), func(t *testing.T) { + result := getCurrentVersion(specs, version.MustParse(test.cVersion)) + if !reflect.DeepEqual(*result, test.expect) { + t.Errorf("%d: getCurrentVersion(, %s) Expected %v, Got %v", i, test.cVersion, test.expect, result) + } + }) + } +} diff --git a/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go b/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go index 72e62c248c12f..47c0f92773863 100644 --- a/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go +++ b/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go @@ -45,7 +45,7 @@ func init() { func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) { tb.Helper() detectParallelOverrideCleanup := detectParallelOverride(tb, f) - originalValue := gate.Enabled(f) + originalEnabled := gate.(featuregate.MutableVersionedFeatureGateForTests).EnabledRawMap() // Specially handle AllAlpha and AllBeta if f == "AllAlpha" || f == "AllBeta" { @@ -67,9 +67,7 @@ func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f fea tb.Cleanup(func() { tb.Helper() detectParallelOverrideCleanup() - if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil { - tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) - } + gate.(featuregate.MutableVersionedFeatureGateForTests).Reset(originalEnabled) }) } diff --git a/staging/src/k8s.io/component-base/logs/example/cmd/logger.go b/staging/src/k8s.io/component-base/logs/example/cmd/logger.go index b0d4f9baaca96..af51c793e3082 100644 --- a/staging/src/k8s.io/component-base/logs/example/cmd/logger.go +++ b/staging/src/k8s.io/component-base/logs/example/cmd/logger.go @@ -67,7 +67,7 @@ func NewLoggerCommand() *cobra.Command { }, } logsapi.AddFeatureGates(featureGate) - featureGate.AddFlag(cmd.Flags()) + featureGate.AddFlag(cmd.Flags(), "") logsapi.AddFlags(c, cmd.Flags()) return cmd } diff --git a/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go b/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go index c39ef9fe7a60f..6c29c206a3b5a 100644 --- a/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go +++ b/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go @@ -83,7 +83,7 @@ func NewLoggerCommand() *cobra.Command { // Shouldn't happen. panic(err) } - featureGate.AddFlag(cmd.Flags()) + featureGate.AddFlag(cmd.Flags(), "") logsapi.AddFlags(c, cmd.Flags()) return cmd } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go index bfd1135e787c1..93ac2b14b3108 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go @@ -247,7 +247,7 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg } // used later to filter the served resource by those that have expired. - resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*c.GenericConfig.Version) + resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion()) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index d4ddf9490ae9f..a70c4126cf2a2 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -31,6 +31,9 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/filters" genericoptions "k8s.io/apiserver/pkg/server/options" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/component-base/featuregate" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" "k8s.io/kube-aggregator/pkg/apiserver" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" @@ -58,10 +61,15 @@ type AggregatorOptions struct { // with a default AggregatorOptions. func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) *cobra.Command { o := *defaults + featureGate := o.ServerRunOptions.FeatureGate.(featuregate.MutableVersionedFeatureGate) + effectiveVersion := o.ServerRunOptions.EffectiveVersion.(utilversion.MutableEffectiveVersion) cmd := &cobra.Command{ Short: "Launch a API aggregator and proxy server", Long: "Launch a API aggregator and proxy server", RunE: func(c *cobra.Command, args []string) error { + if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + return err + } if err := o.Complete(); err != nil { return err } @@ -76,7 +84,11 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) } cmd.SetContext(ctx) - o.AddFlags(cmd.Flags()) + fs := cmd.Flags() + featureGate.AddFlag(fs, "") + effectiveVersion.AddFlags(fs, "") + + o.AddFlags(fs) return cmd } @@ -91,8 +103,13 @@ func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) { // NewDefaultOptions builds a "normal" set of options. You wouldn't normally expose this, but hyperkube isn't cobra compatible func NewDefaultOptions(out, err io.Writer) *AggregatorOptions { + // effectiveVersion is used to set what apis and feature gates the generic api server is compatible with. + // You can also have the flag setting the effectiveVersion of the aggregator apiserver, and + // having a mapping from the aggregator apiserver version to generic apiserver version. + effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) o := &AggregatorOptions{ - ServerRunOptions: genericoptions.NewServerRunOptions(), + ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), RecommendedOptions: genericoptions.NewRecommendedOptions( defaultEtcdPathPrefix, aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion), @@ -117,7 +134,7 @@ func (o AggregatorOptions) Validate(args []string) error { // Complete fills in missing Options. func (o *AggregatorOptions) Complete() error { - return nil + return o.ServerRunOptions.Complete() } // RunAggregator runs the API Aggregator. diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/doc.go b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/doc.go index a8df931cd4fdc..fc5bdc44fefb0 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/doc.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/doc.go @@ -18,6 +18,7 @@ limitations under the License. // +k8s:deepcopy-gen=package // +k8s:conversion-gen=k8s.io/sample-apiserver/pkg/apis/wardle // +k8s:defaulter-gen=TypeMeta +// +k8s:prerelease-lifecycle-gen=true // +groupName=wardle.example.com // Package v1alpha1 is the v1alpha1 version of the API. diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/types.go b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/types.go index 39079e4a8d95a..a0b3eed416e23 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/types.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/types.go @@ -19,6 +19,8 @@ package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:prerelease-lifecycle-gen:introduced=1.0 +// +k8s:prerelease-lifecycle-gen:removed=1.10 // FlunderList is a list of Flunder objects. type FlunderList struct { @@ -47,6 +49,8 @@ type FlunderStatus struct { // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:prerelease-lifecycle-gen:introduced=1.0 +// +k8s:prerelease-lifecycle-gen:removed=1.10 type Flunder struct { metav1.TypeMeta `json:",inline"` @@ -59,6 +63,8 @@ type Flunder struct { // +genclient // +genclient:nonNamespaced // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:prerelease-lifecycle-gen:introduced=1.0 +// +k8s:prerelease-lifecycle-gen:removed=1.10 type Fischer struct { metav1.TypeMeta `json:",inline"` @@ -71,6 +77,8 @@ type Fischer struct { // +genclient:nonNamespaced // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:prerelease-lifecycle-gen:introduced=1.0 +// +k8s:prerelease-lifecycle-gen:removed=1.10 // FischerList is a list of Fischer objects. type FischerList struct { diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/zz_generated.prerelease-lifecycle.go b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/zz_generated.prerelease-lifecycle.go new file mode 100644 index 0000000000000..04a9f6b8c16d1 --- /dev/null +++ b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/zz_generated.prerelease-lifecycle.go @@ -0,0 +1,94 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by prerelease-lifecycle-gen. DO NOT EDIT. + +package v1alpha1 + +// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +func (in *Fischer) APILifecycleIntroduced() (major, minor int) { + return 1, 0 +} + +// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor. +func (in *Fischer) APILifecycleDeprecated() (major, minor int) { + return 1, 3 +} + +// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor. +func (in *Fischer) APILifecycleRemoved() (major, minor int) { + return 1, 10 +} + +// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +func (in *FischerList) APILifecycleIntroduced() (major, minor int) { + return 1, 0 +} + +// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor. +func (in *FischerList) APILifecycleDeprecated() (major, minor int) { + return 1, 3 +} + +// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor. +func (in *FischerList) APILifecycleRemoved() (major, minor int) { + return 1, 10 +} + +// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +func (in *Flunder) APILifecycleIntroduced() (major, minor int) { + return 1, 0 +} + +// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor. +func (in *Flunder) APILifecycleDeprecated() (major, minor int) { + return 1, 3 +} + +// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor. +func (in *Flunder) APILifecycleRemoved() (major, minor int) { + return 1, 10 +} + +// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +func (in *FlunderList) APILifecycleIntroduced() (major, minor int) { + return 1, 0 +} + +// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor. +func (in *FlunderList) APILifecycleDeprecated() (major, minor int) { + return 1, 3 +} + +// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor. +func (in *FlunderList) APILifecycleRemoved() (major, minor int) { + return 1, 10 +} diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go index 384c46d4ae316..a968d36177b18 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go @@ -17,6 +17,8 @@ limitations under the License. package apiserver import ( + "strconv" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -25,6 +27,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/sample-apiserver/pkg/apis/wardle" "k8s.io/sample-apiserver/pkg/apis/wardle/install" wardleregistry "k8s.io/sample-apiserver/pkg/registry" @@ -37,7 +40,8 @@ var ( Scheme = runtime.NewScheme() // Codecs provides methods for retrieving codecs and serializers for specific // versions and content types. - Codecs = serializer.NewCodecFactory(Scheme) + Codecs = serializer.NewCodecFactory(Scheme) + WardleComponentName = "wardle-server" ) func init() { @@ -90,10 +94,10 @@ func (cfg *Config) Complete() CompletedConfig { cfg.GenericConfig.Complete(), &cfg.ExtraConfig, } - + wardleEffectiveVersion := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(WardleComponentName) c.GenericConfig.Version = &version.Info{ - Major: "1", - Minor: "0", + Major: strconv.Itoa(int(wardleEffectiveVersion.BinaryVersion().Major())), + Minor: strconv.Itoa(int(wardleEffectiveVersion.BinaryVersion().Minor())), } return CompletedConfig{&c} diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index d6310425a5071..275c1a8718f27 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -18,6 +18,7 @@ package server import ( "context" + "errors" "fmt" "io" "net" @@ -27,11 +28,14 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/sample-apiserver/pkg/admission/plugin/banflunder" "k8s.io/sample-apiserver/pkg/admission/wardleinitializer" "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1" @@ -55,6 +59,24 @@ type WardleServerOptions struct { AlternateDNS []string } +func mapWardleEffectiveVersionToKubeEffectiveVersion(registry utilversion.ComponentGlobalsRegistry) error { + wardleVer := registry.EffectiveVersionFor(apiserver.WardleComponentName) + kubeVer := registry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer).(utilversion.MutableEffectiveVersion) + // map from wardle emulation version to kube emulation version. + emulationVersionMap := map[string]string{ + "1.2": kubeVer.BinaryVersion().AddMinor(1).String(), + "1.1": kubeVer.BinaryVersion().String(), + "1.0": kubeVer.BinaryVersion().SubtractMinor(1).String(), + } + wardleEmulationVer := wardleVer.EmulationVersion() + if kubeEmulationVer, ok := emulationVersionMap[wardleEmulationVer.String()]; ok { + kubeVer.SetEmulationVersion(version.MustParse(kubeEmulationVer)) + } else { + return fmt.Errorf("cannot find mapping from wardle emulation version: %s to kube version", wardleVer.EmulationVersion().String()) + } + return nil +} + // NewWardleServerOptions returns a new WardleServerOptions func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions { o := &WardleServerOptions{ @@ -94,7 +116,14 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti flags := cmd.Flags() o.RecommendedOptions.AddFlags(flags) - utilfeature.DefaultMutableFeatureGate.AddFlag(flags) + + wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, false)) + _, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + + wardleEffectiveVersion.AddFlags(flags, "wardle-") + featureGate.AddFlag(flags, "") return cmd } @@ -103,6 +132,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti func (o WardleServerOptions) Validate(args []string) error { errors := []error{} errors = append(errors, o.RecommendedOptions.Validate()...) + errors = append(errors, utilversion.DefaultComponentGlobalsRegistry.ValidateAllComponents()...) return utilerrors.NewAggregate(errors) } @@ -114,6 +144,17 @@ func (o *WardleServerOptions) Complete() error { // add admission plugins to the RecommendedPluginOrder o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder") + // convert wardle effective version to kube effective version to be used in generic api server, and set the generic api server feature gate. + if err := mapWardleEffectiveVersionToKubeEffectiveVersion(utilversion.DefaultComponentGlobalsRegistry); err != nil { + return err + } + if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + return err + } + if errs := utilversion.DefaultComponentGlobalsRegistry.ValidateAllComponents(); len(errs) > 0 { + return errors.Join(errs...) + } + return nil } @@ -144,6 +185,9 @@ func (o *WardleServerOptions) Config() (*apiserver.Config, error) { serverConfig.OpenAPIV3Config.Info.Title = "Wardle" serverConfig.OpenAPIV3Config.Info.Version = "0.1" + serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(utilversion.ComponentGenericAPIServer) + serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer) + if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { return nil, err } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go new file mode 100644 index 0000000000000..6ee274c65b64f --- /dev/null +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "testing" + + "k8s.io/apimachinery/pkg/util/version" + utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/sample-apiserver/pkg/apiserver" + + "github.com/stretchr/testify/assert" +) + +func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) { + wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") + defaultKubeEffectiveVersion := utilversion.DefaultKubeEffectiveVersion() + + testCases := []struct { + desc string + wardleEmulationVer *version.Version + expectedKubeEmulationVer *version.Version + }{ + { + desc: "1 version higher than kube binary", + wardleEmulationVer: version.MajorMinor(1, 2), + expectedKubeEmulationVer: defaultKubeEffectiveVersion.BinaryVersion().AddMinor(1), + }, + { + desc: "no mapping", + wardleEmulationVer: version.MajorMinor(1, 10), + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + registry := utilversion.NewComponentGlobalsRegistry() + _ = registry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, true) + _ = registry.Register(utilversion.ComponentGenericAPIServer, defaultKubeEffectiveVersion, nil, true) + + wardleEffectiveVersion.SetEmulationVersion(tc.wardleEmulationVer) + err := mapWardleEffectiveVersionToKubeEffectiveVersion(registry) + if tc.expectedKubeEmulationVer == nil { + if err == nil { + t.Fatal("expected error, no error found") + } + } else { + assert.True(t, registry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer).EmulationVersion().EqualTo(tc.expectedKubeEmulationVer)) + } + }) + } +} diff --git a/test/e2e/dra/test-driver/app/server.go b/test/e2e/dra/test-driver/app/server.go index 3f55c5414634c..59e4ac42a99e1 100644 --- a/test/e2e/dra/test-driver/app/server.go +++ b/test/e2e/dra/test-driver/app/server.go @@ -88,7 +88,7 @@ func NewCommand() *cobra.Command { fs = sharedFlagSets.FlagSet("other") featureGate := featuregate.NewFeatureGate() utilruntime.Must(logsapi.AddFeatureGates(featureGate)) - featureGate.AddFlag(fs) + featureGate.AddFlag(fs, "") fs = cmd.PersistentFlags() for _, f := range sharedFlagSets.FlagSets { diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go index 4488715b3bd58..c092d9d69b666 100644 --- a/test/e2e_node/services/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -29,6 +29,8 @@ import ( netutils "k8s.io/utils/net" utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/e2e/framework" @@ -63,7 +65,9 @@ func NewAPIServer(storageConfig storagebackend.Config) *APIServer { func (a *APIServer) Start(ctx context.Context) error { const tokenFilePath = "known_tokens.csv" - o := options.NewServerRunOptions() + featureGate := utilfeature.DefaultFeatureGate + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + o := options.NewServerRunOptions(featureGate, effectiveVersion) o.Etcd.StorageConfig = a.storageConfig _, ipnet, err := netutils.ParseCIDRSloppy(clusterIPRange) if err != nil { diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index 63bb662561a97..81edfbbc0aa16 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -32,6 +32,7 @@ import ( "testing" "time" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -55,16 +56,19 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/endpoints/handlers" "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" "k8s.io/client-go/metadata" restclient "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/pager" "k8s.io/klog/v2" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/pkg/controlplane" + "k8s.io/kubernetes/pkg/kubeapiserver" "k8s.io/kubernetes/test/integration" "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" @@ -2929,6 +2933,298 @@ func TestDedupOwnerReferences(t *testing.T) { } } +func TestEmulatedStorageVersion(t *testing.T) { + validVap := &admissionregistrationv1.ValidatingAdmissionPolicy{ + Spec: admissionregistrationv1.ValidatingAdmissionPolicySpec{ + MatchConstraints: &admissionregistrationv1.MatchResources{ + ResourceRules: []admissionregistrationv1.NamedRuleWithOperations{ + { + ResourceNames: []string{"foo"}, + RuleWithOperations: admissionregistrationv1.RuleWithOperations{ + Operations: []admissionregistrationv1.OperationType{"CREATE"}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"*"}, + APIVersions: []string{"*"}, + Resources: []string{"*"}, + }, + }, + }, + }, + }, + Validations: []admissionregistrationv1.Validation{ + { + Expression: "true", + Message: "always valid", + }, + }, + }, + } + + type testCase struct { + name string + emulatedVersion string + gvr schema.GroupVersionResource + object runtime.Object + expectedStorageVersion schema.GroupVersion + } + cases := []testCase{ + { + name: "vap first ga release", + emulatedVersion: "1.30", + gvr: schema.GroupVersionResource{ + Group: "admissionregistration.k8s.io", + Version: "v1", + Resource: "validatingadmissionpolicies", + }, + object: validVap, + expectedStorageVersion: schema.GroupVersion{ + Group: "admissionregistration.k8s.io", + Version: "v1beta1", + }, + }, + { + name: "vap after ga release", + emulatedVersion: "1.31", + gvr: schema.GroupVersionResource{ + Group: "admissionregistration.k8s.io", + Version: "v1beta1", + Resource: "validatingadmissionpolicies", + }, + object: validVap, + expectedStorageVersion: schema.GroupVersion{ + Group: "admissionregistration.k8s.io", + Version: "v1", + }, + }, + } + + // Group cases by their emulated version + groupedCases := map[string][]testCase{} + for _, c := range cases { + groupedCases[c.emulatedVersion] = append(groupedCases[c.emulatedVersion], c) + } + + for emulatedVersion, cases := range groupedCases { + t.Run(emulatedVersion, func(t *testing.T) { + server := kubeapiservertesting.StartTestServerOrDie( + t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: emulatedVersion}, + []string{"--emulated-version=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd()) + defer server.TearDownFn() + + client := clientset.NewForConfigOrDie(server.ClientConfig) + dynamicClient := dynamic.NewForConfigOrDie(server.ClientConfig) + + // create test namespace + testNamespace := "test-emulated-storage-version" + _, err := client.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + }, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create test ns: %v", err) + } + + restMapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(client.Discovery())) + + for i, c := range cases { + t.Run(c.name, func(t *testing.T) { + gvk, err := restMapper.KindFor(c.gvr) + if err != nil { + t.Fatalf("failed to get GVK: %v", err) + } + c.object.GetObjectKind().SetGroupVersionKind(gvk) + + mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + t.Fatalf("failed to get RESTMapping: %v", err) + } else if mapping.Resource != c.gvr { + t.Fatalf("expected resource %v, got %v", c.gvr, mapping.Resource) + } + + asUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(c.object) + if err != nil { + t.Fatalf("failed to convert object to unstructured: %v", err) + } + + uns := &unstructured.Unstructured{ + Object: asUnstructured, + } + uns.SetName(fmt.Sprintf("test-object%d", i)) + + ns := testNamespace + if mapping.Scope.Name() == meta.RESTScopeNameRoot { + ns = "" + } + + // create object + created, err := dynamicClient.Resource(c.gvr).Namespace(ns).Create(context.TODO(), uns, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create object: %v", err) + } + + // Fetch object from ETCD + // Use this poor man's way to get the etcd path. This wont + // work for all resources, but should work for most we want + // to test against + resourcePrefix := mapping.Resource.Resource + if special, ok := kubeapiserver.SpecialDefaultResourcePrefixes[c.gvr.GroupResource()]; ok { + resourcePrefix = special + } + etcdPathComponents := []string{ + "/", + server.EtcdStoragePrefix, + resourcePrefix, + } + + if len(ns) > 0 { + etcdPathComponents = append(etcdPathComponents, ns) + } + etcdPathComponents = append(etcdPathComponents, created.GetName()) + etcdPath := path.Join(etcdPathComponents...) + fetched, err := server.EtcdClient.Get(context.TODO(), etcdPath) + if err != nil { + t.Fatalf("failed to fetch object from etcd: %v", err) + } else if fetched.More || fetched.Count != 1 || len(fetched.Kvs) != 1 { + t.Fatalf("unexpected fetched response: %v", fetched) + } + + storedObject := &metav1.PartialObjectMetadata{} + err = json.Unmarshal(fetched.Kvs[0].Value, storedObject) + + if err != nil { + t.Fatalf("failed to decode object: %v", err) + } else if storedObject.GroupVersionKind().GroupVersion() != c.expectedStorageVersion { + t.Fatalf("expected storage version %s, got %s", c.expectedStorageVersion.String(), storedObject.GroupVersionKind().GroupVersion().String()) + } + }) + } + }) + } +} + +func TestEnableEmulationVersion(t *testing.T) { + server := kubeapiservertesting.StartTestServerOrDie(t, + &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, + []string{"--emulated-version=1.31"}, framework.SharedEtcd()) + defer server.TearDownFn() + + rt, err := restclient.TransportFor(server.ClientConfig) + if err != nil { + t.Fatal(err) + } + + tcs := []struct { + path string + expectedStatusCode int + }{ + { + path: "/", + expectedStatusCode: 200, + }, + { + path: "/apis/apps/v1/deployments", + expectedStatusCode: 200, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1/flowschemas", + expectedStatusCode: 200, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas", // introduced at 1.20, removed at 1.26 + expectedStatusCode: 404, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1beta2/flowschemas", // introduced at 1.23, removed at 1.29 + expectedStatusCode: 404, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1beta3/flowschemas", // introduced at 1.26, removed at 1.32 + expectedStatusCode: 200, + }, + } + + for _, tc := range tcs { + t.Run(tc.path, func(t *testing.T) { + req, err := http.NewRequest("GET", server.ClientConfig.Host+tc.path, nil) + if err != nil { + t.Fatal(err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != tc.expectedStatusCode { + t.Errorf("expect status code: %d, got : %d\n", tc.expectedStatusCode, resp.StatusCode) + } + defer func() { + _ = resp.Body.Close() + }() + }) + } +} + +func TestDisableEmulationVersion(t *testing.T) { + server := kubeapiservertesting.StartTestServerOrDie(t, + &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, + []string{}, framework.SharedEtcd()) + defer server.TearDownFn() + + rt, err := restclient.TransportFor(server.ClientConfig) + if err != nil { + t.Fatal(err) + } + + tcs := []struct { + path string + expectedStatusCode int + }{ + { + path: "/", + expectedStatusCode: 200, + }, + { + path: "/apis/apps/v1/deployments", + expectedStatusCode: 200, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1/flowschemas", + expectedStatusCode: 200, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas", // introduced at 1.20, removed at 1.26 + expectedStatusCode: 404, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1beta2/flowschemas", // introduced at 1.23, removed at 1.29 + expectedStatusCode: 404, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1beta3/flowschemas", // introduced at 1.26, removed at 1.32 + expectedStatusCode: 404, + }, + } + + for _, tc := range tcs { + t.Run(tc.path, func(t *testing.T) { + req, err := http.NewRequest("GET", server.ClientConfig.Host+tc.path, nil) + if err != nil { + t.Fatal(err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != tc.expectedStatusCode { + t.Errorf("expect status code: %d, got : %d\n", tc.expectedStatusCode, resp.StatusCode) + } + defer func() { + _ = resp.Body.Close() + }() + }) + } +} + type dependentClient struct { t *testing.T client dynamic.ResourceInterface diff --git a/test/integration/auth/rbac_test.go b/test/integration/auth/rbac_test.go index f162eb6df2da2..f24067e5fc9fe 100644 --- a/test/integration/auth/rbac_test.go +++ b/test/integration/auth/rbac_test.go @@ -29,6 +29,7 @@ import ( rbacapi "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" @@ -80,8 +81,8 @@ type testRESTOptionsGetter struct { config *controlplane.Config } -func (getter *testRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) { - storageConfig, err := getter.config.ControlPlane.Extra.StorageFactory.NewConfig(resource) +func (getter *testRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) { + storageConfig, err := getter.config.ControlPlane.Extra.StorageFactory.NewConfig(resource, example) if err != nil { return generic.RESTOptions{}, fmt.Errorf("failed to get storage: %v", err) } diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index 1b001681b7bd3..3d717bf25bcd9 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -1219,7 +1219,7 @@ func getRESTOptionsGetterForSecrets(t testing.TB, test *transformTest) generic.R t.Fatal("not REST options found") } - opts, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: "secrets"}) + opts, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: "secrets"}, nil) if err != nil { t.Fatal(err) } diff --git a/test/integration/controlplane/transformation/transformation_test.go b/test/integration/controlplane/transformation/transformation_test.go index fa9b74157ef9d..f1c16a9b6109b 100644 --- a/test/integration/controlplane/transformation/transformation_test.go +++ b/test/integration/controlplane/transformation/transformation_test.go @@ -115,7 +115,9 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, return nil, fmt.Errorf("failed to read config file: %w", err) } - if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(tb, nil, e.getEncryptionOptions(reload), e.storageConfig); err != nil { + if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer( + tb, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "0.0"}, + e.getEncryptionOptions(reload), e.storageConfig); err != nil { e.cleanUp() return nil, fmt.Errorf("failed to start KubeAPI server: %w", err) } diff --git a/test/integration/etcd/crd_overlap_storage_test.go b/test/integration/etcd/crd_overlap_storage_test.go index 7aa898a0bf250..ebe1e042dc3ce 100644 --- a/test/integration/etcd/crd_overlap_storage_test.go +++ b/test/integration/etcd/crd_overlap_storage_test.go @@ -32,9 +32,11 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/dynamic" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" ) // TestOverlappingBuiltInResources ensures the list of group-resources the custom resource finalizer should skip is up to date @@ -69,7 +71,9 @@ func TestOverlappingBuiltInResources(t *testing.T) { // TestOverlappingCustomResourceAPIService ensures creating and deleting a custom resource overlapping with APIServices does not destroy APIService data func TestOverlappingCustomResourceAPIService(t *testing.T) { - apiServer := StartRealAPIServerOrDie(t) + apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { + opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30") + }) defer apiServer.Cleanup() apiServiceClient, err := apiregistrationclient.NewForConfig(apiServer.Config) @@ -231,7 +235,9 @@ func TestOverlappingCustomResourceAPIService(t *testing.T) { // TestOverlappingCustomResourceCustomResourceDefinition ensures creating and deleting a custom resource overlapping with CustomResourceDefinition does not destroy CustomResourceDefinition data func TestOverlappingCustomResourceCustomResourceDefinition(t *testing.T) { - apiServer := StartRealAPIServerOrDie(t) + apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { + opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30") + }) defer apiServer.Cleanup() crdClient, err := crdclient.NewForConfig(apiServer.Config) diff --git a/test/integration/etcd/etcd_cross_group_test.go b/test/integration/etcd/etcd_cross_group_test.go index d247edeaa5eb3..f24f0ee39b42e 100644 --- a/test/integration/etcd/etcd_cross_group_test.go +++ b/test/integration/etcd/etcd_cross_group_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/storage" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/dynamic" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" ) @@ -38,6 +39,7 @@ import ( func TestCrossGroupStorage(t *testing.T) { apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { // force enable all resources so we can check storage. + opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30") }) defer apiServer.Cleanup() diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index 224be051e8b4f..eda94aee93fc2 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -36,6 +36,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/dynamic" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" @@ -75,6 +76,7 @@ func TestEtcdStoragePath(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true) featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true) apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { + opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0") }) defer apiServer.Cleanup() defer dumpEtcdKVOnFailure(t, apiServer.KV) diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index ed69e3d532f59..fb57211ea43b3 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -40,6 +40,8 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" cacheddiscovery "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" @@ -91,7 +93,9 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) } - opts := options.NewServerRunOptions() + featureGate := utilfeature.DefaultFeatureGate + effectiveVersion := utilversion.DefaultBuildEffectiveVersion() + opts := options.NewServerRunOptions(featureGate, effectiveVersion) opts.Options.SecureServing.Listener = listener opts.Options.SecureServing.ServerCert.CertDirectory = certDir opts.Options.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index a0c3852618bd0..2ed1f1c001c68 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -292,6 +292,7 @@ func TestAggregatedAPIServer(t *testing.T) { "--etcd-servers", framework.GetEtcdURL(), "--cert-dir", wardleCertDir, "--kubeconfig", wardleToKASKubeConfigFile, + "--wardle-emulated-version", "1.1", }) if err := wardleCmd.Execute(); err != nil { t.Error(err) diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index bb43850466d9c..5d514b360d45b 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -35,6 +35,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" genericapiserver "k8s.io/apiserver/pkg/server" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" client "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/cert" @@ -135,7 +137,9 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) ( t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) } - opts := options.NewServerRunOptions() + featureGate := utilfeature.DefaultFeatureGate + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + opts := options.NewServerRunOptions(featureGate, effectiveVersion) opts.SecureServing.Listener = listener opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") opts.SecureServing.ServerCert.CertDirectory = certDir diff --git a/test/integration/servicecidr/allocator_test.go b/test/integration/servicecidr/allocator_test.go index dd4b45cc52106..ad6d6280dc253 100644 --- a/test/integration/servicecidr/allocator_test.go +++ b/test/integration/servicecidr/allocator_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/kubernetes" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" @@ -122,6 +123,7 @@ func TestServiceAllocIPAddress(t *testing.T) { ModifyServerRunOptions: func(opts *options.ServerRunOptions) { opts.ServiceClusterIPRanges = serviceCIDR opts.GenericServerRunOptions.AdvertiseAddress = netutils.ParseIPSloppy("2001:db8::10") + opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.31") opts.APIEnablement.RuntimeConfig.Set("networking.k8s.io/v1alpha1=true") }, })