Skip to content

Commit

Permalink
apiserver: Add API emulation versioning.
Browse files Browse the repository at this point in the history
Co-authored-by: Siyuan Zhang <sizhang@google.com>
Co-authored-by: Joe Betz <jpbetz@google.com>
Co-authored-by: Alex Zielenski <zielenski@google.com>

Signed-off-by: Siyuan Zhang <sizhang@google.com>
  • Loading branch information
siyuanfoundation committed Jun 25, 2024
1 parent d0579b6 commit 403301b
Show file tree
Hide file tree
Showing 86 changed files with 3,414 additions and 421 deletions.
8 changes: 5 additions & 3 deletions cmd/kube-apiserver/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import (
utilnet "k8s.io/apimachinery/pkg/util/net"
cliflag "k8s.io/component-base/cli/flag"

utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/cluster/ports"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
Expand Down Expand Up @@ -63,10 +65,10 @@ type Extra struct {
MasterCount int
}

// NewServerRunOptions creates a new ServerRunOptions object with default parameters
func NewServerRunOptions() *ServerRunOptions {
// NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run.
func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions {
s := ServerRunOptions{
Options: controlplaneapiserver.NewOptions(),
Options: controlplaneapiserver.NewOptions(featureGate, effectiveVersion),
CloudProvider: kubeoptions.NewCloudProviderOptions(),

Extra: Extra{
Expand Down
16 changes: 15 additions & 1 deletion cmd/kube-apiserver/app/options/options_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,11 @@ import (
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/etcd3"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilversion "k8s.io/apiserver/pkg/util/version"
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
kapi "k8s.io/kubernetes/pkg/apis/core"
Expand All @@ -46,10 +48,15 @@ import (

func TestAddFlags(t *testing.T) {
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
s := NewServerRunOptions()

featureGate := featuregate.NewFeatureGate()
effectiveVersion := utilversion.NewEffectiveVersion("1.32")
s := NewServerRunOptions(featureGate, effectiveVersion)
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")

args := []string{
"--enable-admission-plugins=AlwaysDeny",
Expand Down Expand Up @@ -121,6 +128,7 @@ func TestAddFlags(t *testing.T) {
"--storage-backend=etcd3",
"--service-cluster-ip-range=192.168.128.0/17",
"--lease-reuse-duration-seconds=100",
"--emulated-version=1.31",
}
fs.Parse(args)

Expand All @@ -136,6 +144,8 @@ func TestAddFlags(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
Expand Down Expand Up @@ -337,4 +347,8 @@ func TestAddFlags(t *testing.T) {
if !reflect.DeepEqual(expected, s) {
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
}

if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" {
t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31")
}
}
17 changes: 14 additions & 3 deletions cmd/kube-apiserver/app/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
serverstorage "k8s.io/apiserver/pkg/server/storage"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/util/notfoundhandler"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/apiserver/pkg/util/webhook"
clientgoinformers "k8s.io/client-go/informers"
"k8s.io/client-go/rest"
Expand Down Expand Up @@ -63,7 +64,10 @@ func init() {

// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
s := options.NewServerRunOptions()
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.ComponentGenericAPIServer, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
s := options.NewServerRunOptions(featureGate, effectiveVersion)

cmd := &cobra.Command{
Use: "kube-apiserver",
Long: `The Kubernetes API server validates and configures data
Expand All @@ -83,9 +87,13 @@ cluster's shared state through which all other components interact.`,
verflag.PrintAndExitIfRequested()
fs := cmd.Flags()

if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return err
}

// Activate logging as soon as possible, after that
// show flags with the final logging configuration.
if err := logsapi.ValidateAndApply(s.Logs, utilfeature.DefaultFeatureGate); err != nil {
if err := logsapi.ValidateAndApply(s.Logs, featureGate); err != nil {
return err
}
cliflag.PrintFlags(fs)
Expand All @@ -101,7 +109,7 @@ cluster's shared state through which all other components interact.`,
return utilerrors.NewAggregate(errs)
}
// add feature enablement metrics
utilfeature.DefaultMutableFeatureGate.AddMetrics()
featureGate.AddMetrics()
return Run(cmd.Context(), completedOptions)
},
Args: func(cmd *cobra.Command, args []string) error {
Expand All @@ -118,6 +126,9 @@ cluster's shared state through which all other components interact.`,
fs := cmd.Flags()
namedFlagSets := s.Flags()
verflag.AddFlags(namedFlagSets.FlagSet("global"))
featureGate.AddFlag(namedFlagSets.FlagSet("global"), "")
effectiveVersion.AddFlags(namedFlagSets.FlagSet("global"), "")

globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
for _, f := range namedFlagSets.FlagSets {
Expand Down
21 changes: 20 additions & 1 deletion cmd/kube-apiserver/app/testing/testserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import (
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
clientgotransport "k8s.io/client-go/transport"
Expand Down Expand Up @@ -98,6 +99,9 @@ type TestServerInstanceOptions struct {
// We specify this as on option to pass a common proxyCA to multiple apiservers to simulate
// an apiserver version skew scenario where all apiservers use the same proxyCA to verify client connections.
ProxyCA *ProxyCA
// Set the BinaryVersion of server effective version.
// Default to 1.31
BinaryVersion string
}

// TestServer return values supplied by kube-test-ApiServer
Expand Down Expand Up @@ -177,10 +181,21 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,

fs := pflag.NewFlagSet("test", pflag.PanicOnError)

s := options.NewServerRunOptions()
featureGate := utilfeature.DefaultMutableFeatureGate
binaryVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String()
if instanceOptions.BinaryVersion != "" {
binaryVersion = instanceOptions.BinaryVersion
}
effectiveVersion := utilversion.NewEffectiveVersion(binaryVersion)
_ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true)

s := options.NewServerRunOptions(featureGate, effectiveVersion)

for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")

s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
if err != nil {
Expand Down Expand Up @@ -321,6 +336,10 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
return result, err
}

if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return result, err
}

saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key")
if err != nil {
t.Fatalf("create temp file failed: %v", err)
Expand Down
2 changes: 1 addition & 1 deletion cmd/kube-controller-manager/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
fs := fss.FlagSet("misc")
fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
fs.StringVar(&s.Generic.ClientConnection.Kubeconfig, "kubeconfig", s.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).")
utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "")

return fss
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/kube-scheduler/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ func (o *Options) initFlags() {
o.Authorization.AddFlags(nfs.FlagSet("authorization"))
o.Deprecated.AddFlags(nfs.FlagSet("deprecated"))
options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election"))
utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"))
utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"), "")
o.Metrics.AddFlags(nfs.FlagSet("metrics"))
logsapi.AddFlags(o.Logs, nfs.FlagSet("logs"))

Expand Down
2 changes: 1 addition & 1 deletion pkg/controlplane/apiserver/apiextensions.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ limitations under the License.
package apiserver

import (
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
apiextensionsoptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
Expand All @@ -27,6 +26,7 @@ import (
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/util/webhook"
"k8s.io/client-go/informers"
v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"

"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
)
Expand Down
2 changes: 1 addition & 1 deletion pkg/controlplane/apiserver/apis.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error
nonLegacy := []*genericapiserver.APIGroupInfo{}

// used later in the loop to filter the served resource by those that have expired.
resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*s.GenericAPIServer.Version)
resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion())
if err != nil {
return err
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/controlplane/apiserver/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,9 @@ func BuildGenericConfig(
}

storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
storageFactoryConfig.CurrentVersion = genericConfig.EffectiveVersion
storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(genericConfig.EffectiveVersion)
storageFactory, lastErr = storageFactoryConfig.Complete(s.Etcd).New()
if lastErr != nil {
return
Expand Down
8 changes: 6 additions & 2 deletions pkg/controlplane/apiserver/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
Expand All @@ -32,7 +34,9 @@ import (
)

func TestBuildGenericConfig(t *testing.T) {
opts := options.NewOptions()
featureGate := featuregate.NewFeatureGate()
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
opts := options.NewOptions(featureGate, effectiveVersion)
s := (&apiserveroptions.SecureServingOptions{
BindAddress: netutils.ParseIPSloppy("127.0.0.1"),
}).WithLoopback()
Expand Down Expand Up @@ -66,7 +70,7 @@ func TestBuildGenericConfig(t *testing.T) {
t.Errorf("There are different StorageObjectCountTracker in genericConfig and storageFactory")
}

restOptions, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""})
restOptions, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil)
if err != nil {
t.Fatal(err)
}
Expand Down
10 changes: 8 additions & 2 deletions pkg/controlplane/apiserver/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,10 @@ import (
peerreconcilers "k8s.io/apiserver/pkg/reconcilers"
genericoptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/util/keyutil"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
logsapi "k8s.io/component-base/logs/api/v1"
"k8s.io/component-base/metrics"
Expand Down Expand Up @@ -98,9 +100,9 @@ type CompletedOptions struct {
}

// NewOptions creates a new ServerRunOptions object with default parameters
func NewOptions() *Options {
func NewOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *Options {
s := Options{
GenericServerRunOptions: genericoptions.NewServerRunOptions(),
GenericServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
SecureServing: kubeoptions.NewSecureServingOptions(),
Audit: genericoptions.NewAuditOptions(),
Expand Down Expand Up @@ -202,6 +204,10 @@ func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (Comple
Options: *o,
}

if err := completed.GenericServerRunOptions.Complete(); err != nil {
return CompletedOptions{}, err
}

// set defaults
if err := completed.GenericServerRunOptions.DefaultAdvertiseAddress(completed.SecureServing.SecureServingOptions); err != nil {
return CompletedOptions{}, err
Expand Down
15 changes: 14 additions & 1 deletion pkg/controlplane/apiserver/options/options_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,11 @@ import (
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/etcd3"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilversion "k8s.io/apiserver/pkg/util/version"
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
netutils "k8s.io/utils/net"
Expand All @@ -42,12 +44,16 @@ import (

func TestAddFlags(t *testing.T) {
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
s := NewOptions()
featureGate := featuregate.NewFeatureGate()
effectiveVersion := utilversion.NewEffectiveVersion("1.32")
s := NewOptions(featureGate, effectiveVersion)
var fss cliflag.NamedFlagSets
s.AddFlags(&fss)
for _, f := range fss.FlagSets {
fs.AddFlagSet(f)
}
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")

args := []string{
"--enable-admission-plugins=AlwaysDeny",
Expand Down Expand Up @@ -108,6 +114,7 @@ func TestAddFlags(t *testing.T) {
"--request-timeout=2m",
"--storage-backend=etcd3",
"--lease-reuse-duration-seconds=100",
"--emulated-version=1.31",
}
fs.Parse(args)

Expand All @@ -122,6 +129,8 @@ func TestAddFlags(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
Expand Down Expand Up @@ -292,4 +301,8 @@ func TestAddFlags(t *testing.T) {
if !reflect.DeepEqual(expected, s) {
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
}

if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" {
t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31")
}
}
1 change: 1 addition & 0 deletions pkg/controlplane/apiserver/options/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ func validateUnknownVersionInteroperabilityProxyFlags(options *Options) []error
func (s *Options) Validate() []error {
var errs []error

errs = append(errs, s.GenericServerRunOptions.Validate()...)
errs = append(errs, s.Etcd.Validate()...)
errs = append(errs, validateAPIPriorityAndFairness(s)...)
errs = append(errs, s.SecureServing.Validate()...)
Expand Down
5 changes: 3 additions & 2 deletions pkg/controlplane/apiserver/options/validation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
kubeapiserveradmission "k8s.io/apiserver/pkg/admission"
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
basemetrics "k8s.io/component-base/metrics"
"k8s.io/kubernetes/pkg/features"
Expand Down Expand Up @@ -200,7 +201,7 @@ func TestValidateOptions(t *testing.T) {
name: "validate master count equal 0",
expectErrors: true,
options: &Options{
GenericServerRunOptions: &genericoptions.ServerRunOptions{},
GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},
Expand All @@ -227,7 +228,7 @@ func TestValidateOptions(t *testing.T) {
name: "validate token request enable not attempted",
expectErrors: true,
options: &Options{
GenericServerRunOptions: &genericoptions.ServerRunOptions{},
GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},
Expand Down
2 changes: 1 addition & 1 deletion pkg/controlplane/apiserver/peer.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func BuildPeerProxy(versionedInformer clientgoinformers.SharedInformerFactory, s
// The peer endpoint leases are used to find network locations of apiservers for peer proxy
func CreatePeerEndpointLeaseReconciler(c genericapiserver.Config, storageFactory serverstorage.StorageFactory) (reconcilers.PeerEndpointLeaseReconciler, error) {
ttl := DefaultPeerEndpointReconcilerTTL
config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo"))
config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo"), &api.Endpoints{})
if err != nil {
return nil, fmt.Errorf("error creating storage factory config: %w", err)
}
Expand Down
Loading

0 comments on commit 403301b

Please sign in to comment.