From d1e47a881180b0ad8b7079db1867d14789796858 Mon Sep 17 00:00:00 2001 From: jianhuiz Date: Wed, 30 Mar 2016 09:45:43 -0700 Subject: [PATCH 1/2] duplicate /cmd/kube-apiserver to /federation/cmd/federated-apiserver --- federation/cmd/federated-apiserver/OWNERS | 5 + .../cmd/federated-apiserver/apiserver.go | 54 ++ .../app/options/options.go | 269 ++++++++ .../app/options/options_test.go | 99 +++ .../cmd/federated-apiserver/app/plugins.go | 40 ++ .../cmd/federated-apiserver/app/server.go | 582 ++++++++++++++++++ .../federated-apiserver/app/server_test.go | 227 +++++++ 7 files changed, 1276 insertions(+) create mode 100644 federation/cmd/federated-apiserver/OWNERS create mode 100644 federation/cmd/federated-apiserver/apiserver.go create mode 100644 federation/cmd/federated-apiserver/app/options/options.go create mode 100644 federation/cmd/federated-apiserver/app/options/options_test.go create mode 100644 federation/cmd/federated-apiserver/app/plugins.go create mode 100644 federation/cmd/federated-apiserver/app/server.go create mode 100644 federation/cmd/federated-apiserver/app/server_test.go diff --git a/federation/cmd/federated-apiserver/OWNERS b/federation/cmd/federated-apiserver/OWNERS new file mode 100644 index 0000000000000..c5bdda2d1ad05 --- /dev/null +++ b/federation/cmd/federated-apiserver/OWNERS @@ -0,0 +1,5 @@ +assignees: + - lavalamp + - smarterclayton + - nikhiljindal + - krousey diff --git a/federation/cmd/federated-apiserver/apiserver.go b/federation/cmd/federated-apiserver/apiserver.go new file mode 100644 index 0000000000000..c63ec99c35183 --- /dev/null +++ b/federation/cmd/federated-apiserver/apiserver.go @@ -0,0 +1,54 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// apiserver is the main api server and master for the cluster. +// it is responsible for serving the cluster management API. +package main + +import ( + "fmt" + "math/rand" + "os" + "runtime" + "time" + + "k8s.io/kubernetes/cmd/kube-apiserver/app" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/flag" + "k8s.io/kubernetes/pkg/version/verflag" + + "github.com/spf13/pflag" +) + +func main() { + runtime.GOMAXPROCS(runtime.NumCPU()) + rand.Seed(time.Now().UTC().UnixNano()) + + s := options.NewAPIServer() + s.AddFlags(pflag.CommandLine) + + flag.InitFlags() + util.InitLogs() + defer util.FlushLogs() + + verflag.PrintAndExitIfRequested() + + if err := app.Run(s); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} diff --git a/federation/cmd/federated-apiserver/app/options/options.go b/federation/cmd/federated-apiserver/app/options/options.go new file mode 100644 index 0000000000000..a0ca0be05a86a --- /dev/null +++ b/federation/cmd/federated-apiserver/app/options/options.go @@ -0,0 +1,269 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package options contains flags and options for initializing an apiserver +package options + +import ( + "net" + "strings" + "time" + + "k8s.io/kubernetes/pkg/admission" + "k8s.io/kubernetes/pkg/api" + apiutil "k8s.io/kubernetes/pkg/api/util" + "k8s.io/kubernetes/pkg/api/validation" + "k8s.io/kubernetes/pkg/apimachinery/registered" + "k8s.io/kubernetes/pkg/apiserver" + "k8s.io/kubernetes/pkg/genericapiserver" + kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" + "k8s.io/kubernetes/pkg/master/ports" + etcdstorage "k8s.io/kubernetes/pkg/storage/etcd" + "k8s.io/kubernetes/pkg/util/config" + utilnet "k8s.io/kubernetes/pkg/util/net" + + "github.com/spf13/pflag" +) + +// APIServer runs a kubernetes api server. +type APIServer struct { + *genericapiserver.ServerRunOptions + APIGroupPrefix string + APIPrefix string + AdmissionControl string + AdmissionControlConfigFile string + AdvertiseAddress net.IP + AllowPrivileged bool + AuthorizationMode string + AuthorizationConfig apiserver.AuthorizationConfig + BasicAuthFile string + CloudConfigFile string + CloudProvider string + CorsAllowedOriginList []string + DeleteCollectionWorkers int + DeprecatedStorageVersion string + EnableLogsSupport bool + EnableProfiling bool + EnableWatchCache bool + EnableSwaggerUI bool + EtcdServersOverrides []string + EtcdConfig etcdstorage.EtcdConfig + EventTTL time.Duration + ExternalHost string + KeystoneURL string + KubeletConfig kubeletclient.KubeletClientConfig + KubernetesServiceNodePort int + MasterCount int + MasterServiceNamespace string + MaxConnectionBytesPerSec int64 + MinRequestTimeout int + OIDCCAFile string + OIDCClientID string + OIDCIssuerURL string + OIDCUsernameClaim string + OIDCGroupsClaim string + RuntimeConfig config.ConfigurationMap + SSHKeyfile string + SSHUser string + ServiceAccountKeyFile string + ServiceAccountLookup bool + ServiceClusterIPRange net.IPNet // TODO: make this a list + ServiceNodePortRange utilnet.PortRange + StorageVersions string + // The default values for StorageVersions. StorageVersions overrides + // these; you can change this if you want to change the defaults (e.g., + // for testing). This is not actually exposed as a flag. + DefaultStorageVersions string + TokenAuthFile string + WatchCacheSizes []string +} + +// NewAPIServer creates a new APIServer object with default parameters +func NewAPIServer() *APIServer { + s := APIServer{ + ServerRunOptions: genericapiserver.NewServerRunOptions(), + APIGroupPrefix: "/apis", + APIPrefix: "/api", + AdmissionControl: "AlwaysAdmit", + AuthorizationMode: "AlwaysAllow", + DeleteCollectionWorkers: 1, + EnableLogsSupport: true, + EtcdConfig: etcdstorage.EtcdConfig{ + Prefix: genericapiserver.DefaultEtcdPathPrefix, + }, + EventTTL: 1 * time.Hour, + MasterCount: 1, + MasterServiceNamespace: api.NamespaceDefault, + RuntimeConfig: make(config.ConfigurationMap), + StorageVersions: registered.AllPreferredGroupVersions(), + DefaultStorageVersions: registered.AllPreferredGroupVersions(), + KubeletConfig: kubeletclient.KubeletClientConfig{ + Port: ports.KubeletPort, + EnableHttps: true, + HTTPTimeout: time.Duration(5) * time.Second, + }, + } + + return &s +} + +// dest must be a map of group to groupVersion. +func gvToMap(gvList string, dest map[string]string) { + for _, gv := range strings.Split(gvList, ",") { + if gv == "" { + continue + } + // We accept two formats. "group/version" OR + // "group=group/version". The latter is used when types + // move between groups. + if !strings.Contains(gv, "=") { + dest[apiutil.GetGroup(gv)] = gv + } else { + parts := strings.SplitN(gv, "=", 2) + // TODO: error checking. + dest[parts[0]] = parts[1] + } + } +} + +// StorageGroupsToGroupVersions returns a map from group name to group version, +// computed from the s.DeprecatedStorageVersion and s.StorageVersions flags. +// TODO: can we move the whole storage version concept to the generic apiserver? +func (s *APIServer) StorageGroupsToGroupVersions() map[string]string { + storageVersionMap := map[string]string{} + if s.DeprecatedStorageVersion != "" { + storageVersionMap[""] = s.DeprecatedStorageVersion + } + + // First, get the defaults. + gvToMap(s.DefaultStorageVersions, storageVersionMap) + // Override any defaults with the user settings. + gvToMap(s.StorageVersions, storageVersionMap) + + return storageVersionMap +} + +// AddFlags adds flags for a specific APIServer to the specified FlagSet +func (s *APIServer) AddFlags(fs *pflag.FlagSet) { + // Note: the weird ""+ in below lines seems to be the only way to get gofmt to + // arrange these text blocks sensibly. Grrr. + fs.IntVar(&s.InsecurePort, "insecure-port", s.InsecurePort, ""+ + "The port on which to serve unsecured, unauthenticated access. Default 8080. It is assumed "+ + "that firewall rules are set up such that this port is not reachable from outside of "+ + "the cluster and that port 443 on the cluster's public address is proxied to this "+ + "port. This is performed by nginx in the default setup.") + fs.IntVar(&s.InsecurePort, "port", s.InsecurePort, "DEPRECATED: see --insecure-port instead") + fs.MarkDeprecated("port", "see --insecure-port instead") + fs.IPVar(&s.InsecureBindAddress, "insecure-bind-address", s.InsecureBindAddress, ""+ + "The IP address on which to serve the --insecure-port (set to 0.0.0.0 for all interfaces). "+ + "Defaults to localhost.") + fs.IPVar(&s.InsecureBindAddress, "address", s.InsecureBindAddress, "DEPRECATED: see --insecure-bind-address instead") + fs.MarkDeprecated("address", "see --insecure-bind-address instead") + fs.IPVar(&s.BindAddress, "bind-address", s.BindAddress, ""+ + "The IP address on which to listen for the --secure-port port. The "+ + "associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+ + "clients. If blank, all interfaces will be used (0.0.0.0).") + fs.IPVar(&s.AdvertiseAddress, "advertise-address", s.AdvertiseAddress, ""+ + "The IP address on which to advertise the apiserver to members of the cluster. This "+ + "address must be reachable by the rest of the cluster. If blank, the --bind-address "+ + "will be used. If --bind-address is unspecified, the host's default interface will "+ + "be used.") + fs.IPVar(&s.BindAddress, "public-address-override", s.BindAddress, "DEPRECATED: see --bind-address instead") + fs.MarkDeprecated("public-address-override", "see --bind-address instead") + fs.IntVar(&s.SecurePort, "secure-port", s.SecurePort, ""+ + "The port on which to serve HTTPS with authentication and authorization. If 0, "+ + "don't serve HTTPS at all.") + fs.StringVar(&s.TLSCertFile, "tls-cert-file", s.TLSCertFile, ""+ + "File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert). "+ + "If HTTPS serving is enabled, and --tls-cert-file and --tls-private-key-file are not provided, "+ + "a self-signed certificate and key are generated for the public address and saved to /var/run/kubernetes.") + fs.StringVar(&s.TLSPrivateKeyFile, "tls-private-key-file", s.TLSPrivateKeyFile, "File containing x509 private key matching --tls-cert-file.") + fs.StringVar(&s.CertDirectory, "cert-dir", s.CertDirectory, "The directory where the TLS certs are located (by default /var/run/kubernetes). "+ + "If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.") + fs.StringVar(&s.APIPrefix, "api-prefix", s.APIPrefix, "The prefix for API requests on the server. Default '/api'.") + fs.MarkDeprecated("api-prefix", "--api-prefix is deprecated and will be removed when the v1 API is retired.") + fs.StringVar(&s.DeprecatedStorageVersion, "storage-version", s.DeprecatedStorageVersion, "The version to store the legacy v1 resources with. Defaults to server preferred") + fs.MarkDeprecated("storage-version", "--storage-version is deprecated and will be removed when the v1 API is retired. See --storage-versions instead.") + fs.StringVar(&s.StorageVersions, "storage-versions", s.StorageVersions, "The per-group version to store resources in. "+ + "Specified in the format \"group1/version1,group2/version2,...\". "+ + "In the case where objects are moved from one group to the other, you may specify the format \"group1=group2/v1beta1,group3/v1beta1,...\". "+ + "You only need to pass the groups you wish to change from the defaults. "+ + "It defaults to a list of preferred versions of all registered groups, which is derived from the KUBE_API_VERSIONS environment variable.") + fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services. Empty string for no provider.") + fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") + fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL, "Amount of time to retain events. Default 1 hour.") + fs.StringVar(&s.BasicAuthFile, "basic-auth-file", s.BasicAuthFile, "If set, the file that will be used to admit requests to the secure port of the API server via http basic authentication.") + fs.StringVar(&s.ClientCAFile, "client-ca-file", s.ClientCAFile, "If set, any request presenting a client certificate signed by one of the authorities in the client-ca-file is authenticated with an identity corresponding to the CommonName of the client certificate.") + fs.StringVar(&s.TokenAuthFile, "token-auth-file", s.TokenAuthFile, "If set, the file that will be used to secure the secure port of the API server via token authentication.") + fs.StringVar(&s.OIDCIssuerURL, "oidc-issuer-url", s.OIDCIssuerURL, "The URL of the OpenID issuer, only HTTPS scheme will be accepted. If set, it will be used to verify the OIDC JSON Web Token (JWT)") + fs.StringVar(&s.OIDCClientID, "oidc-client-id", s.OIDCClientID, "The client ID for the OpenID Connect client, must be set if oidc-issuer-url is set") + fs.StringVar(&s.OIDCCAFile, "oidc-ca-file", s.OIDCCAFile, "If set, the OpenID server's certificate will be verified by one of the authorities in the oidc-ca-file, otherwise the host's root CA set will be used") + fs.StringVar(&s.OIDCUsernameClaim, "oidc-username-claim", "sub", ""+ + "The OpenID claim to use as the user name. Note that claims other than the default ('sub') is not "+ + "guaranteed to be unique and immutable. This flag is experimental, please see the authentication documentation for further details.") + fs.StringVar(&s.OIDCGroupsClaim, "oidc-groups-claim", "", "If provided, the name of a custom OpenID Connect claim for specifying user groups. The claim value is expected to be an array of strings. This flag is experimental, please see the authentication documentation for further details.") + fs.StringVar(&s.ServiceAccountKeyFile, "service-account-key-file", s.ServiceAccountKeyFile, "File containing PEM-encoded x509 RSA private or public key, used to verify ServiceAccount tokens. If unspecified, --tls-private-key-file is used.") + fs.BoolVar(&s.ServiceAccountLookup, "service-account-lookup", s.ServiceAccountLookup, "If true, validate ServiceAccount tokens exist in etcd as part of authentication.") + fs.StringVar(&s.KeystoneURL, "experimental-keystone-url", s.KeystoneURL, "If passed, activates the keystone authentication plugin") + fs.StringVar(&s.AuthorizationMode, "authorization-mode", s.AuthorizationMode, "Ordered list of plug-ins to do authorization on secure port. Comma-delimited list of: "+strings.Join(apiserver.AuthorizationModeChoices, ",")) + fs.StringVar(&s.AuthorizationConfig.PolicyFile, "authorization-policy-file", s.AuthorizationConfig.PolicyFile, "File with authorization policy in csv format, used with --authorization-mode=ABAC, on the secure port.") + fs.StringVar(&s.AuthorizationConfig.WebhookConfigFile, "authorization-webhook-config-file", s.AuthorizationConfig.WebhookConfigFile, "File with webhook configuration in kubeconfig format, used with --authorization-mode=Webhook. The API server will query the remote service to determine access on the API server's secure port.") + fs.StringVar(&s.AdmissionControl, "admission-control", s.AdmissionControl, "Ordered list of plug-ins to do admission control of resources into cluster. Comma-delimited list of: "+strings.Join(admission.GetPlugins(), ", ")) + fs.StringVar(&s.AdmissionControlConfigFile, "admission-control-config-file", s.AdmissionControlConfigFile, "File with admission control configuration.") + fs.StringSliceVar(&s.EtcdConfig.ServerList, "etcd-servers", s.EtcdConfig.ServerList, "List of etcd servers to watch (http://ip:port), comma separated.") + fs.StringSliceVar(&s.EtcdServersOverrides, "etcd-servers-overrides", s.EtcdServersOverrides, "Per-resource etcd servers overrides, comma separated. The individual override format: group/resource#servers, where servers are http://ip:port, semicolon separated.") + fs.StringVar(&s.EtcdConfig.Prefix, "etcd-prefix", s.EtcdConfig.Prefix, "The prefix for all resource paths in etcd.") + fs.StringVar(&s.EtcdConfig.KeyFile, "etcd-keyfile", s.EtcdConfig.KeyFile, "SSL key file used to secure etcd communication") + fs.StringVar(&s.EtcdConfig.CertFile, "etcd-certfile", s.EtcdConfig.CertFile, "SSL certification file used to secure etcd communication") + fs.StringVar(&s.EtcdConfig.CAFile, "etcd-cafile", s.EtcdConfig.CAFile, "SSL Certificate Authority file used to secure etcd communication") + fs.BoolVar(&s.EtcdConfig.Quorum, "etcd-quorum-read", s.EtcdConfig.Quorum, "If true, enable quorum read") + fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, "List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. If this list is empty CORS will not be enabled.") + fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged, "If true, allow privileged containers.") + fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, "A CIDR notation IP range from which to assign service cluster IPs. This must not overlap with any IP ranges assigned to nodes for pods.") + fs.IPNetVar(&s.ServiceClusterIPRange, "portal-net", s.ServiceClusterIPRange, "Deprecated: see --service-cluster-ip-range instead.") + fs.MarkDeprecated("portal-net", "see --service-cluster-ip-range instead.") + fs.Var(&s.ServiceNodePortRange, "service-node-port-range", "A port range to reserve for services with NodePort visibility. Example: '30000-32767'. Inclusive at both ends of the range.") + fs.Var(&s.ServiceNodePortRange, "service-node-ports", "Deprecated: see --service-node-port-range instead.") + fs.MarkDeprecated("service-node-ports", "see --service-node-port-range instead.") + fs.StringVar(&s.MasterServiceNamespace, "master-service-namespace", s.MasterServiceNamespace, "The namespace from which the kubernetes master services should be injected into pods") + fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount, "The number of apiservers running in the cluster") + fs.IntVar(&s.DeleteCollectionWorkers, "delete-collection-workers", s.DeleteCollectionWorkers, "Number of workers spawned for DeleteCollection call. These are used to speed up namespace cleanup.") + fs.Var(&s.RuntimeConfig, "runtime-config", "A set of key=value pairs that describe runtime configuration that may be passed to apiserver. apis/ key can be used to turn on/off specific api versions. apis// can be used to turn on/off specific resources. api/all and api/legacy are special keys to control all and legacy api versions respectively.") + fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/") + // TODO: enable cache in integration tests. + fs.BoolVar(&s.EnableWatchCache, "watch-cache", true, "Enable watch caching in the apiserver") + fs.BoolVar(&s.EnableSwaggerUI, "enable-swagger-ui", false, "Enables swagger ui on the apiserver at /swagger-ui") + fs.StringVar(&s.ExternalHost, "external-hostname", "", "The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs.)") + fs.IntVar(&s.MaxRequestsInFlight, "max-requests-inflight", 400, "The maximum number of requests in flight at a given time. When the server exceeds this, it rejects requests. Zero for no limit.") + fs.IntVar(&s.MinRequestTimeout, "min-request-timeout", 1800, "An optional field indicating the minimum number of seconds a handler must keep a request open before timing it out. Currently only honored by the watch request handler, which picks a randomized value above this number as the connection timeout, to spread out load.") + fs.StringVar(&s.LongRunningRequestRE, "long-running-request-regexp", s.LongRunningRequestRE, "A regular expression matching long running requests which should be excluded from maximum inflight request handling.") + fs.StringVar(&s.SSHUser, "ssh-user", "", "If non-empty, use secure SSH proxy to the nodes, using this user name") + fs.StringVar(&s.SSHKeyfile, "ssh-keyfile", "", "If non-empty, use secure SSH proxy to the nodes, using this user keyfile") + fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", 0, "If non-zero, throttle each user connection to this number of bytes/sec. Currently only applies to long-running requests") + // Kubelet related flags: + fs.BoolVar(&s.KubeletConfig.EnableHttps, "kubelet-https", s.KubeletConfig.EnableHttps, "Use https for kubelet connections") + fs.UintVar(&s.KubeletConfig.Port, "kubelet-port", s.KubeletConfig.Port, "Kubelet port") + fs.MarkDeprecated("kubelet-port", "kubelet-port is deprecated and will be removed") + fs.DurationVar(&s.KubeletConfig.HTTPTimeout, "kubelet-timeout", s.KubeletConfig.HTTPTimeout, "Timeout for kubelet operations") + fs.StringVar(&s.KubeletConfig.CertFile, "kubelet-client-certificate", s.KubeletConfig.CertFile, "Path to a client cert file for TLS.") + fs.StringVar(&s.KubeletConfig.KeyFile, "kubelet-client-key", s.KubeletConfig.KeyFile, "Path to a client key file for TLS.") + fs.StringVar(&s.KubeletConfig.CAFile, "kubelet-certificate-authority", s.KubeletConfig.CAFile, "Path to a cert. file for the certificate authority.") + // See #14282 for details on how to test/try this option out. TODO remove this comment once this option is tested in CI. + fs.IntVar(&s.KubernetesServiceNodePort, "kubernetes-service-node-port", 0, "If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be of type NodePort, using this as the value of the port. If zero, the Kubernetes master service will be of type ClusterIP.") + // TODO: delete this flag as soon as we identify and fix all clients that send malformed updates, like #14126. + fs.BoolVar(&validation.RepairMalformedUpdates, "repair-malformed-updates", true, "If true, server will do its best to fix the update request to pass the validation, e.g., setting empty UID in update request to its existing value. This flag can be turned off after we fix all the clients that send malformed updates.") + fs.StringSliceVar(&s.WatchCacheSizes, "watch-cache-sizes", s.WatchCacheSizes, "List of watch cache sizes for every resource (pods, nodes, etc.), comma separated. The individual override format: resource#size, where size is a number. It takes effect when watch-cache is enabled.") +} diff --git a/federation/cmd/federated-apiserver/app/options/options_test.go b/federation/cmd/federated-apiserver/app/options/options_test.go new file mode 100644 index 0000000000000..85f3a77ac2c0f --- /dev/null +++ b/federation/cmd/federated-apiserver/app/options/options_test.go @@ -0,0 +1,99 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "reflect" + "testing" + + "github.com/spf13/pflag" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apis/autoscaling" + "k8s.io/kubernetes/pkg/apis/extensions" +) + +func TestGenerateStorageVersionMap(t *testing.T) { + testCases := []struct { + legacyVersion string + storageVersions string + defaultVersions string + expectedMap map[string]string + }{ + { + legacyVersion: "v1", + storageVersions: "v1,extensions/v1beta1", + expectedMap: map[string]string{ + api.GroupName: "v1", + extensions.GroupName: "extensions/v1beta1", + }, + }, + { + legacyVersion: "", + storageVersions: "extensions/v1beta1,v1", + expectedMap: map[string]string{ + api.GroupName: "v1", + extensions.GroupName: "extensions/v1beta1", + }, + }, + { + legacyVersion: "", + storageVersions: "autoscaling=extensions/v1beta1,v1", + defaultVersions: "extensions/v1beta1,v1,autoscaling/v1", + expectedMap: map[string]string{ + api.GroupName: "v1", + autoscaling.GroupName: "extensions/v1beta1", + extensions.GroupName: "extensions/v1beta1", + }, + }, + { + legacyVersion: "", + storageVersions: "", + expectedMap: map[string]string{}, + }, + } + for i, test := range testCases { + s := APIServer{ + DeprecatedStorageVersion: test.legacyVersion, + StorageVersions: test.storageVersions, + DefaultStorageVersions: test.defaultVersions, + } + output := s.StorageGroupsToGroupVersions() + if !reflect.DeepEqual(test.expectedMap, output) { + t.Errorf("%v: unexpected error. expect: %v, got: %v", i, test.expectedMap, output) + } + } +} + +func TestAddFlagsFlag(t *testing.T) { + // TODO: This only tests the enable-swagger-ui flag for now. + // Expand the test to include other flags as well. + f := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError) + s := NewAPIServer() + s.AddFlags(f) + if s.EnableSwaggerUI { + t.Errorf("Expected s.EnableSwaggerUI to be false by default") + } + + args := []string{ + "--enable-swagger-ui=true", + } + f.Parse(args) + if !s.EnableSwaggerUI { + t.Errorf("Expected s.EnableSwaggerUI to be true") + } +} diff --git a/federation/cmd/federated-apiserver/app/plugins.go b/federation/cmd/federated-apiserver/app/plugins.go new file mode 100644 index 0000000000000..f4742fc46aee0 --- /dev/null +++ b/federation/cmd/federated-apiserver/app/plugins.go @@ -0,0 +1,40 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +// This file exists to force the desired plugin implementations to be linked. +// This should probably be part of some configuration fed into the build for a +// given binary target. +import ( + // Cloud providers + _ "k8s.io/kubernetes/pkg/cloudprovider/providers" + + // Admission policies + _ "k8s.io/kubernetes/plugin/pkg/admission/admit" + _ "k8s.io/kubernetes/plugin/pkg/admission/alwayspullimages" + _ "k8s.io/kubernetes/plugin/pkg/admission/deny" + _ "k8s.io/kubernetes/plugin/pkg/admission/exec" + _ "k8s.io/kubernetes/plugin/pkg/admission/initialresources" + _ "k8s.io/kubernetes/plugin/pkg/admission/limitranger" + _ "k8s.io/kubernetes/plugin/pkg/admission/namespace/autoprovision" + _ "k8s.io/kubernetes/plugin/pkg/admission/namespace/exists" + _ "k8s.io/kubernetes/plugin/pkg/admission/namespace/lifecycle" + _ "k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/label" + _ "k8s.io/kubernetes/plugin/pkg/admission/resourcequota" + _ "k8s.io/kubernetes/plugin/pkg/admission/securitycontext/scdeny" + _ "k8s.io/kubernetes/plugin/pkg/admission/serviceaccount" +) diff --git a/federation/cmd/federated-apiserver/app/server.go b/federation/cmd/federated-apiserver/app/server.go new file mode 100644 index 0000000000000..13e27e8cc06b7 --- /dev/null +++ b/federation/cmd/federated-apiserver/app/server.go @@ -0,0 +1,582 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package app does all of the work necessary to create a Kubernetes +// APIServer by binding together the API, master and APIServer infrastructure. +// It can be configured and called directly or via the hyperkube framework. +package app + +import ( + "crypto/tls" + "fmt" + "net" + "net/url" + "os" + "strconv" + "strings" + + "github.com/golang/glog" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/pkg/admission" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/unversioned" + apiv1 "k8s.io/kubernetes/pkg/api/v1" + "k8s.io/kubernetes/pkg/apimachinery/registered" + "k8s.io/kubernetes/pkg/apis/autoscaling" + autoscalingapiv1 "k8s.io/kubernetes/pkg/apis/autoscaling/v1" + "k8s.io/kubernetes/pkg/apis/batch" + batchapiv1 "k8s.io/kubernetes/pkg/apis/batch/v1" + "k8s.io/kubernetes/pkg/apis/extensions" + extensionsapiv1beta1 "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" + "k8s.io/kubernetes/pkg/apiserver" + "k8s.io/kubernetes/pkg/apiserver/authenticator" + "k8s.io/kubernetes/pkg/capabilities" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/client/restclient" + "k8s.io/kubernetes/pkg/cloudprovider" + serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" + "k8s.io/kubernetes/pkg/genericapiserver" + kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" + "k8s.io/kubernetes/pkg/master" + "k8s.io/kubernetes/pkg/registry/cachesize" + "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/runtime/serializer/versioning" + "k8s.io/kubernetes/pkg/serviceaccount" + "k8s.io/kubernetes/pkg/storage" + etcdstorage "k8s.io/kubernetes/pkg/storage/etcd" + utilnet "k8s.io/kubernetes/pkg/util/net" +) + +// NewAPIServerCommand creates a *cobra.Command object with default parameters +func NewAPIServerCommand() *cobra.Command { + s := options.NewAPIServer() + s.AddFlags(pflag.CommandLine) + cmd := &cobra.Command{ + Use: "kube-apiserver", + Long: `The Kubernetes API server validates and configures data +for the api objects which include pods, services, replicationcontrollers, and +others. The API Server services REST operations and provides the frontend to the +cluster's shared state through which all other components interact.`, + Run: func(cmd *cobra.Command, args []string) { + }, + } + + return cmd +} + +// TODO: Longer term we should read this from some config store, rather than a flag. +func verifyClusterIPFlags(s *options.APIServer) { + if s.ServiceClusterIPRange.IP == nil { + glog.Fatal("No --service-cluster-ip-range specified") + } + var ones, bits = s.ServiceClusterIPRange.Mask.Size() + if bits-ones > 20 { + glog.Fatal("Specified --service-cluster-ip-range is too large") + } +} + +// For testing. +type newEtcdFunc func(runtime.NegotiatedSerializer, string, string, etcdstorage.EtcdConfig) (storage.Interface, error) + +func newEtcd(ns runtime.NegotiatedSerializer, storageGroupVersionString, memoryGroupVersionString string, etcdConfig etcdstorage.EtcdConfig) (etcdStorage storage.Interface, err error) { + if storageGroupVersionString == "" { + return etcdStorage, fmt.Errorf("storageVersion is required to create a etcd storage") + } + storageVersion, err := unversioned.ParseGroupVersion(storageGroupVersionString) + if err != nil { + return nil, fmt.Errorf("couldn't understand storage version %v: %v", storageGroupVersionString, err) + } + memoryVersion, err := unversioned.ParseGroupVersion(memoryGroupVersionString) + if err != nil { + return nil, fmt.Errorf("couldn't understand memory version %v: %v", memoryGroupVersionString, err) + } + + var storageConfig etcdstorage.EtcdStorageConfig + storageConfig.Config = etcdConfig + s, ok := ns.SerializerForMediaType("application/json", nil) + if !ok { + return nil, fmt.Errorf("unable to find serializer for JSON") + } + glog.Infof("constructing etcd storage interface.\n sv: %v\n mv: %v\n", storageVersion, memoryVersion) + encoder := ns.EncoderForVersion(s, storageVersion) + decoder := ns.DecoderToVersion(s, memoryVersion) + if memoryVersion.Group != storageVersion.Group { + // Allow this codec to translate between groups. + if err = versioning.EnableCrossGroupEncoding(encoder, memoryVersion.Group, storageVersion.Group); err != nil { + return nil, fmt.Errorf("error setting up encoder for %v: %v", storageGroupVersionString, err) + } + if err = versioning.EnableCrossGroupDecoding(decoder, storageVersion.Group, memoryVersion.Group); err != nil { + return nil, fmt.Errorf("error setting up decoder for %v: %v", storageGroupVersionString, err) + } + } + storageConfig.Codec = runtime.NewCodec(encoder, decoder) + return storageConfig.NewStorage() +} + +// parse the value of --etcd-servers-overrides and update given storageDestinations. +func updateEtcdOverrides(overrides []string, storageVersions map[string]string, etcdConfig etcdstorage.EtcdConfig, storageDestinations *genericapiserver.StorageDestinations, newEtcdFn newEtcdFunc) { + if len(overrides) == 0 { + return + } + for _, override := range overrides { + tokens := strings.Split(override, "#") + if len(tokens) != 2 { + glog.Errorf("invalid value of etcd server overrides: %s", override) + continue + } + + apiresource := strings.Split(tokens[0], "/") + if len(apiresource) != 2 { + glog.Errorf("invalid resource definition: %s", tokens[0]) + } + group := apiresource[0] + resource := apiresource[1] + + apigroup, err := registered.Group(group) + if err != nil { + glog.Errorf("invalid api group %s: %v", group, err) + continue + } + if _, found := storageVersions[apigroup.GroupVersion.Group]; !found { + glog.Errorf("Couldn't find the storage version for group %s", apigroup.GroupVersion.Group) + continue + } + + servers := strings.Split(tokens[1], ";") + overrideEtcdConfig := etcdConfig + overrideEtcdConfig.ServerList = servers + // Note, internalGV will be wrong for things like batch or + // autoscalers, but they shouldn't be using the override + // storage. + internalGV := apigroup.GroupVersion.Group + "/__internal" + etcdOverrideStorage, err := newEtcdFn(api.Codecs, storageVersions[apigroup.GroupVersion.Group], internalGV, overrideEtcdConfig) + if err != nil { + glog.Fatalf("Invalid storage version or misconfigured etcd for %s: %v", tokens[0], err) + } + + storageDestinations.AddStorageOverride(group, resource, etcdOverrideStorage) + } +} + +// Run runs the specified APIServer. This should never exit. +func Run(s *options.APIServer) error { + verifyClusterIPFlags(s) + + // If advertise-address is not specified, use bind-address. If bind-address + // is not usable (unset, 0.0.0.0, or loopback), we will use the host's default + // interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster) + if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() { + hostIP, err := utilnet.ChooseBindAddress(s.BindAddress) + if err != nil { + glog.Fatalf("Unable to find suitable network address.error='%v' . "+ + "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) + } + s.AdvertiseAddress = hostIP + } + glog.Infof("Will report %v as public IP address.", s.AdvertiseAddress) + + if len(s.EtcdConfig.ServerList) == 0 { + glog.Fatalf("--etcd-servers must be specified") + } + + if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) { + glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort)) + } + + capabilities.Initialize(capabilities.Capabilities{ + AllowPrivileged: s.AllowPrivileged, + // TODO(vmarmol): Implement support for HostNetworkSources. + PrivilegedSources: capabilities.PrivilegedSources{ + HostNetworkSources: []string{}, + HostPIDSources: []string{}, + HostIPCSources: []string{}, + }, + PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, + }) + + cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) + if err != nil { + glog.Fatalf("Cloud provider could not be initialized: %v", err) + } + + // Setup tunneler if needed + var tunneler master.Tunneler + var proxyDialerFn apiserver.ProxyDialerFunc + if len(s.SSHUser) > 0 { + // Get ssh key distribution func, if supported + var installSSH master.InstallSSHKey + if cloud != nil { + if instances, supported := cloud.Instances(); supported { + installSSH = instances.AddSSHKeyToAllInstances + } + } + if s.KubeletConfig.Port == 0 { + glog.Fatalf("Must enable kubelet port if proxy ssh-tunneling is specified.") + } + // Set up the tunneler + // TODO(cjcullen): If we want this to handle per-kubelet ports or other + // kubelet listen-addresses, we need to plumb through options. + healthCheckPath := &url.URL{ + Scheme: "https", + Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.Port), 10)), + Path: "healthz", + } + tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSH) + + // Use the tunneler's dialer to connect to the kubelet + s.KubeletConfig.Dial = tunneler.Dial + // Use the tunneler's dialer when proxying to pods, services, and nodes + proxyDialerFn = tunneler.Dial + } + + // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname + proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} + + kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig) + if err != nil { + glog.Fatalf("Failure to start kubelet client: %v", err) + } + + apiResourceConfigSource, err := parseRuntimeConfig(s) + if err != nil { + glog.Fatalf("error in parsing runtime-config: %s", err) + } + + clientConfig := &restclient.Config{ + Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)), + // Increase QPS limits. The client is currently passed to all admission plugins, + // and those can be throttled in case of higher load on apiserver - see #22340 and #22422 + // for more details. Once #22422 is fixed, we may want to remove it. + QPS: 50, + Burst: 100, + } + if len(s.DeprecatedStorageVersion) != 0 { + gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion) + if err != nil { + glog.Fatalf("error in parsing group version: %s", err) + } + clientConfig.GroupVersion = &gv + } + + client, err := clientset.NewForConfig(clientConfig) + if err != nil { + glog.Errorf("Failed to create clientset: %v", err) + } + + legacyV1Group, err := registered.Group(api.GroupName) + if err != nil { + return err + } + + storageDestinations := genericapiserver.NewStorageDestinations() + + storageVersions := s.StorageGroupsToGroupVersions() + if _, found := storageVersions[legacyV1Group.GroupVersion.Group]; !found { + glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.GroupVersion.Group, storageVersions) + } + etcdStorage, err := newEtcd(api.Codecs, storageVersions[legacyV1Group.GroupVersion.Group], "/__internal", s.EtcdConfig) + if err != nil { + glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err) + } + storageDestinations.AddAPIGroup("", etcdStorage) + + if apiResourceConfigSource.AnyResourcesForVersionEnabled(extensionsapiv1beta1.SchemeGroupVersion) { + glog.Infof("Configuring extensions/v1beta1 storage destination") + expGroup, err := registered.Group(extensions.GroupName) + if err != nil { + glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) + } + if _, found := storageVersions[expGroup.GroupVersion.Group]; !found { + glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.GroupVersion.Group, storageVersions) + } + expEtcdStorage, err := newEtcd(api.Codecs, storageVersions[expGroup.GroupVersion.Group], "extensions/__internal", s.EtcdConfig) + if err != nil { + glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) + } + storageDestinations.AddAPIGroup(extensions.GroupName, expEtcdStorage) + + // Since HPA has been moved to the autoscaling group, we need to make + // sure autoscaling has a storage destination. If the autoscaling group + // itself is on, it will overwrite this decision below. + storageDestinations.AddAPIGroup(autoscaling.GroupName, expEtcdStorage) + + // Since Job has been moved to the batch group, we need to make + // sure batch has a storage destination. If the batch group + // itself is on, it will overwrite this decision below. + storageDestinations.AddAPIGroup(batch.GroupName, expEtcdStorage) + } + + // autoscaling/v1/horizontalpodautoscalers is a move from extensions/v1beta1/horizontalpodautoscalers. + // The storage version needs to be either extensions/v1beta1 or autoscaling/v1. + // Users must roll forward while using 1.2, because we will require the latter for 1.3. + if apiResourceConfigSource.AnyResourcesForVersionEnabled(autoscalingapiv1.SchemeGroupVersion) { + glog.Infof("Configuring autoscaling/v1 storage destination") + autoscalingGroup, err := registered.Group(autoscaling.GroupName) + if err != nil { + glog.Fatalf("Autoscaling API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) + } + // Figure out what storage group/version we should use. + storageGroupVersion, found := storageVersions[autoscalingGroup.GroupVersion.Group] + if !found { + glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", autoscalingGroup.GroupVersion.Group, storageVersions) + } + + if storageGroupVersion != "autoscaling/v1" && storageGroupVersion != "extensions/v1beta1" { + glog.Fatalf("The storage version for autoscaling must be either 'autoscaling/v1' or 'extensions/v1beta1'") + } + glog.Infof("Using %v for autoscaling group storage version", storageGroupVersion) + autoscalingEtcdStorage, err := newEtcd(api.Codecs, storageGroupVersion, "extensions/__internal", s.EtcdConfig) + if err != nil { + glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) + } + storageDestinations.AddAPIGroup(autoscaling.GroupName, autoscalingEtcdStorage) + } + + // batch/v1/job is a move from extensions/v1beta1/job. The storage + // version needs to be either extensions/v1beta1 or batch/v1. Users + // must roll forward while using 1.2, because we will require the + // latter for 1.3. + if apiResourceConfigSource.AnyResourcesForVersionEnabled(batchapiv1.SchemeGroupVersion) { + glog.Infof("Configuring batch/v1 storage destination") + batchGroup, err := registered.Group(batch.GroupName) + if err != nil { + glog.Fatalf("Batch API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err) + } + // Figure out what storage group/version we should use. + storageGroupVersion, found := storageVersions[batchGroup.GroupVersion.Group] + if !found { + glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", batchGroup.GroupVersion.Group, storageVersions) + } + + if storageGroupVersion != "batch/v1" && storageGroupVersion != "extensions/v1beta1" { + glog.Fatalf("The storage version for batch must be either 'batch/v1' or 'extensions/v1beta1'") + } + glog.Infof("Using %v for batch group storage version", storageGroupVersion) + batchEtcdStorage, err := newEtcd(api.Codecs, storageGroupVersion, "extensions/__internal", s.EtcdConfig) + if err != nil { + glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err) + } + storageDestinations.AddAPIGroup(batch.GroupName, batchEtcdStorage) + } + + updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdConfig, &storageDestinations, newEtcd) + + n := s.ServiceClusterIPRange + + // Default to the private server key for service account token signing + if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" { + if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) { + s.ServiceAccountKeyFile = s.TLSPrivateKeyFile + } else { + glog.Warning("No RSA key provided, service account token authentication disabled") + } + } + + var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter + if s.ServiceAccountLookup { + // If we need to look up service accounts and tokens, + // go directly to etcd to avoid recursive auth insanity + serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(etcdStorage) + } + + authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{ + BasicAuthFile: s.BasicAuthFile, + ClientCAFile: s.ClientCAFile, + TokenAuthFile: s.TokenAuthFile, + OIDCIssuerURL: s.OIDCIssuerURL, + OIDCClientID: s.OIDCClientID, + OIDCCAFile: s.OIDCCAFile, + OIDCUsernameClaim: s.OIDCUsernameClaim, + OIDCGroupsClaim: s.OIDCGroupsClaim, + ServiceAccountKeyFile: s.ServiceAccountKeyFile, + ServiceAccountLookup: s.ServiceAccountLookup, + ServiceAccountTokenGetter: serviceAccountGetter, + KeystoneURL: s.KeystoneURL, + }) + + if err != nil { + glog.Fatalf("Invalid Authentication Config: %v", err) + } + + authorizationModeNames := strings.Split(s.AuthorizationMode, ",") + authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationConfig) + if err != nil { + glog.Fatalf("Invalid Authorization Config: %v", err) + } + + admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") + admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile) + + if len(s.ExternalHost) == 0 { + // TODO: extend for other providers + if s.CloudProvider == "gce" { + instances, supported := cloud.Instances() + if !supported { + glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.") + } + name, err := os.Hostname() + if err != nil { + glog.Fatalf("Failed to get hostname: %v", err) + } + addrs, err := instances.NodeAddresses(name) + if err != nil { + glog.Warningf("Unable to obtain external host address from cloud provider: %v", err) + } else { + for _, addr := range addrs { + if addr.Type == api.NodeExternalIP { + s.ExternalHost = addr.Address + } + } + } + } + } + + config := &master.Config{ + Config: &genericapiserver.Config{ + StorageDestinations: storageDestinations, + StorageVersions: storageVersions, + ServiceClusterIPRange: &n, + EnableLogsSupport: s.EnableLogsSupport, + EnableUISupport: true, + EnableSwaggerSupport: true, + EnableSwaggerUI: s.EnableSwaggerUI, + EnableProfiling: s.EnableProfiling, + EnableWatchCache: s.EnableWatchCache, + EnableIndex: true, + APIPrefix: s.APIPrefix, + APIGroupPrefix: s.APIGroupPrefix, + CorsAllowedOriginList: s.CorsAllowedOriginList, + ReadWritePort: s.SecurePort, + PublicAddress: s.AdvertiseAddress, + Authenticator: authenticator, + SupportsBasicAuth: len(s.BasicAuthFile) > 0, + Authorizer: authorizer, + AdmissionControl: admissionController, + APIResourceConfigSource: apiResourceConfigSource, + MasterServiceNamespace: s.MasterServiceNamespace, + MasterCount: s.MasterCount, + ExternalHost: s.ExternalHost, + MinRequestTimeout: s.MinRequestTimeout, + ProxyDialer: proxyDialerFn, + ProxyTLSClientConfig: proxyTLSClientConfig, + ServiceNodePortRange: s.ServiceNodePortRange, + KubernetesServiceNodePort: s.KubernetesServiceNodePort, + Serializer: api.Codecs, + }, + EnableCoreControllers: true, + DeleteCollectionWorkers: s.DeleteCollectionWorkers, + EventTTL: s.EventTTL, + KubeletClient: kubeletClient, + + Tunneler: tunneler, + } + + if s.EnableWatchCache { + cachesize.SetWatchCacheSizes(s.WatchCacheSizes) + } + + m, err := master.New(config) + if err != nil { + return err + } + + m.Run(s.ServerRunOptions) + return nil +} + +func getRuntimeConfigValue(s *options.APIServer, apiKey string, defaultValue bool) bool { + flagValue, ok := s.RuntimeConfig[apiKey] + if ok { + if flagValue == "" { + return true + } + boolValue, err := strconv.ParseBool(flagValue) + if err != nil { + glog.Fatalf("Invalid value of %s: %s, err: %v", apiKey, flagValue, err) + } + return boolValue + } + return defaultValue +} + +// Parses the given runtime-config and formats it into genericapiserver.APIResourceConfigSource +func parseRuntimeConfig(s *options.APIServer) (genericapiserver.APIResourceConfigSource, error) { + v1GroupVersionString := "api/v1" + extensionsGroupVersionString := extensionsapiv1beta1.SchemeGroupVersion.String() + versionToResourceSpecifier := map[unversioned.GroupVersion]string{ + apiv1.SchemeGroupVersion: v1GroupVersionString, + extensionsapiv1beta1.SchemeGroupVersion: extensionsGroupVersionString, + batchapiv1.SchemeGroupVersion: batchapiv1.SchemeGroupVersion.String(), + autoscalingapiv1.SchemeGroupVersion: autoscalingapiv1.SchemeGroupVersion.String(), + } + + resourceConfig := master.DefaultAPIResourceConfigSource() + + // "api/all=false" allows users to selectively enable specific api versions. + enableAPIByDefault := true + allAPIFlagValue, ok := s.RuntimeConfig["api/all"] + if ok && allAPIFlagValue == "false" { + enableAPIByDefault = false + } + + // "api/legacy=false" allows users to disable legacy api versions. + disableLegacyAPIs := false + legacyAPIFlagValue, ok := s.RuntimeConfig["api/legacy"] + if ok && legacyAPIFlagValue == "false" { + disableLegacyAPIs = true + } + _ = disableLegacyAPIs // hush the compiler while we don't have legacy APIs to disable. + + // "={true|false} allows users to enable/disable API. + // This takes preference over api/all and api/legacy, if specified. + for version, resourceSpecifier := range versionToResourceSpecifier { + enableVersion := getRuntimeConfigValue(s, resourceSpecifier, enableAPIByDefault) + if enableVersion { + resourceConfig.EnableVersions(version) + } else { + resourceConfig.DisableVersions(version) + } + } + + for key := range s.RuntimeConfig { + tokens := strings.Split(key, "/") + if len(tokens) != 3 { + continue + } + + switch { + case strings.HasPrefix(key, extensionsGroupVersionString+"/"): + if !resourceConfig.AnyResourcesForVersionEnabled(extensionsapiv1beta1.SchemeGroupVersion) { + return nil, fmt.Errorf("%v is disabled, you cannot configure its resources individually", extensionsapiv1beta1.SchemeGroupVersion) + } + + resource := strings.TrimPrefix(key, extensionsGroupVersionString+"/") + if getRuntimeConfigValue(s, key, false) { + resourceConfig.EnableResources(extensionsapiv1beta1.SchemeGroupVersion.WithResource(resource)) + } else { + resourceConfig.DisableResources(extensionsapiv1beta1.SchemeGroupVersion.WithResource(resource)) + } + + default: + // TODO enable individual resource capability for all GroupVersionResources + return nil, fmt.Errorf("%v resources cannot be enabled/disabled individually", key) + } + } + return resourceConfig, nil +} diff --git a/federation/cmd/federated-apiserver/app/server_test.go b/federation/cmd/federated-apiserver/app/server_test.go new file mode 100644 index 0000000000000..34a35aa755aa6 --- /dev/null +++ b/federation/cmd/federated-apiserver/app/server_test.go @@ -0,0 +1,227 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "reflect" + "regexp" + "strings" + "testing" + + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/apis/extensions" + "k8s.io/kubernetes/pkg/genericapiserver" + "k8s.io/kubernetes/pkg/master" + "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/storage" + etcdstorage "k8s.io/kubernetes/pkg/storage/etcd" +) + +func TestLongRunningRequestRegexp(t *testing.T) { + regexp := regexp.MustCompile(options.NewAPIServer().LongRunningRequestRE) + dontMatch := []string{ + "/api/v1/watch-namespace/", + "/api/v1/namespace-proxy/", + "/api/v1/namespace-watch", + "/api/v1/namespace-proxy", + "/api/v1/namespace-portforward/pods", + "/api/v1/portforward/pods", + ". anything", + "/ that", + } + doMatch := []string{ + "/api/v1/pods/watch", + "/api/v1/watch/stuff", + "/api/v1/default/service/proxy", + "/api/v1/pods/proxy/path/to/thing", + "/api/v1/namespaces/myns/pods/mypod/log", + "/api/v1/namespaces/myns/pods/mypod/logs", + "/api/v1/namespaces/myns/pods/mypod/portforward", + "/api/v1/namespaces/myns/pods/mypod/exec", + "/api/v1/namespaces/myns/pods/mypod/attach", + "/api/v1/namespaces/myns/pods/mypod/log/", + "/api/v1/namespaces/myns/pods/mypod/logs/", + "/api/v1/namespaces/myns/pods/mypod/portforward/", + "/api/v1/namespaces/myns/pods/mypod/exec/", + "/api/v1/namespaces/myns/pods/mypod/attach/", + "/api/v1/watch/namespaces/myns/pods", + } + for _, path := range dontMatch { + if regexp.MatchString(path) { + t.Errorf("path should not have match regexp but did: %s", path) + } + } + for _, path := range doMatch { + if !regexp.MatchString(path) { + t.Errorf("path should have match regexp did not: %s", path) + } + } +} + +func TestUpdateEtcdOverrides(t *testing.T) { + storageVersions := map[string]string{ + "": "v1", + "extensions": "extensions/v1beta1", + } + + testCases := []struct { + apigroup string + resource string + servers []string + }{ + { + apigroup: api.GroupName, + resource: "resource", + servers: []string{"http://127.0.0.1:10000"}, + }, + { + apigroup: api.GroupName, + resource: "resource", + servers: []string{"http://127.0.0.1:10000", "http://127.0.0.1:20000"}, + }, + { + apigroup: extensions.GroupName, + resource: "resource", + servers: []string{"http://127.0.0.1:10000"}, + }, + } + + for _, test := range testCases { + newEtcd := func(_ runtime.NegotiatedSerializer, _, _ string, etcdConfig etcdstorage.EtcdConfig) (storage.Interface, error) { + if !reflect.DeepEqual(test.servers, etcdConfig.ServerList) { + t.Errorf("unexpected server list, expected: %#v, got: %#v", test.servers, etcdConfig.ServerList) + } + return nil, nil + } + storageDestinations := genericapiserver.NewStorageDestinations() + override := test.apigroup + "/" + test.resource + "#" + strings.Join(test.servers, ";") + defaultEtcdConfig := etcdstorage.EtcdConfig{ + Prefix: genericapiserver.DefaultEtcdPathPrefix, + ServerList: []string{"http://127.0.0.1"}, + } + updateEtcdOverrides([]string{override}, storageVersions, defaultEtcdConfig, &storageDestinations, newEtcd) + apigroup, ok := storageDestinations.APIGroups[test.apigroup] + if !ok { + t.Errorf("apigroup: %s not created", test.apigroup) + continue + } + if apigroup.Overrides == nil { + t.Errorf("Overrides not created for: %s", test.apigroup) + continue + } + if _, ok := apigroup.Overrides[test.resource]; !ok { + t.Errorf("override not created for: %s", test.resource) + continue + } + } +} + +func TestParseRuntimeConfig(t *testing.T) { + testCases := []struct { + runtimeConfig map[string]string + expectedAPIConfig func() *genericapiserver.ResourceConfig + err bool + }{ + { + runtimeConfig: map[string]string{}, + expectedAPIConfig: func() *genericapiserver.ResourceConfig { + return master.DefaultAPIResourceConfigSource() + }, + err: false, + }, + { + // Cannot override v1 resources. + runtimeConfig: map[string]string{ + "api/v1/pods": "false", + }, + expectedAPIConfig: func() *genericapiserver.ResourceConfig { + return master.DefaultAPIResourceConfigSource() + }, + err: true, + }, + { + // Disable v1. + runtimeConfig: map[string]string{ + "api/v1": "false", + }, + expectedAPIConfig: func() *genericapiserver.ResourceConfig { + config := master.DefaultAPIResourceConfigSource() + config.DisableVersions(unversioned.GroupVersion{Group: "", Version: "v1"}) + return config + }, + err: false, + }, + { + // Disable extensions. + runtimeConfig: map[string]string{ + "extensions/v1beta1": "false", + }, + expectedAPIConfig: func() *genericapiserver.ResourceConfig { + config := master.DefaultAPIResourceConfigSource() + config.DisableVersions(unversioned.GroupVersion{Group: "extensions", Version: "v1beta1"}) + return config + }, + err: false, + }, + { + // Disable deployments. + runtimeConfig: map[string]string{ + "extensions/v1beta1/deployments": "false", + }, + expectedAPIConfig: func() *genericapiserver.ResourceConfig { + config := master.DefaultAPIResourceConfigSource() + config.DisableResources(unversioned.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "deployments"}) + return config + }, + err: false, + }, + { + // Enable deployments and disable jobs. + runtimeConfig: map[string]string{ + "extensions/v1beta1/anything": "true", + "extensions/v1beta1/jobs": "false", + }, + expectedAPIConfig: func() *genericapiserver.ResourceConfig { + config := master.DefaultAPIResourceConfigSource() + config.DisableResources(unversioned.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "jobs"}) + config.EnableResources(unversioned.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "anything"}) + return config + }, + err: false, + }, + } + for _, test := range testCases { + s := &options.APIServer{ + RuntimeConfig: test.runtimeConfig, + } + actualDisablers, err := parseRuntimeConfig(s) + + if err == nil && test.err { + t.Fatalf("expected error for test: %v", test) + } else if err != nil && !test.err { + t.Fatalf("unexpected error: %s, for test: %v", err, test) + } + + expectedConfig := test.expectedAPIConfig() + if err == nil && !reflect.DeepEqual(actualDisablers, expectedConfig) { + t.Fatalf("%v: unexpected apiResourceDisablers. Actual: %q\n expected: %q", test.runtimeConfig, actualDisablers, expectedConfig) + } + } + +} From ca72bbabe1abf3d7c47c58eb0c31ddb5a1660df1 Mon Sep 17 00:00:00 2001 From: jianhuiz Date: Wed, 30 Mar 2016 10:10:47 -0700 Subject: [PATCH 2/2] update references of kube-apiserver to federated-apiserver --- federation/cmd/federated-apiserver/apiserver.go | 4 ++-- federation/cmd/federated-apiserver/app/server.go | 6 +++--- federation/cmd/federated-apiserver/app/server_test.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/federation/cmd/federated-apiserver/apiserver.go b/federation/cmd/federated-apiserver/apiserver.go index c63ec99c35183..564ba3e012861 100644 --- a/federation/cmd/federated-apiserver/apiserver.go +++ b/federation/cmd/federated-apiserver/apiserver.go @@ -25,8 +25,8 @@ import ( "runtime" "time" - "k8s.io/kubernetes/cmd/kube-apiserver/app" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/federation/cmd/federated-apiserver/app" + "k8s.io/kubernetes/federation/cmd/federated-apiserver/app/options" "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/pkg/version/verflag" diff --git a/federation/cmd/federated-apiserver/app/server.go b/federation/cmd/federated-apiserver/app/server.go index 13e27e8cc06b7..024d62e7f420a 100644 --- a/federation/cmd/federated-apiserver/app/server.go +++ b/federation/cmd/federated-apiserver/app/server.go @@ -32,7 +32,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/federation/cmd/federated-apiserver/app/options" "k8s.io/kubernetes/pkg/admission" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/unversioned" @@ -68,8 +68,8 @@ func NewAPIServerCommand() *cobra.Command { s := options.NewAPIServer() s.AddFlags(pflag.CommandLine) cmd := &cobra.Command{ - Use: "kube-apiserver", - Long: `The Kubernetes API server validates and configures data + Use: "federated-apiserver", + Long: `The Kubernetes federation API server validates and configures data for the api objects which include pods, services, replicationcontrollers, and others. The API Server services REST operations and provides the frontend to the cluster's shared state through which all other components interact.`, diff --git a/federation/cmd/federated-apiserver/app/server_test.go b/federation/cmd/federated-apiserver/app/server_test.go index 34a35aa755aa6..b3e56db655d7f 100644 --- a/federation/cmd/federated-apiserver/app/server_test.go +++ b/federation/cmd/federated-apiserver/app/server_test.go @@ -22,7 +22,7 @@ import ( "strings" "testing" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/federation/cmd/federated-apiserver/app/options" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/unversioned" "k8s.io/kubernetes/pkg/apis/extensions"