diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 4bf52c570f770..3f582eaf57891 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -46746,7 +46746,8 @@ "required": [ "service", "caBundle", - "priority" + "groupPriorityMinimum", + "versionPriority" ], "properties": { "caBundle": { @@ -46758,15 +46759,15 @@ "description": "Group is the API group name this server hosts", "type": "string" }, + "groupPriorityMinimum": { + "description": "GroupPriorityMininum is the priority this group should have at least. Higher priority means that the group is prefered by clients over lower priority ones. Note that other versions of this group might specify even higher GroupPriorityMininum values such that the whole group gets a higher priority. The primary sort is based on GroupPriorityMinimum, ordered highest number to lowest (20 before 10). The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo) We'd recommend something like: *.k8s.io (except extensions) at 18000 and PaaSes (OpenShift, Deis) are recommended to be in the 2000s", + "type": "integer", + "format": "int32" + }, "insecureSkipTLSVerify": { "description": "InsecureSkipTLSVerify disables TLS certificate verification when communicating with this server. This is strongly discouraged. You should use the CABundle instead.", "type": "boolean" }, - "priority": { - "description": "Priority controls the ordering of this API group in the overall discovery document that gets served. Client tools like `kubectl` use this ordering to derive preference, so this ordering mechanism is important. Values must be between 1 and 1000 The primary sort is based on priority, ordered lowest number to highest (10 before 20). The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo) We'd recommend something like: *.k8s.io (except extensions) at 100, extensions at 150 PaaSes (OpenShift, Deis) are recommended to be in the 200s", - "type": "integer", - "format": "int64" - }, "service": { "description": "Service is a reference to the service for this API server. It must communicate on port 443 If the Service is nil, that means the handling for the API groupversion is handled locally on this server. The call will simply delegate to the normal handler chain to be fulfilled.", "$ref": "#/definitions/io.k8s.kube-aggregator.pkg.apis.apiregistration.v1beta1.ServiceReference" @@ -46774,6 +46775,11 @@ "version": { "description": "Version is the API version this server hosts. For example, \"v1\"", "type": "string" + }, + "versionPriority": { + "description": "VersionPriority controls the ordering of this API version inside of its group. Must be greater than zero. The primary sort is based on VersionPriority, ordered highest to lowest (20 before 10). The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo) Since it's inside of a group, the number can be small, probably in the 10s.", + "type": "integer", + "format": "int32" } } }, diff --git a/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml b/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml index a453b4cfdf447..c3742493c3c26 100644 --- a/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml +++ b/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml @@ -31,7 +31,7 @@ spec: spec: containers: - name: autoscaler - image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.1.2 + image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.1.2-r2 resources: requests: cpu: "20m" diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml index 82e1211d7f06d..9870d09a81a5b 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml @@ -27,7 +27,7 @@ spec: hostNetwork: true containers: - name: fluentd-gcp - image: gcr.io/google-containers/fluentd-gcp:2.0.5 + image: gcr.io/google-containers/fluentd-gcp:2.0.6 # If fluentd consumes its own logs, the following situation may happen: # fluentd fails to send a chunk to the server => writes it to the log => # tries to send this message to the server => fails to send a chunk and so on. diff --git a/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml b/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml index feb6e5ea546db..4e1fbf285bc44 100644 --- a/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml +++ b/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml @@ -27,7 +27,7 @@ spec: dnsPolicy: Default containers: - name: metadata-proxy - image: gcr.io/google-containers/metadata-proxy:0.1 + image: gcr.io/google-containers/metadata-proxy:0.1.1 imagePullPolicy: Always securityContext: privileged: true diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 259a4704f3f5f..49eefb0e526ef 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -211,7 +211,7 @@ ENABLE_RESCHEDULER="${KUBE_ENABLE_RESCHEDULER:-true}" # new subnetwork will be created for the cluster. ENABLE_IP_ALIASES=${KUBE_GCE_ENABLE_IP_ALIASES:-false} if [ ${ENABLE_IP_ALIASES} = true ]; then - # Size of ranges allocated to each node. gcloud alpha supports only /32 and /24. + # Size of ranges allocated to each node. Currently supports only /32 and /24. IP_ALIAS_SIZE=${KUBE_GCE_IP_ALIAS_SIZE:-/24} IP_ALIAS_SUBNETWORK=${KUBE_GCE_IP_ALIAS_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-default} # Reserve the services IP space to avoid being allocated for other GCP resources. diff --git a/cluster/gce/gci/master-helper.sh b/cluster/gce/gci/master-helper.sh index 6357202c161a6..ed7bf31c97aee 100755 --- a/cluster/gce/gci/master-helper.sh +++ b/cluster/gce/gci/master-helper.sh @@ -76,7 +76,7 @@ function replicate-master-instance() { function create-master-instance-internal() { local gcloud="gcloud" if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then - gcloud="gcloud alpha" + gcloud="gcloud beta" fi local -r master_name="${1}" diff --git a/cmd/kube-apiserver/app/aggregator.go b/cmd/kube-apiserver/app/aggregator.go index 28dccf84b3e91..38fe52aaac270 100644 --- a/cmd/kube-apiserver/app/aggregator.go +++ b/cmd/kube-apiserver/app/aggregator.go @@ -25,6 +25,8 @@ import ( "net/http" "strings" + "github.com/golang/glog" + apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -144,16 +146,59 @@ func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delega } func makeAPIService(gv schema.GroupVersion) *apiregistration.APIService { + apiServicePriority, ok := apiVersionPriorities[gv] + if !ok { + // if we aren't found, then we shouldn't register ourselves because it could result in a CRD group version + // being permanently stuck in the APIServices list. + glog.Infof("Skipping APIService creation for %v", gv) + return nil + } return &apiregistration.APIService{ ObjectMeta: metav1.ObjectMeta{Name: gv.Version + "." + gv.Group}, Spec: apiregistration.APIServiceSpec{ - Group: gv.Group, - Version: gv.Version, - Priority: 100, + Group: gv.Group, + Version: gv.Version, + GroupPriorityMinimum: apiServicePriority.group, + VersionPriority: apiServicePriority.version, }, } } +type priority struct { + group int32 + version int32 +} + +// The proper way to resolve this letting the aggregator know the desired group and version-within-group order of the underlying servers +// is to refactor the genericapiserver.DelegationTarget to include a list of priorities based on which APIs were installed. +// This requires the APIGroupInfo struct to evolve and include the concept of priorities and to avoid mistakes, the core storage map there needs to be updated. +// That ripples out every bit as far as you'd expect, so for 1.7 we'll include the list here instead of being built up during storage. +var apiVersionPriorities = map[schema.GroupVersion]priority{ + {Group: "", Version: "v1"}: {group: 18000, version: 1}, + // extensions is above the rest for CLI compatibility, though the level of unqalified resource compatibility we + // can reasonably expect seems questionable. + {Group: "extensions", Version: "v1beta1"}: {group: 17900, version: 1}, + // to my knowledge, nothing below here collides + {Group: "apps", Version: "v1beta1"}: {group: 17800, version: 1}, + {Group: "authentication.k8s.io", Version: "v1"}: {group: 17700, version: 15}, + {Group: "authentication.k8s.io", Version: "v1beta1"}: {group: 17700, version: 9}, + {Group: "authorization.k8s.io", Version: "v1"}: {group: 17600, version: 15}, + {Group: "authorization.k8s.io", Version: "v1beta1"}: {group: 17600, version: 9}, + {Group: "autoscaling", Version: "v1"}: {group: 17500, version: 15}, + {Group: "autoscaling", Version: "v2alpha1"}: {group: 17500, version: 9}, + {Group: "batch", Version: "v1"}: {group: 17400, version: 15}, + {Group: "batch", Version: "v2alpha1"}: {group: 17400, version: 9}, + {Group: "certificates.k8s.io", Version: "v1beta1"}: {group: 17300, version: 9}, + {Group: "networking.k8s.io", Version: "v1"}: {group: 17200, version: 15}, + {Group: "policy", Version: "v1beta1"}: {group: 17100, version: 9}, + {Group: "rbac.authorization.k8s.io", Version: "v1beta1"}: {group: 17000, version: 12}, + {Group: "rbac.authorization.k8s.io", Version: "v1alpha1"}: {group: 17000, version: 9}, + {Group: "settings.k8s.io", Version: "v1alpha1"}: {group: 16900, version: 9}, + {Group: "storage.k8s.io", Version: "v1"}: {group: 16800, version: 15}, + {Group: "storage.k8s.io", Version: "v1beta1"}: {group: 16800, version: 9}, + {Group: "apiextensions.k8s.io", Version: "v1beta1"}: {group: 16700, version: 9}, +} + func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*apiregistration.APIService { apiServices := []*apiregistration.APIService{} @@ -175,14 +220,9 @@ func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, } apiService := makeAPIService(schema.GroupVersion{Group: tokens[2], Version: tokens[3]}) - - // TODO this is probably an indication that we need explicit and precise control over the discovery chain - // but for now its a special case - // apps has to come last for compatibility with 1.5 kubectl clients - if apiService.Spec.Group == "apps" { - apiService.Spec.Priority = 110 + if apiService == nil { + continue } - registration.AddAPIServiceToSync(apiService) apiServices = append(apiServices, apiService) } diff --git a/hack/make-rules/test-cmd-util.sh b/hack/make-rules/test-cmd-util.sh index a38c0c953b938..53c5dc6a659e9 100644 --- a/hack/make-rules/test-cmd-util.sh +++ b/hack/make-rules/test-cmd-util.sh @@ -1771,6 +1771,32 @@ run_non_native_resource_tests() { # Test that we can list this new third party resource kube::test::get_object_assert bars "{{range.items}}{{$id_field}}:{{end}}" 'test:' + # Test that we can watch the resource. + # Start watcher in background with process substitution, + # so we can read from stdout asynchronously. + kube::log::status "Testing ThirdPartyResource watching" + exec 3< <(kubectl "${kube_flags[@]}" get bars --request-timeout=1m --watch-only -o name & echo $! ; wait) + local watch_pid + read <&3 watch_pid + + # We can't be sure when the watch gets established, + # so keep triggering events (in the background) until something comes through. + local tries=0 + while [ ${tries} -lt 10 ]; do + tries=$((tries+1)) + kubectl "${kube_flags[@]}" patch bars/test -p "{\"patched\":\"${tries}\"}" --type=merge + sleep 1 + done & + local patch_pid=$! + + # Wait up to 30s for a complete line of output. + local watch_output + read <&3 -t 30 watch_output + # Stop the watcher and the patch loop. + kill -9 ${watch_pid} + kill -9 ${patch_pid} + kube::test::if_has_string "${watch_output}" 'bars/test' + # Delete the resource kubectl "${kube_flags[@]}" delete bars test diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index 78e055dde9fd2..425d44d37eca6 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -1972,6 +1972,9 @@ func validateContainers(containers []api.Container, volumes sets.String, fldPath } else { allNames.Insert(ctr.Name) } + // TODO: do not validate leading and trailing whitespace to preserve backward compatibility. + // for example: https://github.com/openshift/origin/issues/14659 image = " " is special token in pod template + // others may have done similar if len(ctr.Image) == 0 { allErrs = append(allErrs, field.Required(idxPath.Child("image"), "")) } @@ -2211,12 +2214,33 @@ func ValidateTolerations(tolerations []api.Toleration, fldPath *field.Path) fiel return allErrors } +// validateContainersOnlyForPod does additional validation for containers on a pod versus a pod template +// it only does additive validation of fields not covered in validateContainers +func validateContainersOnlyForPod(containers []api.Container, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + for i, ctr := range containers { + idxPath := fldPath.Index(i) + if len(ctr.Image) != len(strings.TrimSpace(ctr.Image)) { + allErrs = append(allErrs, field.Invalid(idxPath.Child("image"), ctr.Image, "must not have leading or trailing whitespace")) + } + } + return allErrs +} + // ValidatePod tests if required fields in the pod are set. func ValidatePod(pod *api.Pod) field.ErrorList { fldPath := field.NewPath("metadata") allErrs := ValidateObjectMeta(&pod.ObjectMeta, true, ValidatePodName, fldPath) allErrs = append(allErrs, ValidatePodSpecificAnnotations(pod.ObjectMeta.Annotations, &pod.Spec, fldPath.Child("annotations"))...) allErrs = append(allErrs, ValidatePodSpec(&pod.Spec, field.NewPath("spec"))...) + + // we do additional validation only pertinent for pods and not pod templates + // this was done to preserve backwards compatibility + specPath := field.NewPath("spec") + + allErrs = append(allErrs, validateContainersOnlyForPod(pod.Spec.Containers, specPath.Child("containers"))...) + allErrs = append(allErrs, validateContainersOnlyForPod(pod.Spec.InitContainers, specPath.Child("initContainers"))...) + return allErrs } @@ -2605,6 +2629,10 @@ func ValidateContainerUpdates(newContainers, oldContainers []api.Container, fldP if len(ctr.Image) == 0 { allErrs = append(allErrs, field.Required(fldPath.Index(i).Child("image"), "")) } + // this is only called from ValidatePodUpdate so its safe to check leading/trailing whitespace. + if len(strings.TrimSpace(ctr.Image)) != len(ctr.Image) { + allErrs = append(allErrs, field.Invalid(fldPath.Index(i).Child("image"), ctr.Image, "must not have leading or trailing whitespace")) + } } return allErrs, false } diff --git a/pkg/api/validation/validation_test.go b/pkg/api/validation/validation_test.go index b56c059ea45cc..671d0dd11f46d 100644 --- a/pkg/api/validation/validation_test.go +++ b/pkg/api/validation/validation_test.go @@ -3080,6 +3080,9 @@ func TestValidateContainers(t *testing.T) { successCase := []api.Container{ {Name: "abc", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}, + // backwards compatibility to ensure containers in pod template spec do not check for this + {Name: "def", Image: " ", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}, + {Name: "ghi", Image: " some ", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}, {Name: "123", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}, {Name: "abc-123", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}, { @@ -3234,6 +3237,7 @@ func TestValidateContainers(t *testing.T) { }) errorCases := map[string][]api.Container{ "zero-length name": {{Name: "", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}, + "zero-length-image": {{Name: "abc", Image: "", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}, "name > 63 characters": {{Name: strings.Repeat("a", 64), Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}, "name not a DNS label": {{Name: "a.b.c", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}, "name not unique": { @@ -4214,6 +4218,28 @@ func TestValidatePod(t *testing.T) { }, }, }, + "image whitespace": { + expectedError: "spec.containers[0].image", + spec: api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: "ns"}, + Spec: api.PodSpec{ + RestartPolicy: api.RestartPolicyAlways, + DNSPolicy: api.DNSClusterFirst, + Containers: []api.Container{{Name: "ctr", Image: " ", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}, + }, + }, + }, + "image leading and trailing whitespace": { + expectedError: "spec.containers[0].image", + spec: api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: "ns"}, + Spec: api.PodSpec{ + RestartPolicy: api.RestartPolicyAlways, + DNSPolicy: api.DNSClusterFirst, + Containers: []api.Container{{Name: "ctr", Image: " something ", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}, + }, + }, + }, "bad namespace": { expectedError: "metadata.namespace", spec: api.Pod{ diff --git a/pkg/controller/controller_ref_manager.go b/pkg/controller/controller_ref_manager.go index 96b546f6f27b6..c83f9a5341a61 100644 --- a/pkg/controller/controller_ref_manager.go +++ b/pkg/controller/controller_ref_manager.go @@ -341,7 +341,8 @@ func (m *ReplicaSetControllerRefManager) ClaimReplicaSets(sets []*extensions.Rep return claimed, utilerrors.NewAggregate(errlist) } -// AdoptReplicaSet sends a patch to take control of the ReplicaSet. It returns the error if +// AdoptReplicaSet sends a patch to take control of the ReplicaSet and also +// sets the finalizers to foregroundDeletion. It returns the error if // the patching fails. func (m *ReplicaSetControllerRefManager) AdoptReplicaSet(rs *extensions.ReplicaSet) error { if err := m.canAdopt(); err != nil { @@ -350,7 +351,14 @@ func (m *ReplicaSetControllerRefManager) AdoptReplicaSet(rs *extensions.ReplicaS // Note that ValidateOwnerReferences() will reject this patch if another // OwnerReference exists with controller=true. addControllerPatch := fmt.Sprintf( - `{"metadata":{"ownerReferences":[{"apiVersion":"%s","kind":"%s","name":"%s","uid":"%s","controller":true,"blockOwnerDeletion":true}],"uid":"%s"}}`, + `{ + "metadata": + { + "ownerReferences": [{"apiVersion":"%s","kind":"%s","name":"%s","uid":"%s","controller":true,"blockOwnerDeletion":true}], + "uid":"%s", + "finalizers": ["foregroundDeletion"] + } + }`, m.controllerKind.GroupVersion(), m.controllerKind.Kind, m.controller.GetName(), m.controller.GetUID(), rs.UID) return m.rsControl.PatchReplicaSet(rs.Namespace, rs.Name, []byte(addControllerPatch)) diff --git a/pkg/controller/deployment/sync.go b/pkg/controller/deployment/sync.go index 7e43d636ca1d1..17735762c98b5 100644 --- a/pkg/controller/deployment/sync.go +++ b/pkg/controller/deployment/sync.go @@ -300,6 +300,7 @@ func (dc *DeploymentController) getNewReplicaSet(d *extensions.Deployment, rsLis Name: d.Name + "-" + podTemplateSpecHash, Namespace: d.Namespace, OwnerReferences: []metav1.OwnerReference{*newControllerRef(d)}, + Finalizers: []string{metav1.FinalizerDeleteDependents}, }, Spec: extensions.ReplicaSetSpec{ Replicas: new(int32), diff --git a/pkg/kubelet/dockershim/docker_container.go b/pkg/kubelet/dockershim/docker_container.go index 283dc644dbd9c..ac79f3fe64ad9 100644 --- a/pkg/kubelet/dockershim/docker_container.go +++ b/pkg/kubelet/dockershim/docker_container.go @@ -233,6 +233,7 @@ func (ds *dockerService) removeContainerLogSymlink(containerID string) error { func (ds *dockerService) StartContainer(containerID string) error { err := ds.client.StartContainer(containerID) if err != nil { + err = transformStartContainerError(err) return fmt.Errorf("failed to start container %q: %v", containerID, err) } // Create container log symlink. diff --git a/pkg/kubelet/dockershim/helpers.go b/pkg/kubelet/dockershim/helpers.go index 22ab777e6afdd..c5b99a32d5750 100644 --- a/pkg/kubelet/dockershim/helpers.go +++ b/pkg/kubelet/dockershim/helpers.go @@ -55,6 +55,10 @@ const ( var ( conflictRE = regexp.MustCompile(`Conflict. (?:.)+ is already in use by container ([0-9a-z]+)`) + // this is hacky, but extremely common. + // if a container starts but the executable file is not found, runc gives a message that matches + startRE = regexp.MustCompile(`\\\\\\\"(.*)\\\\\\\": executable file not found`) + // Docker changes the security option separator from ':' to '=' in the 1.23 // API version. optsSeparatorChangeVersion = semver.MustParse(securityOptSeparatorChangeVersion) @@ -359,6 +363,19 @@ func recoverFromCreationConflictIfNeeded(client libdocker.Interface, createConfi return client.CreateContainer(createConfig) } +// transformStartContainerError does regex parsing on returned error +// for where container runtimes are giving less than ideal error messages. +func transformStartContainerError(err error) error { + if err == nil { + return nil + } + matches := startRE.FindStringSubmatch(err.Error()) + if len(matches) > 0 { + return fmt.Errorf("executable not found in $PATH") + } + return err +} + // getSecurityOptSeparator returns the security option separator based on the // docker API version. // TODO: Remove this function along with the relevant code when we no longer diff --git a/pkg/kubelet/kuberuntime/BUILD b/pkg/kubelet/kuberuntime/BUILD index 620d13aa24758..110b40c6c83ef 100644 --- a/pkg/kubelet/kuberuntime/BUILD +++ b/pkg/kubelet/kuberuntime/BUILD @@ -55,6 +55,7 @@ go_library( "//vendor/github.com/fsnotify/fsnotify:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index 73fbed910993d..bd7a5ba6c92db 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -24,9 +24,12 @@ import ( "os" "path/filepath" "sort" + "strings" "sync" "time" + "google.golang.org/grpc" + "github.com/armon/circbuf" "github.com/golang/glog" @@ -45,6 +48,30 @@ import ( "k8s.io/kubernetes/pkg/util/tail" ) +// recordContainerEvent should be used by the runtime manager for all container related events. +// it has sanity checks to ensure that we do not write events that can abuse our masters. +// in particular, it ensures that a containerID never appears in an event message as that +// is prone to causing a lot of distinct events that do not count well. +// it replaces any reference to a containerID with the containerName which is stable, and is what users know. +func (m *kubeGenericRuntimeManager) recordContainerEvent(pod *v1.Pod, container *v1.Container, containerID, eventType, reason, message string, args ...interface{}) { + ref, err := kubecontainer.GenerateContainerRef(pod, container) + if err != nil { + glog.Errorf("Can't make a ref to pod %q, container %v: %v", format.Pod(pod), container.Name, err) + return + } + eventMessage := message + if len(args) > 0 { + eventMessage = fmt.Sprintf(message, args...) + } + // this is a hack, but often the error from the runtime includes the containerID + // which kills our ability to deduplicate events. this protection makes a huge + // difference in the number of unique events + if containerID != "" { + eventMessage = strings.Replace(eventMessage, containerID, container.Name, -1) + } + m.recorder.Event(events.ToObjectReference(ref), eventType, reason, eventMessage) +} + // startContainer starts a container and returns a message indicates why it is failed on error. // It starts the container through the following steps: // * pull the image @@ -74,15 +101,16 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb containerConfig, err := m.generateContainerConfig(container, pod, restartCount, podIP, imageRef) if err != nil { - m.recorder.Eventf(ref, v1.EventTypeWarning, events.FailedToCreateContainer, "Failed to create container with error: %v", err) + m.recordContainerEvent(pod, container, "", v1.EventTypeWarning, events.FailedToCreateContainer, "Error: %v", grpc.ErrorDesc(err)) return "Generate Container Config Failed", err } containerID, err := m.runtimeService.CreateContainer(podSandboxID, containerConfig, podSandboxConfig) if err != nil { - m.recorder.Eventf(ref, v1.EventTypeWarning, events.FailedToCreateContainer, "Failed to create container with error: %v", err) + m.recordContainerEvent(pod, container, containerID, v1.EventTypeWarning, events.FailedToCreateContainer, "Error: %v", grpc.ErrorDesc(err)) return "Create Container Failed", err } - m.recorder.Eventf(ref, v1.EventTypeNormal, events.CreatedContainer, "Created container with id %v", containerID) + m.recordContainerEvent(pod, container, containerID, v1.EventTypeNormal, events.CreatedContainer, "Created container") + if ref != nil { m.containerRefManager.SetRef(kubecontainer.ContainerID{ Type: m.runtimeName, @@ -93,11 +121,10 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb // Step 3: start the container. err = m.runtimeService.StartContainer(containerID) if err != nil { - m.recorder.Eventf(ref, v1.EventTypeWarning, events.FailedToStartContainer, - "Failed to start container with id %v with error: %v", containerID, err) + m.recordContainerEvent(pod, container, containerID, v1.EventTypeWarning, events.FailedToStartContainer, "Error: %v", grpc.ErrorDesc(err)) return "Start Container Failed", err } - m.recorder.Eventf(ref, v1.EventTypeNormal, events.StartedContainer, "Started container with id %v", containerID) + m.recordContainerEvent(pod, container, containerID, v1.EventTypeNormal, events.StartedContainer, "Started container") // Symlink container logs to the legacy container log location for cluster logging // support. @@ -120,8 +147,7 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb } msg, handlerErr := m.runner.Run(kubeContainerID, pod, container, container.Lifecycle.PostStart) if handlerErr != nil { - err := fmt.Errorf("PostStart handler: %v", handlerErr) - m.generateContainerEvent(kubeContainerID, v1.EventTypeWarning, events.FailedPostStartHook, msg) + m.recordContainerEvent(pod, container, kubeContainerID.ID, v1.EventTypeWarning, events.FailedPostStartHook, msg) m.killContainer(pod, kubeContainerID, container.Name, "FailedPostStartHook", nil) return "PostStart Hook Failed", err } @@ -439,16 +465,6 @@ func toKubeContainerStatus(status *runtimeapi.ContainerStatus, runtimeName strin return cStatus } -// generateContainerEvent generates an event for the container. -func (m *kubeGenericRuntimeManager) generateContainerEvent(containerID kubecontainer.ContainerID, eventType, reason, message string) { - ref, ok := m.containerRefManager.GetRef(containerID) - if !ok { - glog.Warningf("No ref for container %q", containerID) - return - } - m.recorder.Event(events.ToObjectReference(ref), eventType, reason, message) -} - // executePreStopHook runs the pre-stop lifecycle hooks if applicable and returns the duration it takes. func (m *kubeGenericRuntimeManager) executePreStopHook(pod *v1.Pod, containerID kubecontainer.ContainerID, containerSpec *v1.Container, gracePeriod int64) int64 { glog.V(3).Infof("Running preStop hook for container %q", containerID.String()) @@ -460,7 +476,7 @@ func (m *kubeGenericRuntimeManager) executePreStopHook(pod *v1.Pod, containerID defer utilruntime.HandleCrash() if msg, err := m.runner.Run(containerID, pod, containerSpec, containerSpec.Lifecycle.PreStop); err != nil { glog.Errorf("preStop hook for container %q failed: %v", containerSpec.Name, err) - m.generateContainerEvent(containerID, v1.EventTypeWarning, events.FailedPreStopHook, msg) + m.recordContainerEvent(pod, containerSpec, containerID.ID, v1.EventTypeWarning, events.FailedPreStopHook, msg) } }() @@ -568,7 +584,7 @@ func (m *kubeGenericRuntimeManager) killContainer(pod *v1.Pod, containerID kubec if reason != "" { message = fmt.Sprint(message, ":", reason) } - m.generateContainerEvent(containerID, v1.EventTypeNormal, events.KillingContainer, message) + m.recordContainerEvent(pod, containerSpec, containerID.ID, v1.EventTypeNormal, events.KillingContainer, message) m.containerRefManager.ClearRef(containerID) return err diff --git a/pkg/kubelet/pod_workers.go b/pkg/kubelet/pod_workers.go index 8efe6da79800b..ea2ace249cf31 100644 --- a/pkg/kubelet/pod_workers.go +++ b/pkg/kubelet/pod_workers.go @@ -180,7 +180,13 @@ func (p *podWorkers) managePodLoop(podUpdates <-chan UpdatePodOptions) { } if err != nil { glog.Errorf("Error syncing pod %s (%q), skipping: %v", update.Pod.UID, format.Pod(update.Pod), err) - p.recorder.Eventf(update.Pod, v1.EventTypeWarning, events.FailedSync, "Error syncing pod, skipping: %v", err) + // if we failed sync, we throw more specific events for why it happened. + // as a result, i question the value of this event. + // TODO: determine if we can remove this in a future release. + // do not include descriptive text that can vary on why it failed so in a pathological + // scenario, kubelet does not create enough discrete events that miss default aggregation + // window. + p.recorder.Eventf(update.Pod, v1.EventTypeWarning, events.FailedSync, "Error syncing pod") } p.wrapUp(update.Pod.UID, err) } diff --git a/pkg/master/master.go b/pkg/master/master.go index 7574a657df0c3..6760d4e9b189f 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -249,6 +249,10 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget, // The order here is preserved in discovery. // If resources with identical names exist in more than one of these groups (e.g. "deployments.apps"" and "deployments.extensions"), // the order of this list determines which group an unqualified resource name (e.g. "deployments") should prefer. + // This priority order is used for local discovery, but it ends up aggregated in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go + // with specific priorities. + // TODO: describe the priority all the way down in the RESTStorageProviders and plumb it back through the various discovery + // handlers that we have. restStorageProviders := []RESTStorageProvider{ authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authenticator}, authorizationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorizer}, diff --git a/pkg/master/thirdparty/tprregistration_controller.go b/pkg/master/thirdparty/tprregistration_controller.go index d731635d29006..6e0bcc3fac513 100644 --- a/pkg/master/thirdparty/tprregistration_controller.go +++ b/pkg/master/thirdparty/tprregistration_controller.go @@ -255,9 +255,10 @@ func (c *tprRegistrationController) handleVersionUpdate(groupVersion schema.Grou c.apiServiceRegistration.AddAPIServiceToSync(&apiregistration.APIService{ ObjectMeta: metav1.ObjectMeta{Name: apiServiceName}, Spec: apiregistration.APIServiceSpec{ - Group: groupVersion.Group, - Version: groupVersion.Version, - Priority: 500, // TPRs should have relatively low priority + Group: groupVersion.Group, + Version: groupVersion.Version, + GroupPriorityMinimum: 1000, // TPRs should have relatively low priority + VersionPriority: 100, // TPRs should have relatively low priority }, }) diff --git a/pkg/master/thirdparty/tprregistration_controller_test.go b/pkg/master/thirdparty/tprregistration_controller_test.go index 570157523445b..97c6e28ec2f04 100644 --- a/pkg/master/thirdparty/tprregistration_controller_test.go +++ b/pkg/master/thirdparty/tprregistration_controller_test.go @@ -84,9 +84,10 @@ func TestHandleVersionUpdate(t *testing.T) { { ObjectMeta: metav1.ObjectMeta{Name: "v1.group.com"}, Spec: apiregistration.APIServiceSpec{ - Group: "group.com", - Version: "v1", - Priority: 500, + Group: "group.com", + Version: "v1", + GroupPriorityMinimum: 1000, + VersionPriority: 100, }, }, }, @@ -121,9 +122,10 @@ func TestHandleVersionUpdate(t *testing.T) { { ObjectMeta: metav1.ObjectMeta{Name: "v1.group.com"}, Spec: apiregistration.APIServiceSpec{ - Group: "group.com", - Version: "v1", - Priority: 500, + Group: "group.com", + Version: "v1", + GroupPriorityMinimum: 1000, + VersionPriority: 100, }, }, }, diff --git a/pkg/registry/admissionregistration/rest/storage_apiserver.go b/pkg/registry/admissionregistration/rest/storage_apiserver.go index a315cabf5d919..a5dd06d803f61 100644 --- a/pkg/registry/admissionregistration/rest/storage_apiserver.go +++ b/pkg/registry/admissionregistration/rest/storage_apiserver.go @@ -32,6 +32,8 @@ type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(admissionregistration.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(admissionregistrationv1alpha1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[admissionregistrationv1alpha1.SchemeGroupVersion.Version] = p.v1alpha1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/apps/rest/storage_apps.go b/pkg/registry/apps/rest/storage_apps.go index 56f98143b21d3..339451b3d2d9f 100644 --- a/pkg/registry/apps/rest/storage_apps.go +++ b/pkg/registry/apps/rest/storage_apps.go @@ -33,6 +33,8 @@ type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apps.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(appsapiv1beta1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[appsapiv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/authentication/rest/storage_authentication.go b/pkg/registry/authentication/rest/storage_authentication.go index 4f8d1ce698b2f..f8f47939012dd 100644 --- a/pkg/registry/authentication/rest/storage_authentication.go +++ b/pkg/registry/authentication/rest/storage_authentication.go @@ -40,6 +40,8 @@ func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorag // } apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(authentication.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(authenticationv1beta1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[authenticationv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/authorization/rest/storage_authorization.go b/pkg/registry/authorization/rest/storage_authorization.go index f875e5c5573dd..a543949fb2653 100644 --- a/pkg/registry/authorization/rest/storage_authorization.go +++ b/pkg/registry/authorization/rest/storage_authorization.go @@ -41,6 +41,8 @@ func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorag } apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(authorization.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(authorizationv1beta1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[authorizationv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/autoscaling/rest/storage_autoscaling.go b/pkg/registry/autoscaling/rest/storage_autoscaling.go index 60b3bf3a19d47..5fcbfc1ad2c06 100644 --- a/pkg/registry/autoscaling/rest/storage_autoscaling.go +++ b/pkg/registry/autoscaling/rest/storage_autoscaling.go @@ -32,6 +32,8 @@ type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(autoscaling.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(autoscalingapiv2alpha1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[autoscalingapiv2alpha1.SchemeGroupVersion.Version] = p.v2alpha1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/batch/rest/storage_batch.go b/pkg/registry/batch/rest/storage_batch.go index d523b8ab0877f..55582a8029199 100644 --- a/pkg/registry/batch/rest/storage_batch.go +++ b/pkg/registry/batch/rest/storage_batch.go @@ -34,6 +34,8 @@ type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(batch.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(batchapiv2alpha1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[batchapiv2alpha1.SchemeGroupVersion.Version] = p.v2alpha1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/certificates/rest/storage_certificates.go b/pkg/registry/certificates/rest/storage_certificates.go index 7a6a643732675..c03fb9ecea09d 100644 --- a/pkg/registry/certificates/rest/storage_certificates.go +++ b/pkg/registry/certificates/rest/storage_certificates.go @@ -31,6 +31,8 @@ type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(certificates.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(certificatesapiv1beta1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[certificatesapiv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/extensions/rest/storage_extensions.go b/pkg/registry/extensions/rest/storage_extensions.go index f0cfe5518c6b2..ec3bdeb0cf40a 100644 --- a/pkg/registry/extensions/rest/storage_extensions.go +++ b/pkg/registry/extensions/rest/storage_extensions.go @@ -48,6 +48,8 @@ type RESTStorageProvider struct { func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(extensions.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(extensionsapiv1beta1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[extensionsapiv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/extensions/thirdpartyresourcedata/codec.go b/pkg/registry/extensions/thirdpartyresourcedata/codec.go index b06c2a81a3b36..f9751bd949cae 100644 --- a/pkg/registry/extensions/thirdpartyresourcedata/codec.go +++ b/pkg/registry/extensions/thirdpartyresourcedata/codec.go @@ -528,6 +528,12 @@ func (t *thirdPartyResourceDataEncoder) Encode(obj runtime.Object, stream io.Wri } return nil + case *metav1.WatchEvent: + // This is the same as the InternalEvent case above, except the caller + // already did the conversion for us (see #44350). + // In theory, we probably don't need the InternalEvent case anymore, + // but the test coverage for TPR is too low to risk removing it. + return json.NewEncoder(stream).Encode(obj) case *metav1.Status, *metav1.APIResourceList: return t.delegate.Encode(obj, stream) default: diff --git a/pkg/registry/networking/rest/storage_settings.go b/pkg/registry/networking/rest/storage_settings.go index c662025dd2211..3b773bd9e3456 100644 --- a/pkg/registry/networking/rest/storage_settings.go +++ b/pkg/registry/networking/rest/storage_settings.go @@ -31,6 +31,8 @@ type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(networking.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(networkingapiv1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[networkingapiv1.SchemeGroupVersion.Version] = p.v1alpha1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/policy/rest/storage_policy.go b/pkg/registry/policy/rest/storage_policy.go index c1067d5b0f977..26a9594eebd9c 100644 --- a/pkg/registry/policy/rest/storage_policy.go +++ b/pkg/registry/policy/rest/storage_policy.go @@ -31,6 +31,8 @@ type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(policy.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(policyapiv1beta1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[policyapiv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/rbac/rest/storage_rbac.go b/pkg/registry/rbac/rest/storage_rbac.go index fdddc867f32ac..34df168d877f6 100644 --- a/pkg/registry/rbac/rest/storage_rbac.go +++ b/pkg/registry/rbac/rest/storage_rbac.go @@ -65,6 +65,8 @@ var _ genericapiserver.PostStartHookProvider = RESTStorageProvider{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(rbac.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(rbacapiv1alpha1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[rbacapiv1alpha1.SchemeGroupVersion.Version] = p.storage(rbacapiv1alpha1.SchemeGroupVersion, apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/settings/rest/storage_settings.go b/pkg/registry/settings/rest/storage_settings.go index 8e9a41b4db41c..949faac847c7b 100644 --- a/pkg/registry/settings/rest/storage_settings.go +++ b/pkg/registry/settings/rest/storage_settings.go @@ -31,6 +31,8 @@ type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(settings.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(settingsapiv1alpha1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[settingsapiv1alpha1.SchemeGroupVersion.Version] = p.v1alpha1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/storage/rest/storage_storage.go b/pkg/registry/storage/rest/storage_storage.go index c12f8038243c3..0eebcd0dbee06 100644 --- a/pkg/registry/storage/rest/storage_storage.go +++ b/pkg/registry/storage/rest/storage_storage.go @@ -33,6 +33,8 @@ type RESTStorageProvider struct { func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(storageapi.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs) + // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. + // TODO refactor the plumbing to provide the information in the APIGroupInfo if apiResourceConfigSource.AnyResourcesForVersionEnabled(storageapiv1beta1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[storageapiv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/helpers.go b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/helpers.go index 683b7ce537add..1f6790b32e342 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/helpers.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/helpers.go @@ -23,12 +23,12 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) -func SortedByGroup(servers []*APIService) [][]*APIService { - serversByPriority := ByPriority(servers) - sort.Sort(serversByPriority) +func SortedByGroupAndVersion(servers []*APIService) [][]*APIService { + serversByGroupPriorityMinimum := ByGroupPriorityMinimum(servers) + sort.Sort(serversByGroupPriorityMinimum) ret := [][]*APIService{} - for _, curr := range serversByPriority { + for _, curr := range serversByGroupPriorityMinimum { // check to see if we already have an entry for this group existingIndex := -1 for j, groupInReturn := range ret { @@ -40,6 +40,7 @@ func SortedByGroup(servers []*APIService) [][]*APIService { if existingIndex >= 0 { ret[existingIndex] = append(ret[existingIndex], curr) + sort.Sort(ByVersionPriority(ret[existingIndex])) continue } @@ -49,15 +50,32 @@ func SortedByGroup(servers []*APIService) [][]*APIService { return ret } -type ByPriority []*APIService +// ByGroupPriorityMinimum sorts with the highest group number first, then by name. +// This is not a simple reverse, because we want the name sorting to be alpha, not +// reverse alpha. +type ByGroupPriorityMinimum []*APIService -func (s ByPriority) Len() int { return len(s) } -func (s ByPriority) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s ByPriority) Less(i, j int) bool { - if s[i].Spec.Priority == s[j].Spec.Priority { - return s[i].Name < s[j].Name +func (s ByGroupPriorityMinimum) Len() int { return len(s) } +func (s ByGroupPriorityMinimum) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s ByGroupPriorityMinimum) Less(i, j int) bool { + if s[i].Spec.GroupPriorityMinimum != s[j].Spec.GroupPriorityMinimum { + return s[i].Spec.GroupPriorityMinimum > s[j].Spec.GroupPriorityMinimum } - return s[i].Spec.Priority < s[j].Spec.Priority + return s[i].Name < s[j].Name +} + +// ByVersionPriority sorts with the highest version number first, then by name. +// This is not a simple reverse, because we want the name sorting to be alpha, not +// reverse alpha. +type ByVersionPriority []*APIService + +func (s ByVersionPriority) Len() int { return len(s) } +func (s ByVersionPriority) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s ByVersionPriority) Less(i, j int) bool { + if s[i].Spec.VersionPriority != s[j].Spec.VersionPriority { + return s[i].Spec.VersionPriority > s[j].Spec.VersionPriority + } + return s[i].Name < s[j].Name } // APIServiceNameToGroupVersion returns the GroupVersion for a given apiServiceName. The name diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/types.go b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/types.go index b4a99abcd65b6..2ab58332e4580 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/types.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/types.go @@ -53,14 +53,19 @@ type APIServiceSpec struct { // CABundle is a PEM encoded CA bundle which will be used to validate an API server's serving certificate. CABundle []byte - // Priority controls the ordering of this API group in the overall discovery document that gets served. - // Client tools like `kubectl` use this ordering to derive preference, so this ordering mechanism is important. - // Values must be between 1 and 1000 - // The primary sort is based on priority, ordered lowest number to highest (10 before 20). + // GroupPriorityMininum is the priority this group should have at least. Higher priority means that the group is prefered by clients over lower priority ones. + // Note that other versions of this group might specify even higher GroupPriorityMininum values such that the whole group gets a higher priority. + // The primary sort is based on GroupPriorityMinimum, ordered highest number to lowest (20 before 10). // The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo) - // We'd recommend something like: *.k8s.io (except extensions) at 100, extensions at 150 - // PaaSes (OpenShift, Deis) are recommended to be in the 200s - Priority int64 + // We'd recommend something like: *.k8s.io (except extensions) at 18000 and + // PaaSes (OpenShift, Deis) are recommended to be in the 2000s + GroupPriorityMinimum int32 + + // VersionPriority controls the ordering of this API version inside of its group. Must be greater than zero. + // The primary sort is based on VersionPriority, ordered highest to lowest (20 before 10). + // The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo) + // Since it's inside of a group, the number can be small, probably in the 10s. + VersionPriority int32 } type ConditionStatus string diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/generated.pb.go b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/generated.pb.go index 01dd7adeb8317..47e567af7476a 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/generated.pb.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/generated.pb.go @@ -255,9 +255,12 @@ func (m *APIServiceSpec) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintGenerated(dAtA, i, uint64(len(m.CABundle))) i += copy(dAtA[i:], m.CABundle) } - dAtA[i] = 0x30 + dAtA[i] = 0x38 i++ - i = encodeVarintGenerated(dAtA, i, uint64(m.Priority)) + i = encodeVarintGenerated(dAtA, i, uint64(m.GroupPriorityMinimum)) + dAtA[i] = 0x40 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.VersionPriority)) return i, nil } @@ -402,7 +405,8 @@ func (m *APIServiceSpec) Size() (n int) { l = len(m.CABundle) n += 1 + l + sovGenerated(uint64(l)) } - n += 1 + sovGenerated(uint64(m.Priority)) + n += 1 + sovGenerated(uint64(m.GroupPriorityMinimum)) + n += 1 + sovGenerated(uint64(m.VersionPriority)) return n } @@ -488,7 +492,8 @@ func (this *APIServiceSpec) String() string { `Version:` + fmt.Sprintf("%v", this.Version) + `,`, `InsecureSkipTLSVerify:` + fmt.Sprintf("%v", this.InsecureSkipTLSVerify) + `,`, `CABundle:` + valueToStringGenerated(this.CABundle) + `,`, - `Priority:` + fmt.Sprintf("%v", this.Priority) + `,`, + `GroupPriorityMinimum:` + fmt.Sprintf("%v", this.GroupPriorityMinimum) + `,`, + `VersionPriority:` + fmt.Sprintf("%v", this.VersionPriority) + `,`, `}`, }, "") return s @@ -1140,11 +1145,30 @@ func (m *APIServiceSpec) Unmarshal(dAtA []byte) error { m.CABundle = []byte{} } iNdEx = postIndex - case 6: + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field GroupPriorityMinimum", wireType) + } + m.GroupPriorityMinimum = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.GroupPriorityMinimum |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 8: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Priority", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field VersionPriority", wireType) } - m.Priority = 0 + m.VersionPriority = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -1154,7 +1178,7 @@ func (m *APIServiceSpec) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Priority |= (int64(b) & 0x7F) << shift + m.VersionPriority |= (int32(b) & 0x7F) << shift if b < 0x80 { break } @@ -1479,56 +1503,58 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 801 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xcf, 0x6f, 0x1b, 0x45, - 0x14, 0xf6, 0x3a, 0xfe, 0x95, 0x49, 0x68, 0xc3, 0xa0, 0x0a, 0xcb, 0x12, 0xeb, 0x60, 0x24, 0x70, - 0x11, 0xdd, 0x25, 0x05, 0xf1, 0x43, 0x9c, 0xba, 0x39, 0x54, 0x96, 0x5c, 0xa8, 0xc6, 0x51, 0x0f, - 0x08, 0x89, 0x8e, 0xd7, 0x2f, 0xeb, 0xc1, 0xd9, 0x1f, 0xcc, 0xcc, 0x5a, 0xf2, 0xad, 0x7f, 0x02, - 0x17, 0xfe, 0xa7, 0x1c, 0x38, 0xf4, 0x98, 0x93, 0x45, 0xcc, 0x7f, 0xd1, 0x0b, 0x68, 0x66, 0x67, - 0x77, 0x5d, 0xdb, 0x15, 0x51, 0x94, 0x8b, 0xe5, 0x79, 0xef, 0x7d, 0xdf, 0xf7, 0xe6, 0xbd, 0x6f, - 0x07, 0xbd, 0x9c, 0x7d, 0x27, 0x1c, 0x16, 0xbb, 0xb3, 0x74, 0x0c, 0x3c, 0x02, 0x09, 0xc2, 0x9d, - 0x43, 0x34, 0x89, 0xb9, 0xbb, 0x96, 0x78, 0x44, 0x83, 0x80, 0x43, 0x40, 0x65, 0xcc, 0xdd, 0x64, - 0x16, 0xb8, 0x34, 0x61, 0x42, 0xfd, 0x70, 0x08, 0x98, 0x90, 0x9c, 0x4a, 0x16, 0x47, 0xee, 0xfc, - 0x64, 0x0c, 0x92, 0x9e, 0xb8, 0x01, 0x44, 0xc0, 0xa9, 0x84, 0x89, 0x93, 0xf0, 0x58, 0xc6, 0xf8, - 0xdb, 0x8c, 0xc8, 0x51, 0x44, 0xbf, 0x96, 0x44, 0x4e, 0x32, 0x0b, 0x1c, 0x45, 0xe4, 0x6c, 0x10, - 0x39, 0x86, 0xa8, 0xf3, 0x28, 0x60, 0x72, 0x9a, 0x8e, 0x1d, 0x3f, 0x0e, 0xdd, 0x20, 0x0e, 0x62, - 0x57, 0xf3, 0x8d, 0xd3, 0x73, 0x7d, 0xd2, 0x07, 0xfd, 0x2f, 0xd3, 0xe9, 0x7c, 0x6d, 0x1a, 0xa6, - 0x09, 0x0b, 0xa9, 0x3f, 0x65, 0x11, 0xf0, 0x45, 0xd9, 0x6d, 0x08, 0x92, 0xba, 0xf3, 0xad, 0xee, - 0x3a, 0xee, 0xbb, 0x50, 0x3c, 0x8d, 0x24, 0x0b, 0x61, 0x0b, 0xf0, 0xcd, 0xff, 0x01, 0x84, 0x3f, - 0x85, 0x90, 0x6e, 0xe1, 0xbe, 0x7a, 0x17, 0x2e, 0x95, 0xec, 0xc2, 0x65, 0x91, 0x14, 0x92, 0x6f, - 0x82, 0x7a, 0x7f, 0x55, 0x11, 0x7a, 0xf2, 0x7c, 0x30, 0x02, 0x3e, 0x67, 0x3e, 0xe0, 0x97, 0xa8, - 0xa5, 0xee, 0x31, 0xa1, 0x92, 0xb6, 0xad, 0x63, 0xab, 0x7f, 0xf0, 0xf8, 0x4b, 0xc7, 0x4c, 0x77, - 0x9d, 0xb6, 0x1c, 0xad, 0xaa, 0x76, 0xe6, 0x27, 0xce, 0x4f, 0xe3, 0xdf, 0xc0, 0x97, 0xcf, 0x40, - 0x52, 0x0f, 0x5f, 0x2e, 0xbb, 0x95, 0xd5, 0xb2, 0x8b, 0xca, 0x18, 0x29, 0x58, 0x31, 0x43, 0x35, - 0x91, 0x80, 0xdf, 0xae, 0x6a, 0xf6, 0xa7, 0xce, 0x2d, 0x77, 0xe7, 0x94, 0x4d, 0x8f, 0x12, 0xf0, - 0xbd, 0x43, 0x23, 0x5a, 0x53, 0x27, 0xa2, 0x25, 0xf0, 0xef, 0xa8, 0x21, 0x24, 0x95, 0xa9, 0x68, - 0xef, 0x69, 0xb1, 0xc1, 0x5d, 0x88, 0x69, 0x42, 0xef, 0x9e, 0x91, 0x6b, 0x64, 0x67, 0x62, 0x84, - 0x7a, 0x57, 0x55, 0xf4, 0x41, 0x59, 0x7c, 0x1a, 0x47, 0x13, 0xa6, 0x88, 0xf0, 0x0f, 0xa8, 0x26, - 0x17, 0x09, 0xe8, 0x99, 0xee, 0x7b, 0x9f, 0xe5, 0xcd, 0x9e, 0x2d, 0x12, 0x78, 0xb3, 0xec, 0x7e, - 0xb8, 0x03, 0xa2, 0x52, 0x44, 0x83, 0xf0, 0xf7, 0xc5, 0x3d, 0xaa, 0x1a, 0xfe, 0xf1, 0xdb, 0xe2, - 0x6f, 0x96, 0xdd, 0xfb, 0x05, 0xec, 0xed, 0x7e, 0xf0, 0x1c, 0xe1, 0x0b, 0x2a, 0xe4, 0x19, 0xa7, - 0x91, 0xc8, 0x68, 0x59, 0x08, 0x66, 0x1c, 0x9f, 0xdf, 0x6c, 0xb3, 0x0a, 0xe1, 0x75, 0x8c, 0x24, - 0x1e, 0x6e, 0xb1, 0x91, 0x1d, 0x0a, 0xf8, 0x53, 0xd4, 0xe0, 0x40, 0x45, 0x1c, 0xb5, 0x6b, 0xba, - 0xe5, 0x62, 0x5e, 0x44, 0x47, 0x89, 0xc9, 0xe2, 0x87, 0xa8, 0x19, 0x82, 0x10, 0x34, 0x80, 0x76, - 0x5d, 0x17, 0xde, 0x37, 0x85, 0xcd, 0x67, 0x59, 0x98, 0xe4, 0xf9, 0xde, 0x95, 0x85, 0xee, 0x95, - 0x73, 0x1a, 0x32, 0x21, 0xf1, 0x2f, 0x5b, 0x6e, 0x75, 0x6e, 0x76, 0x27, 0x85, 0xd6, 0x5e, 0x3d, - 0x32, 0x72, 0xad, 0x3c, 0xb2, 0xe6, 0xd4, 0x29, 0xaa, 0x33, 0x09, 0xa1, 0x9a, 0xfa, 0x5e, 0xff, - 0xe0, 0xf1, 0xe9, 0x1d, 0xb8, 0xc7, 0x7b, 0xcf, 0xe8, 0xd5, 0x07, 0x8a, 0x99, 0x64, 0x02, 0xbd, - 0x7f, 0xab, 0xeb, 0x57, 0x53, 0x0e, 0xc6, 0x09, 0x6a, 0x8a, 0xec, 0x68, 0x6e, 0x76, 0x7b, 0xf3, - 0x1a, 0x5a, 0x02, 0xe7, 0xc0, 0x21, 0xf2, 0xc1, 0x3b, 0x50, 0xf3, 0xcd, 0xa3, 0xb9, 0x0c, 0xfe, - 0x04, 0xd5, 0x03, 0x1e, 0xa7, 0x89, 0x31, 0x59, 0xd1, 0xe9, 0x53, 0x15, 0x24, 0x59, 0x4e, 0xed, - 0x6b, 0x0e, 0x5c, 0xb0, 0x38, 0xd2, 0x26, 0x5a, 0xdb, 0xd7, 0x8b, 0x2c, 0x4c, 0xf2, 0x3c, 0x1e, - 0xa1, 0x07, 0x2c, 0x12, 0xe0, 0xa7, 0x1c, 0x46, 0x33, 0x96, 0x9c, 0x0d, 0x47, 0x2f, 0x80, 0xb3, - 0xf3, 0x85, 0x76, 0x44, 0xcb, 0xfb, 0xc8, 0x00, 0x1f, 0x0c, 0x76, 0x15, 0x91, 0xdd, 0x58, 0xdc, - 0x47, 0x2d, 0x9f, 0x7a, 0x69, 0x34, 0xb9, 0xc8, 0x0c, 0x73, 0xe8, 0x1d, 0xaa, 0xed, 0x9d, 0x3e, - 0xc9, 0x62, 0xa4, 0xc8, 0xe2, 0x2f, 0x50, 0x2b, 0xe1, 0x2c, 0xe6, 0x4c, 0x2e, 0xda, 0x8d, 0x63, - 0xab, 0xbf, 0x57, 0xee, 0xfa, 0xb9, 0x89, 0x93, 0xa2, 0xa2, 0xf7, 0xa7, 0x85, 0x8e, 0x36, 0x3f, - 0x72, 0xfc, 0xca, 0x42, 0xc8, 0xcf, 0x3f, 0x2c, 0xd1, 0xb6, 0xb4, 0x0d, 0x86, 0x77, 0x60, 0x83, - 0xe2, 0x6b, 0x2d, 0xdf, 0xca, 0x22, 0x24, 0xc8, 0x9a, 0x66, 0x0f, 0xd0, 0xd1, 0xe6, 0xfa, 0xb0, - 0x8b, 0xf6, 0x23, 0x1a, 0x82, 0x48, 0xa8, 0x9f, 0x3f, 0x28, 0xef, 0x1b, 0x9a, 0xfd, 0x1f, 0xf3, - 0x04, 0x29, 0x6b, 0xf0, 0x31, 0xaa, 0xa9, 0x83, 0x59, 0x6c, 0xf1, 0x52, 0xaa, 0x5a, 0xa2, 0x33, - 0xde, 0xc3, 0xcb, 0x6b, 0xbb, 0xf2, 0xfa, 0xda, 0xae, 0x5c, 0x5d, 0xdb, 0x95, 0x57, 0x2b, 0xdb, - 0xba, 0x5c, 0xd9, 0xd6, 0xeb, 0x95, 0x6d, 0xfd, 0xbd, 0xb2, 0xad, 0x3f, 0xfe, 0xb1, 0x2b, 0x3f, - 0x37, 0xcd, 0x2d, 0xfe, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xd6, 0x26, 0x9e, 0xdb, 0xcf, 0x07, 0x00, + // 833 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xcf, 0x4f, 0xe3, 0x46, + 0x14, 0x8e, 0x21, 0x21, 0x61, 0xa0, 0x0b, 0x9d, 0xee, 0x6a, 0x23, 0xd4, 0x1a, 0x9a, 0x4a, 0x2d, + 0x5b, 0x69, 0xed, 0xb2, 0xad, 0xfa, 0x43, 0x3d, 0x61, 0x0e, 0x08, 0x09, 0x5a, 0x34, 0x41, 0x1c, + 0xaa, 0x4a, 0xdd, 0x89, 0xf3, 0x30, 0xd3, 0xe0, 0xb1, 0x3b, 0x33, 0x8e, 0x94, 0xdb, 0xfe, 0x09, + 0xbd, 0xf4, 0x7f, 0xe2, 0xd0, 0xc3, 0x1e, 0x39, 0x45, 0x25, 0x95, 0xfa, 0x47, 0xec, 0xa9, 0x9a, + 0xf1, 0xd8, 0x0e, 0x49, 0x56, 0xbb, 0x42, 0x5c, 0xa2, 0xcc, 0x7b, 0xef, 0xfb, 0xbe, 0xf7, 0xde, + 0x7c, 0x1e, 0xf4, 0x72, 0xf0, 0xbd, 0xf4, 0x58, 0xe2, 0x0f, 0xb2, 0x1e, 0x08, 0x0e, 0x0a, 0xa4, + 0x3f, 0x04, 0xde, 0x4f, 0x84, 0x3f, 0x95, 0x78, 0x4e, 0xa3, 0x48, 0x40, 0x44, 0x55, 0x22, 0xfc, + 0x74, 0x10, 0xf9, 0x34, 0x65, 0x52, 0xff, 0x08, 0x88, 0x98, 0x54, 0x82, 0x2a, 0x96, 0x70, 0x7f, + 0xb8, 0xd7, 0x03, 0x45, 0xf7, 0xfc, 0x08, 0x38, 0x08, 0xaa, 0xa0, 0xef, 0xa5, 0x22, 0x51, 0x09, + 0xfe, 0x2e, 0x27, 0xf2, 0x34, 0xd1, 0x6f, 0x15, 0x91, 0x97, 0x0e, 0x22, 0x4f, 0x13, 0x79, 0x33, + 0x44, 0x9e, 0x25, 0xda, 0x7a, 0x1e, 0x31, 0x75, 0x99, 0xf5, 0xbc, 0x30, 0x89, 0xfd, 0x28, 0x89, + 0x12, 0xdf, 0xf0, 0xf5, 0xb2, 0x0b, 0x73, 0x32, 0x07, 0xf3, 0x2f, 0xd7, 0xd9, 0xfa, 0xc6, 0x36, + 0x4c, 0x53, 0x16, 0xd3, 0xf0, 0x92, 0x71, 0x10, 0xa3, 0xaa, 0xdb, 0x18, 0x14, 0xf5, 0x87, 0x73, + 0xdd, 0x6d, 0xf9, 0x6f, 0x43, 0x89, 0x8c, 0x2b, 0x16, 0xc3, 0x1c, 0xe0, 0xdb, 0x77, 0x01, 0x64, + 0x78, 0x09, 0x31, 0x9d, 0xc3, 0x7d, 0xfd, 0x36, 0x5c, 0xa6, 0xd8, 0x95, 0xcf, 0xb8, 0x92, 0x4a, + 0xcc, 0x82, 0x3a, 0x7f, 0x2f, 0x21, 0xb4, 0x7f, 0x7a, 0xd4, 0x05, 0x31, 0x64, 0x21, 0xe0, 0x97, + 0xa8, 0xa5, 0xe7, 0xe8, 0x53, 0x45, 0xdb, 0xce, 0x8e, 0xb3, 0xbb, 0xf6, 0xe2, 0x2b, 0xcf, 0x6e, + 0x77, 0x9a, 0xb6, 0x5a, 0xad, 0xae, 0xf6, 0x86, 0x7b, 0xde, 0xcf, 0xbd, 0xdf, 0x21, 0x54, 0x27, + 0xa0, 0x68, 0x80, 0xaf, 0xc7, 0xdb, 0xb5, 0xc9, 0x78, 0x1b, 0x55, 0x31, 0x52, 0xb2, 0x62, 0x86, + 0xea, 0x32, 0x85, 0xb0, 0xbd, 0x64, 0xd8, 0x0f, 0xbd, 0x7b, 0xde, 0x9d, 0x57, 0x35, 0xdd, 0x4d, + 0x21, 0x0c, 0xd6, 0xad, 0x68, 0x5d, 0x9f, 0x88, 0x91, 0xc0, 0x7f, 0xa0, 0x15, 0xa9, 0xa8, 0xca, + 0x64, 0x7b, 0xd9, 0x88, 0x1d, 0x3d, 0x84, 0x98, 0x21, 0x0c, 0x1e, 0x59, 0xb9, 0x95, 0xfc, 0x4c, + 0xac, 0x50, 0xe7, 0x66, 0x09, 0x7d, 0x54, 0x15, 0x1f, 0x24, 0xbc, 0xcf, 0x34, 0x11, 0xfe, 0x11, + 0xd5, 0xd5, 0x28, 0x05, 0xb3, 0xd3, 0xd5, 0xe0, 0x8b, 0xa2, 0xd9, 0xb3, 0x51, 0x0a, 0x6f, 0xc6, + 0xdb, 0x4f, 0x17, 0x40, 0x74, 0x8a, 0x18, 0x10, 0xfe, 0xa1, 0x9c, 0x63, 0xc9, 0xc0, 0x3f, 0xbd, + 0x2b, 0xfe, 0x66, 0xbc, 0xbd, 0x51, 0xc2, 0xee, 0xf6, 0x83, 0x87, 0x08, 0x5f, 0x51, 0xa9, 0xce, + 0x04, 0xe5, 0x32, 0xa7, 0x65, 0x31, 0xd8, 0x75, 0x7c, 0xf9, 0x7e, 0x37, 0xab, 0x11, 0xc1, 0x96, + 0x95, 0xc4, 0xc7, 0x73, 0x6c, 0x64, 0x81, 0x02, 0xfe, 0x1c, 0xad, 0x08, 0xa0, 0x32, 0xe1, 0xed, + 0xba, 0x69, 0xb9, 0xdc, 0x17, 0x31, 0x51, 0x62, 0xb3, 0xf8, 0x19, 0x6a, 0xc6, 0x20, 0x25, 0x8d, + 0xa0, 0xdd, 0x30, 0x85, 0x1b, 0xb6, 0xb0, 0x79, 0x92, 0x87, 0x49, 0x91, 0xef, 0xdc, 0x38, 0xe8, + 0x51, 0xb5, 0xa7, 0x63, 0x26, 0x15, 0xfe, 0x75, 0xce, 0xad, 0xde, 0xfb, 0xcd, 0xa4, 0xd1, 0xc6, + 0xab, 0x9b, 0x56, 0xae, 0x55, 0x44, 0xa6, 0x9c, 0x7a, 0x89, 0x1a, 0x4c, 0x41, 0xac, 0xb7, 0xbe, + 0xbc, 0xbb, 0xf6, 0xe2, 0xe0, 0x01, 0xdc, 0x13, 0x7c, 0x60, 0xf5, 0x1a, 0x47, 0x9a, 0x99, 0xe4, + 0x02, 0x9d, 0xff, 0x96, 0xa7, 0x47, 0xd3, 0x0e, 0xc6, 0x29, 0x6a, 0xca, 0xfc, 0x68, 0x27, 0xbb, + 0xbf, 0x79, 0x2d, 0x2d, 0x81, 0x0b, 0x10, 0xc0, 0x43, 0x08, 0xd6, 0xf4, 0x7e, 0x8b, 0x68, 0x21, + 0x83, 0x3f, 0x43, 0x8d, 0x48, 0x24, 0x59, 0x6a, 0x4d, 0x56, 0x76, 0x7a, 0xa8, 0x83, 0x24, 0xcf, + 0xe9, 0xfb, 0x1a, 0x82, 0x90, 0x2c, 0xe1, 0xc6, 0x44, 0x53, 0xf7, 0x75, 0x9e, 0x87, 0x49, 0x91, + 0xc7, 0x5d, 0xf4, 0x84, 0x71, 0x09, 0x61, 0x26, 0xa0, 0x3b, 0x60, 0xe9, 0xd9, 0x71, 0xf7, 0x1c, + 0x04, 0xbb, 0x18, 0x19, 0x47, 0xb4, 0x82, 0x4f, 0x2c, 0xf0, 0xc9, 0xd1, 0xa2, 0x22, 0xb2, 0x18, + 0x8b, 0x77, 0x51, 0x2b, 0xa4, 0x41, 0xc6, 0xfb, 0x57, 0xb9, 0x61, 0xd6, 0x83, 0x75, 0x7d, 0x7b, + 0x07, 0xfb, 0x79, 0x8c, 0x94, 0x59, 0x7c, 0x8a, 0x1e, 0x9b, 0x96, 0x4f, 0x05, 0x4b, 0x04, 0x53, + 0xa3, 0x13, 0xc6, 0x59, 0x9c, 0xc5, 0xed, 0xe6, 0x8e, 0xb3, 0xdb, 0x08, 0x3e, 0xb6, 0xea, 0x8f, + 0x0f, 0x17, 0xd4, 0x90, 0x85, 0x48, 0xbc, 0x8f, 0x36, 0xec, 0x6c, 0x45, 0xa6, 0xdd, 0x32, 0x64, + 0x4f, 0x2d, 0xd9, 0xc6, 0xf9, 0xdd, 0x34, 0x99, 0xad, 0xef, 0xfc, 0xe5, 0xa0, 0xcd, 0xd9, 0xb7, + 0x04, 0xbf, 0x72, 0x10, 0x0a, 0x8b, 0xef, 0x57, 0xb6, 0x1d, 0xe3, 0xb6, 0xe3, 0x07, 0x70, 0x5b, + 0xf9, 0x28, 0x54, 0x4f, 0x72, 0x19, 0x92, 0x64, 0x4a, 0xb3, 0x03, 0x68, 0x73, 0xd6, 0x25, 0xd8, + 0x47, 0xab, 0x9c, 0xc6, 0x20, 0x53, 0x1a, 0x16, 0xef, 0xd6, 0x87, 0x96, 0x66, 0xf5, 0xa7, 0x22, + 0x41, 0xaa, 0x1a, 0xbc, 0x83, 0xea, 0xfa, 0x60, 0xfd, 0x53, 0x3e, 0xc8, 0xba, 0x96, 0x98, 0x4c, + 0xf0, 0xec, 0xfa, 0xd6, 0xad, 0xbd, 0xbe, 0x75, 0x6b, 0x37, 0xb7, 0x6e, 0xed, 0xd5, 0xc4, 0x75, + 0xae, 0x27, 0xae, 0xf3, 0x7a, 0xe2, 0x3a, 0xff, 0x4c, 0x5c, 0xe7, 0xcf, 0x7f, 0xdd, 0xda, 0x2f, + 0x4d, 0x3b, 0xc5, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x1f, 0x2d, 0x18, 0xd6, 0x36, 0x08, 0x00, 0x00, } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/generated.proto b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/generated.proto index a8f712975e7c5..10b7b2799ba7b 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/generated.proto +++ b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/generated.proto @@ -91,14 +91,19 @@ message APIServiceSpec { // CABundle is a PEM encoded CA bundle which will be used to validate an API server's serving certificate. optional bytes caBundle = 5; - // Priority controls the ordering of this API group in the overall discovery document that gets served. - // Client tools like `kubectl` use this ordering to derive preference, so this ordering mechanism is important. - // Values must be between 1 and 1000 - // The primary sort is based on priority, ordered lowest number to highest (10 before 20). + // GroupPriorityMininum is the priority this group should have at least. Higher priority means that the group is prefered by clients over lower priority ones. + // Note that other versions of this group might specify even higher GroupPriorityMininum values such that the whole group gets a higher priority. + // The primary sort is based on GroupPriorityMinimum, ordered highest number to lowest (20 before 10). // The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo) - // We'd recommend something like: *.k8s.io (except extensions) at 100, extensions at 150 - // PaaSes (OpenShift, Deis) are recommended to be in the 200s - optional int64 priority = 6; + // We'd recommend something like: *.k8s.io (except extensions) at 18000 and + // PaaSes (OpenShift, Deis) are recommended to be in the 2000s + optional int32 groupPriorityMinimum = 7; + + // VersionPriority controls the ordering of this API version inside of its group. Must be greater than zero. + // The primary sort is based on VersionPriority, ordered highest to lowest (20 before 10). + // The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo) + // Since it's inside of a group, the number can be small, probably in the 10s. + optional int32 versionPriority = 8; } // APIServiceStatus contains derived information about an API server diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/types.go b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/types.go index 80f2520885ee9..095758cfb0bab 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/types.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/types.go @@ -53,14 +53,22 @@ type APIServiceSpec struct { // CABundle is a PEM encoded CA bundle which will be used to validate an API server's serving certificate. CABundle []byte `json:"caBundle" protobuf:"bytes,5,opt,name=caBundle"` - // Priority controls the ordering of this API group in the overall discovery document that gets served. - // Client tools like `kubectl` use this ordering to derive preference, so this ordering mechanism is important. - // Values must be between 1 and 1000 - // The primary sort is based on priority, ordered lowest number to highest (10 before 20). + // GroupPriorityMininum is the priority this group should have at least. Higher priority means that the group is prefered by clients over lower priority ones. + // Note that other versions of this group might specify even higher GroupPriorityMininum values such that the whole group gets a higher priority. + // The primary sort is based on GroupPriorityMinimum, ordered highest number to lowest (20 before 10). // The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo) - // We'd recommend something like: *.k8s.io (except extensions) at 100, extensions at 150 - // PaaSes (OpenShift, Deis) are recommended to be in the 200s - Priority int64 `json:"priority" protobuf:"varint,6,opt,name=priority"` + // We'd recommend something like: *.k8s.io (except extensions) at 18000 and + // PaaSes (OpenShift, Deis) are recommended to be in the 2000s + GroupPriorityMinimum int32 `json:"groupPriorityMinimum" protobuf:"varint,7,opt,name=groupPriorityMinimum"` + + // VersionPriority controls the ordering of this API version inside of its group. Must be greater than zero. + // The primary sort is based on VersionPriority, ordered highest to lowest (20 before 10). + // The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo) + // Since it's inside of a group, the number can be small, probably in the 10s. + VersionPriority int32 `json:"versionPriority" protobuf:"varint,8,opt,name=versionPriority"` + + // leaving this here so everyone remembers why proto index 6 is skipped + // Priority int64 `json:"priority" protobuf:"varint,6,opt,name=priority"` } type ConditionStatus string diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/zz_generated.conversion.go b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/zz_generated.conversion.go index 086104e29aa93..702430d294fab 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/zz_generated.conversion.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1/zz_generated.conversion.go @@ -142,7 +142,8 @@ func autoConvert_v1beta1_APIServiceSpec_To_apiregistration_APIServiceSpec(in *AP out.Version = in.Version out.InsecureSkipTLSVerify = in.InsecureSkipTLSVerify out.CABundle = *(*[]byte)(unsafe.Pointer(&in.CABundle)) - out.Priority = in.Priority + out.GroupPriorityMinimum = in.GroupPriorityMinimum + out.VersionPriority = in.VersionPriority return nil } @@ -161,7 +162,8 @@ func autoConvert_apiregistration_APIServiceSpec_To_v1beta1_APIServiceSpec(in *ap } else { out.CABundle = *(*[]byte)(unsafe.Pointer(&in.CABundle)) } - out.Priority = in.Priority + out.GroupPriorityMinimum = in.GroupPriorityMinimum + out.VersionPriority = in.VersionPriority return nil } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/validation/validation.go b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/validation/validation.go index 22cd157a8798a..725cf71337f6f 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/validation/validation.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/validation/validation.go @@ -58,9 +58,11 @@ func ValidateAPIService(apiService *apiregistration.APIService) field.ErrorList allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "version"), apiService.Spec.Version, errString)) } - if apiService.Spec.Priority <= 0 || apiService.Spec.Priority > 1000 { - allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "priority"), apiService.Spec.Priority, "priority must be positive and less than 1000")) - + if apiService.Spec.GroupPriorityMinimum <= 0 || apiService.Spec.GroupPriorityMinimum > 20000 { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "groupPriorityMinimum"), apiService.Spec.GroupPriorityMinimum, "must be positive and less than 20000")) + } + if apiService.Spec.VersionPriority <= 0 || apiService.Spec.VersionPriority > 1000 { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "versionPriority"), apiService.Spec.VersionPriority, "must be positive and less than 1000")) } if apiService.Spec.Service == nil { diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_apis.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_apis.go index 352d43cc69811..e7b15b1882140 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_apis.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_apis.go @@ -73,7 +73,7 @@ func (r *apisHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - apiServicesByGroup := apiregistrationapi.SortedByGroup(apiServices) + apiServicesByGroup := apiregistrationapi.SortedByGroupAndVersion(apiServices) for _, apiGroupServers := range apiServicesByGroup { // skip the legacy group if len(apiGroupServers[0].Spec.Group) == 0 { @@ -91,7 +91,7 @@ func (r *apisHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { // convertToDiscoveryAPIGroup takes apiservices in a single group and returns a discovery compatible object. // if none of the services are available, it will return nil. func convertToDiscoveryAPIGroup(apiServices []*apiregistrationapi.APIService) *metav1.APIGroup { - apiServicesByGroup := apiregistrationapi.SortedByGroup(apiServices)[0] + apiServicesByGroup := apiregistrationapi.SortedByGroupAndVersion(apiServices)[0] var discoveryGroup *metav1.APIGroup diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_apis_test.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_apis_test.go index 55a9ebf9c8753..d3aba698bd37c 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_apis_test.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_apis_test.go @@ -60,9 +60,9 @@ func TestAPIs(t *testing.T) { Namespace: "ns", Name: "api", }, - Group: "foo", - Version: "v1", - Priority: 10, + Group: "foo", + Version: "v1", + GroupPriorityMinimum: 11, }, Status: apiregistration.APIServiceStatus{ Conditions: []apiregistration.APIServiceCondition{ @@ -77,9 +77,9 @@ func TestAPIs(t *testing.T) { Namespace: "ns", Name: "api", }, - Group: "bar", - Version: "v1", - Priority: 11, + Group: "bar", + Version: "v1", + GroupPriorityMinimum: 10, }, Status: apiregistration.APIServiceStatus{ Conditions: []apiregistration.APIServiceCondition{ @@ -131,9 +131,10 @@ func TestAPIs(t *testing.T) { Namespace: "ns", Name: "api", }, - Group: "foo", - Version: "v1", - Priority: 20, + Group: "foo", + Version: "v1", + GroupPriorityMinimum: 20, + VersionPriority: 10, }, Status: apiregistration.APIServiceStatus{ Conditions: []apiregistration.APIServiceCondition{ @@ -148,9 +149,9 @@ func TestAPIs(t *testing.T) { Namespace: "ns", Name: "api", }, - Group: "bar", - Version: "v2", - Priority: 11, + Group: "bar", + Version: "v2", + GroupPriorityMinimum: 11, }, Status: apiregistration.APIServiceStatus{ Conditions: []apiregistration.APIServiceCondition{ @@ -165,9 +166,10 @@ func TestAPIs(t *testing.T) { Namespace: "ns", Name: "api", }, - Group: "foo", - Version: "v2", - Priority: 1, + Group: "foo", + Version: "v2", + GroupPriorityMinimum: 1, + VersionPriority: 15, }, Status: apiregistration.APIServiceStatus{ Conditions: []apiregistration.APIServiceCondition{ @@ -182,9 +184,9 @@ func TestAPIs(t *testing.T) { Namespace: "ns", Name: "api", }, - Group: "bar", - Version: "v1", - Priority: 11, + Group: "bar", + Version: "v1", + GroupPriorityMinimum: 11, }, Status: apiregistration.APIServiceStatus{ Conditions: []apiregistration.APIServiceCondition{ @@ -336,9 +338,10 @@ func TestAPIGroup(t *testing.T) { Namespace: "ns", Name: "api", }, - Group: "foo", - Version: "v1", - Priority: 20, + Group: "foo", + Version: "v1", + GroupPriorityMinimum: 20, + VersionPriority: 10, }, Status: apiregistration.APIServiceStatus{ Conditions: []apiregistration.APIServiceCondition{ @@ -353,9 +356,9 @@ func TestAPIGroup(t *testing.T) { Namespace: "ns", Name: "api", }, - Group: "bar", - Version: "v2", - Priority: 11, + Group: "bar", + Version: "v2", + GroupPriorityMinimum: 11, }, Status: apiregistration.APIServiceStatus{ Conditions: []apiregistration.APIServiceCondition{ @@ -370,9 +373,10 @@ func TestAPIGroup(t *testing.T) { Namespace: "ns", Name: "api", }, - Group: "foo", - Version: "v2", - Priority: 1, + Group: "foo", + Version: "v2", + GroupPriorityMinimum: 1, + VersionPriority: 15, }, Status: apiregistration.APIServiceStatus{ Conditions: []apiregistration.APIServiceCondition{ @@ -387,9 +391,9 @@ func TestAPIGroup(t *testing.T) { Namespace: "ns", Name: "api", }, - Group: "bar", - Version: "v1", - Priority: 11, + Group: "bar", + Version: "v1", + GroupPriorityMinimum: 11, }, Status: apiregistration.APIServiceStatus{ Conditions: []apiregistration.APIServiceCondition{ diff --git a/test/e2e/garbage_collector.go b/test/e2e/garbage_collector.go index 0efc60f21f1e6..bbe655003fc51 100644 --- a/test/e2e/garbage_collector.go +++ b/test/e2e/garbage_collector.go @@ -106,14 +106,15 @@ func newOwnerRC(f *framework.Framework, name string, replicas int32) *v1.Replica } } -// verifyRemainingDeploymentsAndReplicaSets verifies if the number -// of the remaining deployment and rs are deploymentNum and rsNum. -// It returns error if the communication with the API server fails. -func verifyRemainingDeploymentsAndReplicaSets( +// verifyRemainingDeploymentsReplicaSetsPods verifies if the number +// of the remaining deployments, replica set and pods are deploymentNum, +// rsNum and podNum. It returns error if the communication with the API +// server fails. +func verifyRemainingDeploymentsReplicaSetsPods( f *framework.Framework, clientSet clientset.Interface, deployment *v1beta1.Deployment, - deploymentNum, rsNum int, + deploymentNum, rsNum, podNum int, ) (bool, error) { var ret = true rs, err := clientSet.Extensions().ReplicaSets(f.Namespace.Name).List(metav1.ListOptions{}) @@ -132,6 +133,15 @@ func verifyRemainingDeploymentsAndReplicaSets( ret = false By(fmt.Sprintf("expected %d Deploymentss, got %d Deployments", deploymentNum, len(deployments.Items))) } + pods, err := clientSet.CoreV1().Pods(f.Namespace.Name).List(metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("Failed to list pods: %v", err) + } + if len(pods.Items) != podNum { + ret = false + By(fmt.Sprintf("expected %v Pods, got %d Pods", podNum, len(pods.Items))) + } + return ret, nil } @@ -397,7 +407,7 @@ var _ = framework.KubeDescribe("Garbage collector", func() { } By("wait for all rs to be garbage collected") err = wait.PollImmediate(500*time.Millisecond, 1*time.Minute, func() (bool, error) { - return verifyRemainingDeploymentsAndReplicaSets(f, clientSet, deployment, 0, 0) + return verifyRemainingDeploymentsReplicaSetsPods(f, clientSet, deployment, 0, 0, 0) }) if err == wait.ErrWaitTimeout { err = fmt.Errorf("Failed to wait for all rs to be garbage collected: %v", err) @@ -446,7 +456,7 @@ var _ = framework.KubeDescribe("Garbage collector", func() { } By("wait for 2 Minute to see if the garbage collector mistakenly deletes the rs") err = wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { - return verifyRemainingDeploymentsAndReplicaSets(f, clientSet, deployment, 0, 1) + return verifyRemainingDeploymentsReplicaSetsPods(f, clientSet, deployment, 0, 1, 2) }) if err != nil { err = fmt.Errorf("Failed to wait to see if the garbage collecter mistakenly deletes the rs: %v", err) diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index f2fb02a2fa08c..a940efe81f939 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -315,10 +315,11 @@ func TestAggregatedAPIServer(t *testing.T) { Namespace: "kube-wardle", Name: "api", }, - Group: "wardle.k8s.io", - Version: "v1alpha1", - CABundle: wardleCA, - Priority: 200, + Group: "wardle.k8s.io", + Version: "v1alpha1", + CABundle: wardleCA, + GroupPriorityMinimum: 200, + VersionPriority: 200, }, }) if err != nil { @@ -337,9 +338,10 @@ func TestAggregatedAPIServer(t *testing.T) { Spec: apiregistrationv1beta1.APIServiceSpec{ // register this as a loca service so it doesn't try to lookup the default kubernetes service // which will have an unroutable IP address since its fake. - Group: "", - Version: "v1", - Priority: 100, + Group: "", + Version: "v1", + GroupPriorityMinimum: 100, + VersionPriority: 100, }, }) if err != nil {