Skip to content
This repository has been archived by the owner on Feb 11, 2021. It is now read-only.

Commit

Permalink
Merge pull request kubernetes#46864 from luxas/kubeadm_fix_v16_csrs
Browse files Browse the repository at this point in the history 
Automatic merge from submit-queue (batch tested with PRs 46897, 46899, 46864, 46854, 46875)

kubeadm: Make kubeadm use the right CSR approver for the right version

**What this PR does / why we need it**:

fixes regression caused in: kubernetes#45619

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

Fixes: kubernetes/kubeadm#289

**Special notes for your reviewer**:

cc @pipejakob our e2e CI should probably go green after this change

**Release note**:

```release-note
NONE
```
@mikedanese @pipejakob @timothysc @liggitt
  • Loading branch information
Kubernetes Submit Queue authored Jun 6, 2017
2 parents 3338d78 + de2ef8f commit f04a774
Show file tree
Hide file tree
Showing 8 changed files with 67 additions and 30 deletions.
2 changes: 1 addition & 1 deletion cmd/kubeadm/app/cmd/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ go_library(
"//pkg/printers:go_default_library",
"//pkg/util/i18n:go_default_library",
"//pkg/util/initsystem:go_default_library",
"//pkg/util/version:go_default_library",
"//pkg/version:go_default_library",
"//vendor/github.com/blang/semver:go_default_library",
"//vendor/github.com/ghodss/yaml:go_default_library",
"//vendor/github.com/renstrom/dedent:go_default_library",
"//vendor/github.com/spf13/cobra:go_default_library",
Expand Down
13 changes: 5 additions & 8 deletions cmd/kubeadm/app/cmd/defaults.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,9 @@ import (
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"

"github.com/blang/semver"
"k8s.io/kubernetes/pkg/util/version"
)

var minK8sVersion = semver.MustParse(kubeadmconstants.MinimumControlPlaneVersion)

func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {

// Choose the right address for the API Server to advertise. If the advertise address is localhost or 0.0.0.0, the default interface's IP address is used
Expand All @@ -48,13 +45,13 @@ func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
}
cfg.KubernetesVersion = ver

// Omit the "v" in the beginning, otherwise semver will fail
k8sVersion, err := semver.Parse(cfg.KubernetesVersion[1:])
// Parse the given kubernetes version and make sure it's higher than the lowest supported
k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
if err != nil {
return fmt.Errorf("couldn't parse kubernetes version %q: %v", cfg.KubernetesVersion, err)
}
if k8sVersion.LT(minK8sVersion) {
return fmt.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= v%s. Current version: %s", kubeadmconstants.MinimumControlPlaneVersion, cfg.KubernetesVersion)
if k8sVersion.LessThan(kubeadmconstants.MinimumControlPlaneVersion) {
return fmt.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= %s. Current version: %s", kubeadmconstants.MinimumControlPlaneVersion.String(), cfg.KubernetesVersion)
}

fmt.Printf("[init] Using Kubernetes version: %s\n", cfg.KubernetesVersion)
Expand Down
5 changes: 4 additions & 1 deletion cmd/kubeadm/app/constants/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@ go_library(
name = "go_default_library",
srcs = ["constants.go"],
tags = ["automanaged"],
deps = ["//vendor/k8s.io/client-go/pkg/api/v1:go_default_library"],
deps = [
"//pkg/util/version:go_default_library",
"//vendor/k8s.io/client-go/pkg/api/v1:go_default_library",
],
)

filegroup(
Expand Down
13 changes: 10 additions & 3 deletions cmd/kubeadm/app/constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"time"

"k8s.io/client-go/pkg/api/v1"
"k8s.io/kubernetes/pkg/util/version"
)

const (
Expand Down Expand Up @@ -58,9 +59,6 @@ const (
ControllerManagerKubeConfigFileName = "controller-manager.conf"
SchedulerKubeConfigFileName = "scheduler.conf"

// Important: a "v"-prefix shouldn't exist here; semver doesn't allow that
MinimumControlPlaneVersion = "1.6.0-beta.3"

// Some well-known users and groups in the core Kubernetes authorization system

ControllerManagerUser = "system:kube-controller-manager"
Expand Down Expand Up @@ -109,4 +107,13 @@ var (

// DefaultTokenUsages specifies the default functions a token will get
DefaultTokenUsages = []string{"signing", "authentication"}

// MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy
MinimumControlPlaneVersion = version.MustParseSemantic("v1.6.0")

// MinimumCSRSARApproverVersion specifies the minimum kubernetes version that can be used for enabling the new-in-v1.7 CSR approver based on a SubjectAccessReview
MinimumCSRSARApproverVersion = version.MustParseSemantic("v1.7.0-beta.0")

// MinimumAPIAggregationVersion specifies the minimum kubernetes version that can be used enabling the API aggregation in the apiserver and the front proxy flags
MinimumAPIAggregationVersion = version.MustParseSemantic("v1.7.0-alpha.1")
)
1 change: 1 addition & 0 deletions cmd/kubeadm/app/master/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ go_library(
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/images:go_default_library",
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
"//pkg/bootstrap/api:go_default_library",
"//pkg/kubeapiserver/authorizer/modes:go_default_library",
"//pkg/kubectl/cmd/util:go_default_library",
"//pkg/util/version:go_default_library",
Expand Down
16 changes: 9 additions & 7 deletions cmd/kubeadm/app/master/manifests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/images"
bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
"k8s.io/kubernetes/pkg/util/version"
Expand All @@ -53,10 +54,6 @@ const (
kubeProxy = "kube-proxy"
)

var (
v170 = version.MustParseSemantic("v1.7.0-alpha.0")
)

// WriteStaticPodManifests builds manifest objects based on user provided configuration and then dumps it to disk
// where kubelet will pick and schedule them.
func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
Expand Down Expand Up @@ -97,7 +94,7 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
kubeControllerManager: componentPod(api.Container{
Name: kubeControllerManager,
Image: images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
Command: getControllerManagerCommand(cfg, false),
Command: getControllerManagerCommand(cfg, false, k8sVersion),
VolumeMounts: volumeMounts,
LivenessProbe: componentProbe(10252, "/healthz", api.URISchemeHTTP),
Resources: componentResources("200m"),
Expand Down Expand Up @@ -349,7 +346,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k
"requestheader-client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName),
"requestheader-allowed-names": "front-proxy-client",
}
if k8sVersion.AtLeast(v170) {
if k8sVersion.AtLeast(kubeadmconstants.MinimumAPIAggregationVersion) {
// add options which allow the kube-apiserver to act as a front-proxy to aggregated API servers
defaultArguments["proxy-client-cert-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName)
defaultArguments["proxy-client-key-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName)
Expand Down Expand Up @@ -409,7 +406,7 @@ func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string {
return command
}

func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) []string {
func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k8sVersion *version.Version) []string {
var command []string

// self-hosted controller-manager needs to wait on a lock
Expand All @@ -428,6 +425,11 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted
"use-service-account-credentials": "true",
"controllers": "*,bootstrapsigner,tokencleaner",
}
if k8sVersion.LessThan(kubeadmconstants.MinimumCSRSARApproverVersion) {
// enable the former CSR group approver for v1.6 clusters.
// TODO(luxas): Remove this once we're targeting v1.8 at HEAD
defaultArguments["insecure-experimental-approve-all-kubelet-csrs-for-group"] = bootstrapapi.BootstrapGroup
}

command = getComponentBaseCommand(controllerManager)
command = append(command, getExtraParameters(cfg.ControllerManagerExtraArgs, defaultArguments)...)
Expand Down
36 changes: 29 additions & 7 deletions cmd/kubeadm/app/master/manifests_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -667,7 +667,8 @@ func TestGetControllerManagerCommand(t *testing.T) {
}{
{
cfg: &kubeadmapi.MasterConfiguration{
CertificatesDir: testCertsDir,
CertificatesDir: testCertsDir,
KubernetesVersion: "v1.7.0",
},
expected: []string{
"kube-controller-manager",
Expand All @@ -684,8 +685,28 @@ func TestGetControllerManagerCommand(t *testing.T) {
},
{
cfg: &kubeadmapi.MasterConfiguration{
CloudProvider: "foo",
CertificatesDir: testCertsDir,
CertificatesDir: testCertsDir,
KubernetesVersion: "v1.6.4",
},
expected: []string{
"kube-controller-manager",
"--address=127.0.0.1",
"--leader-elect=true",
"--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/controller-manager.conf",
"--root-ca-file=" + testCertsDir + "/ca.crt",
"--service-account-private-key-file=" + testCertsDir + "/sa.key",
"--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
"--cluster-signing-key-file=" + testCertsDir + "/ca.key",
"--use-service-account-credentials=true",
"--controllers=*,bootstrapsigner,tokencleaner",
"--insecure-experimental-approve-all-kubelet-csrs-for-group=system:bootstrappers",
},
},
{
cfg: &kubeadmapi.MasterConfiguration{
CloudProvider: "foo",
CertificatesDir: testCertsDir,
KubernetesVersion: "v1.7.0",
},
expected: []string{
"kube-controller-manager",
Expand All @@ -703,8 +724,9 @@ func TestGetControllerManagerCommand(t *testing.T) {
},
{
cfg: &kubeadmapi.MasterConfiguration{
Networking: kubeadm.Networking{PodSubnet: "bar"},
CertificatesDir: testCertsDir,
Networking: kubeadm.Networking{PodSubnet: "bar"},
CertificatesDir: testCertsDir,
KubernetesVersion: "v1.7.0",
},
expected: []string{
"kube-controller-manager",
Expand All @@ -724,7 +746,7 @@ func TestGetControllerManagerCommand(t *testing.T) {
}

for _, rt := range tests {
actual := getControllerManagerCommand(rt.cfg, false)
actual := getControllerManagerCommand(rt.cfg, false, version.MustParseSemantic(rt.cfg.KubernetesVersion))
sort.Strings(actual)
sort.Strings(rt.expected)
if !reflect.DeepEqual(actual, rt.expected) {
Expand Down Expand Up @@ -935,7 +957,7 @@ func TestVersionCompare(t *testing.T) {
"v1.7.1",
}
for _, v := range versions {
if !version.MustParseSemantic(v).AtLeast(v170) {
if !version.MustParseSemantic(v).AtLeast(kubeadmconstants.MinimumAPIAggregationVersion) {
t.Errorf("err")
}
}
Expand Down
11 changes: 8 additions & 3 deletions cmd/kubeadm/app/master/selfhosted.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,12 @@ func launchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clie
func launchSelfHostedControllerManager(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
start := time.Now()

ctrlMgr := getControllerManagerDeployment(cfg, volumes, volumeMounts)
kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
if err != nil {
return err
}

ctrlMgr := getControllerManagerDeployment(cfg, volumes, volumeMounts, kubeVersion)
if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(&ctrlMgr); err != nil {
return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeControllerManager, err)
}
Expand Down Expand Up @@ -232,7 +237,7 @@ func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, vo
return ds
}

func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount) ext.Deployment {
func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) ext.Deployment {
d := ext.Deployment{
TypeMeta: metav1.TypeMeta{
APIVersion: "extensions/v1beta1",
Expand Down Expand Up @@ -268,7 +273,7 @@ func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes
{
Name: "self-hosted-" + kubeControllerManager,
Image: images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
Command: getControllerManagerCommand(cfg, true),
Command: getControllerManagerCommand(cfg, true, kubeVersion),
VolumeMounts: volumeMounts,
LivenessProbe: componentProbe(10252, "/healthz", v1.URISchemeHTTP),
Resources: componentResources("200m"),
Expand Down

0 comments on commit f04a774

Please sign in to comment.