Skip to content

Commit

Permalink
Merge pull request #46879 from luxas/kubeadm_enable_node_authorizer
Browse files Browse the repository at this point in the history 
Automatic merge from submit-queue

kubeadm: Enable the Node Authorizer/Admission plugin in v1.7

**What this PR does / why we need it**:

This is similar to #46796, but for kubeadm.
Basically it was a part of #46796, but there were some other upgradability and compability concerns for kubeadm I took care of while working today.

Example:

```console
$ kubeadm init --kubernetes-version v1.7.0-beta.0
[kubeadm] WARNING: kubeadm is in beta, please do not use it for production clusters.
[init] Using Kubernetes version: v1.7.0-beta.0
[init] Using Authorization mode: [RBAC Node]
...
$ sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf get secret foo
Error from server (Forbidden): User "system:node:thegopher" cannot get secrets in the namespace "default".: "no path found to object" (get secrets foo)

$ echo '{"apiVersion":"v1","kind":"Node","metadata":{"name":"foo"}}' | sudo kubectl create -f - --kubeconfig=/etc/kubernetes/kubelet.conf
Error from server (Forbidden): error when creating "STDIN": nodes "foo" is forbidden: node thegopher cannot modify node foo
```

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

**Special notes for your reviewer**:

Depends on #46864 (uses that PR as a base, will rebase once it's merged)

Please only review the second commit. Will also fix tests in a minute.

**Release note**:

```release-note
kubeadm: Enable the Node Authorizer/Admission plugin in v1.7 
```
@mikedanese @liggitt @pipejakob @roberthbailey @jbeda @timothysc
  • Loading branch information
Kubernetes Submit Queue authored Jun 8, 2017
2 parents b77ed78 + b7700ef commit 2bcd3d1
Show file tree
Hide file tree
Showing 12 changed files with 176 additions and 33 deletions.
4 changes: 4 additions & 0 deletions cmd/kubeadm/app/cmd/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ go_library(
"//cmd/kubeadm/app/util/token:go_default_library",
"//pkg/api:go_default_library",
"//pkg/bootstrap/api:go_default_library",
"//pkg/kubeapiserver/authorizer/modes:go_default_library",
"//pkg/kubectl/cmd/util:go_default_library",
"//pkg/printers:go_default_library",
"//pkg/util/i18n:go_default_library",
Expand All @@ -54,6 +55,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
Expand All @@ -65,6 +67,7 @@ go_library(
go_test(
name = "go_default_test",
srcs = [
"defaults_test.go",
"reset_test.go",
"token_test.go",
],
Expand All @@ -73,6 +76,7 @@ go_test(
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//pkg/util/version:go_default_library",
],
)

Expand Down
18 changes: 17 additions & 1 deletion cmd/kubeadm/app/cmd/defaults.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,12 @@ import (
"net"

netutil "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/sets"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
"k8s.io/kubernetes/pkg/util/version"
)

Expand Down Expand Up @@ -54,8 +56,12 @@ func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
return fmt.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= %s. Current version: %s", kubeadmconstants.MinimumControlPlaneVersion.String(), cfg.KubernetesVersion)
}

// Defaulting is made here because it's dependent on the version currently, which is determined above
// TODO(luxas): Cleanup this once we have dropped v1.6 support and move this code into the API group defaulting
cfg.AuthorizationModes = defaultAuthorizationModes(cfg.AuthorizationModes, k8sVersion)

fmt.Printf("[init] Using Kubernetes version: %s\n", cfg.KubernetesVersion)
fmt.Printf("[init] Using Authorization mode: %v\n", cfg.AuthorizationModes)
fmt.Printf("[init] Using Authorization modes: %v\n", cfg.AuthorizationModes)

// Warn about the limitations with the current cloudprovider solution.
if cfg.CloudProvider != "" {
Expand All @@ -73,3 +79,13 @@ func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {

return nil
}

func defaultAuthorizationModes(authzModes []string, k8sVersion *version.Version) []string {
if k8sVersion.AtLeast(kubeadmconstants.MinimumNodeAuthorizerVersion) {
strset := sets.NewString(authzModes...)
if !strset.Has(authzmodes.ModeNode) {
return append([]string{authzmodes.ModeNode}, authzModes...)
}
}
return authzModes
}
54 changes: 54 additions & 0 deletions cmd/kubeadm/app/cmd/defaults_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
"strings"
"testing"

"k8s.io/kubernetes/pkg/util/version"
)

func TestDefaultAuthorizationModes(t *testing.T) {
var tests = []struct {
authzModes []string
version string
expected []string
}{
{[]string{"RBAC"}, "v1.6.0", []string{"RBAC"}},
{[]string{"RBAC", "ABAC"}, "v1.6.4", []string{"RBAC", "ABAC"}},
{[]string{"RBAC", "ABAC"}, "v1.7.0-beta.0", []string{"RBAC", "ABAC"}},
{[]string{"RBAC"}, "v1.7.0", []string{"Node", "RBAC"}},
{[]string{"RBAC", "Webhook"}, "v1.7.0-beta.1", []string{"Node", "RBAC", "Webhook"}},
{[]string{"RBAC", "Webhook", "Node"}, "v1.7.0", []string{"RBAC", "Webhook", "Node"}},
{[]string{"Node", "RBAC", "Webhook"}, "v1.7.0", []string{"Node", "RBAC", "Webhook"}},
}
for _, rt := range tests {
k8sVersion, err := version.ParseSemantic(rt.version)
if err != nil {
t.Fatalf("couldn't parse given version")
}
actual := defaultAuthorizationModes(rt.authzModes, k8sVersion)
if strings.Join(actual, ",") != strings.Join(rt.expected, ",") {
t.Errorf(
"failed TestDefaultAuthorizationModes:\n\texpected: %s\n\t actual: %s",
strings.Join(rt.expected, ","),
strings.Join(actual, ","),
)
}
}
}
8 changes: 7 additions & 1 deletion cmd/kubeadm/app/cmd/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/util/version"
)

var (
Expand Down Expand Up @@ -253,13 +254,18 @@ func (i *Init) Run(out io.Writer) error {

// PHASE 5: Install and deploy all addons, and configure things as necessary

k8sVersion, err := version.ParseSemantic(i.cfg.KubernetesVersion)
if err != nil {
return fmt.Errorf("couldn't parse kubernetes version %q: %v", i.cfg.KubernetesVersion, err)
}

// Create the necessary ServiceAccounts
err = apiconfigphase.CreateServiceAccounts(client)
if err != nil {
return err
}

err = apiconfigphase.CreateRBACRules(client)
err = apiconfigphase.CreateRBACRules(client, k8sVersion)
if err != nil {
return err
}
Expand Down
15 changes: 8 additions & 7 deletions cmd/kubeadm/app/constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,11 @@ const (

// Some well-known users and groups in the core Kubernetes authorization system

ControllerManagerUser = "system:kube-controller-manager"
SchedulerUser = "system:kube-scheduler"
MastersGroup = "system:masters"
NodesGroup = "system:nodes"
ControllerManagerUser = "system:kube-controller-manager"
SchedulerUser = "system:kube-scheduler"
MastersGroup = "system:masters"
NodesGroup = "system:nodes"
NodesClusterRoleBinding = "system:node"

// Constants for what we name our ServiceAccounts with limited access to the cluster in case of RBAC
KubeDNSServiceAccountName = "kube-dns"
Expand All @@ -89,9 +90,6 @@ const (

// MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports
MinExternalEtcdVersion = "3.0.14"

// DefaultAdmissionControl specifies the default admission control options that will be used
DefaultAdmissionControl = "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds"
)

var (
Expand All @@ -116,4 +114,7 @@ var (

// MinimumAPIAggregationVersion specifies the minimum kubernetes version that can be used enabling the API aggregation in the apiserver and the front proxy flags
MinimumAPIAggregationVersion = version.MustParseSemantic("v1.7.0-alpha.1")

// MinimumNodeAuthorizerVersion specifies the minimum kubernetes version that can be used for enabling the node authorizer
MinimumNodeAuthorizerVersion = version.MustParseSemantic("v1.7.0-beta.1")
)
1 change: 1 addition & 0 deletions cmd/kubeadm/app/master/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/pkg/api/v1:go_default_library",
Expand Down
40 changes: 26 additions & 14 deletions cmd/kubeadm/app/master/manifests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/sets"
api "k8s.io/client-go/pkg/api/v1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
Expand All @@ -43,6 +44,9 @@ import (
const (
DefaultCloudConfigPath = "/etc/kubernetes/cloud-config"

defaultv16AdmissionControl = "NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota"
defaultv17AdmissionControl = "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota"

etcd = "etcd"
apiServer = "apiserver"
controllerManager = "controller-manager"
Expand Down Expand Up @@ -326,7 +330,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k

defaultArguments := map[string]string{
"insecure-port": "0",
"admission-control": kubeadmconstants.DefaultAdmissionControl,
"admission-control": defaultv16AdmissionControl,
"service-cluster-ip-range": cfg.Networking.ServiceSubnet,
"service-account-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName),
"client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName),
Expand All @@ -351,6 +355,10 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k
defaultArguments["proxy-client-cert-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName)
defaultArguments["proxy-client-key-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName)
}
if k8sVersion.AtLeast(kubeadmconstants.MinimumNodeAuthorizerVersion) {
// enable the NodeRestriction admission plugin
defaultArguments["admission-control"] = defaultv17AdmissionControl
}

command = getComponentBaseCommand(apiServer)
command = append(command, getExtraParameters(cfg.APIServerExtraArgs, defaultArguments)...)
Expand Down Expand Up @@ -505,21 +513,25 @@ func getSelfHostedAPIServerEnv() []api.EnvVar {

func getAuthzParameters(modes []string) []string {
command := []string{}
// RBAC is always on. If the user specified
authzModes := []string{authzmodes.ModeRBAC}
for _, authzMode := range modes {
if len(authzMode) != 0 && authzMode != authzmodes.ModeRBAC {
authzModes = append(authzModes, authzMode)
}
switch authzMode {
case authzmodes.ModeABAC:
command = append(command, "--authorization-policy-file="+kubeadmconstants.AuthorizationPolicyPath)
case authzmodes.ModeWebhook:
command = append(command, "--authorization-webhook-config-file="+kubeadmconstants.AuthorizationWebhookConfigPath)
}
strset := sets.NewString(modes...)

// RBAC must always be set, prepend that if not present in the list
// TODO(luxas): In v1.8, require the Node and RBAC authorizers to be present
// In v1.8 we can utilize the API defaulting/validation to do enforce this,
// currently this logic has to be split up into two places, which is sub-optimal.
// Cleanup work is possible in and should be done in v1.8
if !strset.Has(authzmodes.ModeRBAC) {
modes = append([]string{authzmodes.ModeRBAC}, modes...)
}

if strset.Has(authzmodes.ModeABAC) {
command = append(command, "--authorization-policy-file="+kubeadmconstants.AuthorizationPolicyPath)
}
if strset.Has(authzmodes.ModeWebhook) {
command = append(command, "--authorization-webhook-config-file="+kubeadmconstants.AuthorizationWebhookConfigPath)
}

command = append(command, "--authorization-mode="+strings.Join(authzModes, ","))
command = append(command, "--authorization-mode="+strings.Join(modes, ","))
return command
}

Expand Down
18 changes: 13 additions & 5 deletions cmd/kubeadm/app/master/manifests_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -522,7 +522,7 @@ func TestGetAPIServerCommand(t *testing.T) {
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
Expand Down Expand Up @@ -554,7 +554,7 @@ func TestGetAPIServerCommand(t *testing.T) {
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
Expand Down Expand Up @@ -587,7 +587,7 @@ func TestGetAPIServerCommand(t *testing.T) {
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
Expand Down Expand Up @@ -622,7 +622,7 @@ func TestGetAPIServerCommand(t *testing.T) {
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
Expand Down Expand Up @@ -884,7 +884,15 @@ func TestGetAuthzParameters(t *testing.T) {
{
authMode: []string{"ABAC", "RBAC", "Webhook"},
expected: []string{
"--authorization-mode=RBAC,ABAC,Webhook",
"--authorization-mode=ABAC,RBAC,Webhook",
"--authorization-policy-file=/etc/kubernetes/abac_policy.json",
"--authorization-webhook-config-file=/etc/kubernetes/webhook_authz.conf",
},
},
{
authMode: []string{"Node", "RBAC", "Webhook", "ABAC"},
expected: []string{
"--authorization-mode=Node,RBAC,Webhook,ABAC",
"--authorization-policy-file=/etc/kubernetes/abac_policy.json",
"--authorization-webhook-config-file=/etc/kubernetes/webhook_authz.conf",
},
Expand Down
1 change: 1 addition & 0 deletions cmd/kubeadm/app/phases/apiconfig/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ go_library(
"//pkg/bootstrap/api:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/util/node:go_default_library",
"//pkg/util/version:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
Expand Down
Loading

0 comments on commit 2bcd3d1

Please sign in to comment.