From 0b01cd743b0e9de4d6b19e0eb25b7b9a23ca0e8d Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Thu, 15 Jun 2017 18:05:34 +0300 Subject: [PATCH 1/2] Improve security of Juju deployed clusters --- .../templates/apilb.conf | 3 - cluster/juju/layers/kubernetes-e2e/README.md | 3 +- cluster/juju/layers/kubernetes-e2e/layer.yaml | 1 + .../juju/layers/kubernetes-e2e/metadata.yaml | 3 + .../kubernetes-e2e/reactive/kubernetes_e2e.py | 68 ++++++-- .../juju/layers/kubernetes-master/config.yaml | 5 + .../reactive/kubernetes_master.py | 162 ++++++++++++++---- .../reactive/kubernetes_worker.py | 80 +++++++-- 8 files changed, 254 insertions(+), 71 deletions(-) diff --git a/cluster/juju/layers/kubeapi-load-balancer/templates/apilb.conf b/cluster/juju/layers/kubeapi-load-balancer/templates/apilb.conf index 0cb18f58e7241..88f185acef32c 100644 --- a/cluster/juju/layers/kubeapi-load-balancer/templates/apilb.conf +++ b/cluster/juju/layers/kubeapi-load-balancer/templates/apilb.conf @@ -33,9 +33,6 @@ server { proxy_set_header Connection $http_connection; proxy_set_header X-Stream-Protocol-Version $http_x_stream_protocol_version; - proxy_ssl_certificate {{ server_certificate }}; - proxy_ssl_certificate_key {{ server_key }}; - add_header X-Stream-Protocol-Version $upstream_http_x_stream_protocol_version; proxy_pass https://target_service; diff --git a/cluster/juju/layers/kubernetes-e2e/README.md b/cluster/juju/layers/kubernetes-e2e/README.md index d60fbd0c2dc73..3f516d7fc42f7 100644 --- a/cluster/juju/layers/kubernetes-e2e/README.md +++ b/cluster/juju/layers/kubernetes-e2e/README.md @@ -21,7 +21,8 @@ and then relate the `kubernetes-e2e` charm. ```shell juju deploy kubernetes-core juju deploy cs:~containers/kubernetes-e2e -juju add-relation kubernetes-e2e kubernetes-master +juju add-relation kubernetes-e2e:kube-control kubernetes-master:kube-control +juju add-relation kubernetes-e2e:kubernetes-master kubernetes-master:kube-api-endpoint juju add-relation kubernetes-e2e easyrsa ``` diff --git a/cluster/juju/layers/kubernetes-e2e/layer.yaml b/cluster/juju/layers/kubernetes-e2e/layer.yaml index 06d431b0452c7..b913883bb0d17 100644 --- a/cluster/juju/layers/kubernetes-e2e/layer.yaml +++ b/cluster/juju/layers/kubernetes-e2e/layer.yaml @@ -4,6 +4,7 @@ includes: - layer:tls-client - layer:snap - interface:http + - interface:kube-control options: tls-client: ca_certificate_path: '/srv/kubernetes/ca.crt' diff --git a/cluster/juju/layers/kubernetes-e2e/metadata.yaml b/cluster/juju/layers/kubernetes-e2e/metadata.yaml index a67fd67ee62af..9f4690a1e2b0c 100644 --- a/cluster/juju/layers/kubernetes-e2e/metadata.yaml +++ b/cluster/juju/layers/kubernetes-e2e/metadata.yaml @@ -14,6 +14,8 @@ series: requires: kubernetes-master: interface: http + kube-control: + interface: kube-control resources: kubectl: type: file @@ -23,3 +25,4 @@ resources: type: file filename: kubernetes-test.snap description: kubernetes-test snap + diff --git a/cluster/juju/layers/kubernetes-e2e/reactive/kubernetes_e2e.py b/cluster/juju/layers/kubernetes-e2e/reactive/kubernetes_e2e.py index 1ab6f5f739115..76a97aa0d6336 100644 --- a/cluster/juju/layers/kubernetes-e2e/reactive/kubernetes_e2e.py +++ b/cluster/juju/layers/kubernetes-e2e/reactive/kubernetes_e2e.py @@ -38,15 +38,22 @@ def reset_delivery_states(): @when('kubernetes-e2e.installed') +def report_status(): + ''' Report the status of the charm. ''' + messaging() + + def messaging(): ''' Probe our relations to determine the propper messaging to the end user ''' missing_services = [] if not is_state('kubernetes-master.available'): - missing_services.append('kubernetes-master') + missing_services.append('kubernetes-master:http') if not is_state('certificates.available'): missing_services.append('certificates') + if not is_state('kubeconfig.ready'): + missing_services.append('kubernetes-master:kube-control') if missing_services: if len(missing_services) > 1: @@ -80,16 +87,15 @@ def install_snaps(): @when('tls_client.ca.saved', 'tls_client.client.certificate.saved', 'tls_client.client.key.saved', 'kubernetes-master.available', - 'kubernetes-e2e.installed') + 'kubernetes-e2e.installed', 'kube-control.auth.available') @when_not('kubeconfig.ready') -def prepare_kubeconfig_certificates(master): +def prepare_kubeconfig_certificates(master, kube_control): ''' Prepare the data to feed to create the kubeconfig file. ''' layer_options = layer.options('tls-client') # Get all the paths to the tls information required for kubeconfig. ca = layer_options.get('ca_certificate_path') - key = layer_options.get('client_key_path') - cert = layer_options.get('client_certificate_path') + creds = kube_control.get_auth_credentials() servers = get_kube_api_servers(master) @@ -97,17 +103,28 @@ def prepare_kubeconfig_certificates(master): kubeconfig_path = '/home/ubuntu/.kube/config' # Create kubernetes configuration in the default location for ubuntu. - create_kubeconfig('/root/.kube/config', servers[0], ca, key, cert, - user='root') - create_kubeconfig(kubeconfig_path, servers[0], ca, key, cert, - user='ubuntu') + create_kubeconfig('/root/.kube/config', servers[0], ca, + token=creds['client_token'], user='root') + create_kubeconfig(kubeconfig_path, servers[0], ca, + token=creds['client_token'], user='ubuntu') # Set permissions on the ubuntu users kubeconfig to ensure a consistent UX cmd = ['chown', 'ubuntu:ubuntu', kubeconfig_path] check_call(cmd) - + messaging() set_state('kubeconfig.ready') +@when('kube-control.connected') +def request_credentials(kube_control): + """ Request authorization creds.""" + + # The kube-cotrol interface is created to support RBAC. + # At this point we might as well do the right thing and return the hostname + # even if it will only be used when we enable RBAC + user = 'system:masters' + kube_control.set_auth_request(user) + + @when('kubernetes-e2e.installed', 'kubeconfig.ready') def set_app_version(): ''' Declare the application version to juju ''' @@ -124,19 +141,40 @@ def set_app_version(): hookenv.application_version_set(version_from.rstrip()) -def create_kubeconfig(kubeconfig, server, ca, key, certificate, user='ubuntu', - context='juju-context', cluster='juju-cluster'): +def create_kubeconfig(kubeconfig, server, ca, key=None, certificate=None, + user='ubuntu', context='juju-context', + cluster='juju-cluster', password=None, token=None): '''Create a configuration for Kubernetes based on path using the supplied arguments for values of the Kubernetes server, CA, key, certificate, user context and cluster.''' + if not key and not certificate and not password and not token: + raise ValueError('Missing authentication mechanism.') + + # token and password are mutually exclusive. Error early if both are + # present. The developer has requested an impossible situation. + # see: kubectl config set-credentials --help + if token and password: + raise ValueError('Token and Password are mutually exclusive.') # Create the config file with the address of the master server. cmd = 'kubectl config --kubeconfig={0} set-cluster {1} ' \ '--server={2} --certificate-authority={3} --embed-certs=true' check_call(split(cmd.format(kubeconfig, cluster, server, ca))) + # Delete old users + cmd = 'kubectl config --kubeconfig={0} unset users' + check_call(split(cmd.format(kubeconfig))) # Create the credentials using the client flags. - cmd = 'kubectl config --kubeconfig={0} set-credentials {1} ' \ - '--client-key={2} --client-certificate={3} --embed-certs=true' - check_call(split(cmd.format(kubeconfig, user, key, certificate))) + cmd = 'kubectl config --kubeconfig={0} ' \ + 'set-credentials {1} '.format(kubeconfig, user) + + if key and certificate: + cmd = '{0} --client-key={1} --client-certificate={2} '\ + '--embed-certs=true'.format(cmd, key, certificate) + if password: + cmd = "{0} --username={1} --password={2}".format(cmd, user, password) + # This is mutually exclusive from password. They will not work together. + if token: + cmd = "{0} --token={1}".format(cmd, token) + check_call(split(cmd)) # Create a default context with the cluster. cmd = 'kubectl config --kubeconfig={0} set-context {1} ' \ '--cluster={2} --user={3}' diff --git a/cluster/juju/layers/kubernetes-master/config.yaml b/cluster/juju/layers/kubernetes-master/config.yaml index 0fde833377a7e..b58087c24e31f 100644 --- a/cluster/juju/layers/kubernetes-master/config.yaml +++ b/cluster/juju/layers/kubernetes-master/config.yaml @@ -26,3 +26,8 @@ options: default: "stable" description: | Snap channel to install Kubernetes master services from + client_password: + type: string + default: "" + description: | + Password to be used for admin user (leave empty for random password). diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 387d2559720c6..6158c1c47db6e 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -37,7 +37,7 @@ from charms.reactive import set_state from charms.reactive import is_state from charms.reactive import when, when_any, when_not -from charms.reactive.helpers import data_changed +from charms.reactive.helpers import data_changed, any_file_changed from charms.kubernetes.common import get_version from charms.kubernetes.common import retry from charms.kubernetes.flagmanager import FlagManager @@ -77,8 +77,8 @@ def reset_states_for_delivery(): '''An upgrade charm event was triggered by Juju, react to that here.''' migrate_from_pre_snaps() install_snaps() + set_state('reconfigure.authentication.setup') remove_state('authentication.setup') - remove_state('kubernetes-master.components.started') def rename_file_idempotent(source, destination): @@ -162,6 +162,22 @@ def channel_changed(): install_snaps() +@when('config.changed.client_password', 'leadership.is_leader') +def password_changed(): + """Handle password change via the charms config.""" + password = hookenv.config('client_password') + if password == "" and is_state('client.password.initialised'): + # password_changed is called during an upgrade. Nothing to do. + return + elif password == "": + # Password not initialised + password = token_generator() + setup_basic_auth(password, "admin", "admin") + set_state('reconfigure.authentication.setup') + remove_state('authentication.setup') + set_state('client.password.initialised') + + @when('cni.connected') @when_not('cni.configured') def configure_cni(cni): @@ -187,19 +203,23 @@ def setup_leader_authentication(): keys = [service_key, basic_auth, known_tokens] # Try first to fetch data from an old leadership broadcast. - if not get_keys_from_leader(keys): - if not os.path.isfile(basic_auth): - setup_basic_auth('admin', 'admin', 'admin') + if not get_keys_from_leader(keys) \ + or is_state('reconfigure.authentication.setup'): + last_pass = get_password('basic_auth.csv', 'admin') + setup_basic_auth(last_pass, 'admin', 'admin') + if not os.path.isfile(known_tokens): setup_tokens(None, 'admin', 'admin') setup_tokens(None, 'kubelet', 'kubelet') setup_tokens(None, 'kube_proxy', 'kube_proxy') + # Generate the default service account token key os.makedirs('/root/cdk', exist_ok=True) if not os.path.isfile(service_key): cmd = ['openssl', 'genrsa', '-out', service_key, '2048'] check_call(cmd) + remove_state('reconfigure.authentication.setup') api_opts.add('service-account-key-file', service_key) controller_opts.add('service-account-private-key-file', service_key) @@ -215,32 +235,36 @@ def setup_leader_authentication(): # eg: # {'/root/cdk/serviceaccount.key': 'RSA:2471731...'} charms.leadership.leader_set(leader_data) - + remove_state('kubernetes-master.components.started') set_state('authentication.setup') @when_not('leadership.is_leader') -@when_not('authentication.setup') def setup_non_leader_authentication(): - api_opts = FlagManager('kube-apiserver') - controller_opts = FlagManager('kube-controller-manager') service_key = '/root/cdk/serviceaccount.key' basic_auth = '/root/cdk/basic_auth.csv' known_tokens = '/root/cdk/known_tokens.csv' - hookenv.status_set('maintenance', 'Rendering authentication templates.') - keys = [service_key, basic_auth, known_tokens] if not get_keys_from_leader(keys): # the keys were not retrieved. Non-leaders have to retry. return + if not any_file_changed(keys) and is_state('authentication.setup'): + # No change detected and we have already setup the authentication + return + + hookenv.status_set('maintenance', 'Rendering authentication templates.') + api_opts = FlagManager('kube-apiserver') api_opts.add('basic-auth-file', basic_auth) api_opts.add('token-auth-file', known_tokens) api_opts.add('service-account-key-file', service_key) + + controller_opts = FlagManager('kube-controller-manager') controller_opts.add('service-account-private-key-file', service_key) + remove_state('kubernetes-master.components.started') set_state('authentication.setup') @@ -346,6 +370,22 @@ def send_cluster_dns_detail(kube_control): kube_control.set_dns(53, hookenv.config('dns_domain'), dns_ip) +@when('kube-control.auth.requested') +@when('authentication.setup') +@when('leadership.is_leader') +def send_tokens(kube_control): + """Send the tokens to the workers.""" + kubelet_token = get_token('kubelet') + proxy_token = get_token('kube_proxy') + admin_token = get_token('admin') + + # Send the data + requests = kube_control.auth_user() + for request in requests: + kube_control.sign_auth_request(request[0], kubelet_token, + proxy_token, admin_token) + + @when_not('kube-control.connected') def missing_kube_control(): """Inform the operator they need to add the kube-control relation. @@ -443,7 +483,7 @@ def addons_ready(): @when('loadbalancer.available', 'certificates.ca.available', - 'certificates.client.cert.available') + 'certificates.client.cert.available', 'authentication.setup') def loadbalancer_kubeconfig(loadbalancer, ca, client): # Get the potential list of loadbalancers from the relation object. hosts = loadbalancer.get_addresses_ports() @@ -455,7 +495,8 @@ def loadbalancer_kubeconfig(loadbalancer, ca, client): build_kubeconfig(server) -@when('certificates.ca.available', 'certificates.client.cert.available') +@when('certificates.ca.available', 'certificates.client.cert.available', + 'authentication.setup') @when_not('loadbalancer.available') def create_self_config(ca, client): '''Create a kubernetes configuration for the master unit.''' @@ -664,37 +705,54 @@ def build_kubeconfig(server): # Get all the paths to the tls information required for kubeconfig. ca = layer_options.get('ca_certificate_path') ca_exists = ca and os.path.isfile(ca) - key = layer_options.get('client_key_path') - key_exists = key and os.path.isfile(key) - cert = layer_options.get('client_certificate_path') - cert_exists = cert and os.path.isfile(cert) + client_pass = get_password('basic_auth.csv', 'admin') # Do we have everything we need? - if ca_exists and key_exists and cert_exists: - # Cache last server string to know if we need to regenerate the config. - if not data_changed('kubeconfig.server', server): - return + if ca_exists and client_pass: # Create an absolute path for the kubeconfig file. kubeconfig_path = os.path.join(os.sep, 'home', 'ubuntu', 'config') # Create the kubeconfig on this system so users can access the cluster. - create_kubeconfig(kubeconfig_path, server, ca, key, cert) + + create_kubeconfig(kubeconfig_path, server, ca, + user='admin', password=client_pass) # Make the config file readable by the ubuntu users so juju scp works. cmd = ['chown', 'ubuntu:ubuntu', kubeconfig_path] check_call(cmd) -def create_kubeconfig(kubeconfig, server, ca, key, certificate, user='ubuntu', - context='juju-context', cluster='juju-cluster'): +def create_kubeconfig(kubeconfig, server, ca, key=None, certificate=None, + user='ubuntu', context='juju-context', + cluster='juju-cluster', password=None, token=None): '''Create a configuration for Kubernetes based on path using the supplied arguments for values of the Kubernetes server, CA, key, certificate, user context and cluster.''' + if not key and not certificate and not password and not token: + raise ValueError('Missing authentication mechanism.') + + # token and password are mutually exclusive. Error early if both are + # present. The developer has requested an impossible situation. + # see: kubectl config set-credentials --help + if token and password: + raise ValueError('Token and Password are mutually exclusive.') # Create the config file with the address of the master server. cmd = 'kubectl config --kubeconfig={0} set-cluster {1} ' \ '--server={2} --certificate-authority={3} --embed-certs=true' check_call(split(cmd.format(kubeconfig, cluster, server, ca))) + # Delete old users + cmd = 'kubectl config --kubeconfig={0} unset users' + check_call(split(cmd.format(kubeconfig))) # Create the credentials using the client flags. - cmd = 'kubectl config --kubeconfig={0} set-credentials {1} ' \ - '--client-key={2} --client-certificate={3} --embed-certs=true' - check_call(split(cmd.format(kubeconfig, user, key, certificate))) + cmd = 'kubectl config --kubeconfig={0} ' \ + 'set-credentials {1} '.format(kubeconfig, user) + + if key and certificate: + cmd = '{0} --client-key={1} --client-certificate={2} '\ + '--embed-certs=true'.format(cmd, key, certificate) + if password: + cmd = "{0} --username={1} --password={2}".format(cmd, user, password) + # This is mutually exclusive from password. They will not work together. + if token: + cmd = "{0} --token={1}".format(cmd, token) + check_call(split(cmd)) # Create a default context with the cluster. cmd = 'kubectl config --kubeconfig={0} set-context {1} ' \ '--cluster={2} --user={3}' @@ -781,7 +839,6 @@ def configure_master_services(): api_opts.add('service-cluster-ip-range', service_cidr()) api_opts.add('min-request-timeout', '300') api_opts.add('v', '4') - api_opts.add('client-ca-file', ca_cert_path) api_opts.add('tls-cert-file', server_cert_path) api_opts.add('tls-private-key-file', server_key_path) api_opts.add('kubelet-certificate-authority', ca_cert_path) @@ -821,6 +878,7 @@ def configure_master_services(): cmd = ['snap', 'set', 'kube-apiserver'] + api_opts.to_s().split(' ') check_call(cmd) + cmd = ( ['snap', 'set', 'kube-controller-manager'] + controller_opts.to_s().split(' ') @@ -830,14 +888,16 @@ def configure_master_services(): check_call(cmd) -def setup_basic_auth(username='admin', password='admin', user='admin'): +def setup_basic_auth(password=None, username='admin', uid='admin'): '''Create the htacces file and the tokens.''' root_cdk = '/root/cdk' if not os.path.isdir(root_cdk): os.makedirs(root_cdk) htaccess = os.path.join(root_cdk, 'basic_auth.csv') + if not password: + password = token_generator() with open(htaccess, 'w') as stream: - stream.write('{0},{1},{2}'.format(username, password, user)) + stream.write('{0},{1},{2}'.format(password, username, uid)) def setup_tokens(token, username, user): @@ -847,12 +907,49 @@ def setup_tokens(token, username, user): os.makedirs(root_cdk) known_tokens = os.path.join(root_cdk, 'known_tokens.csv') if not token: - alpha = string.ascii_letters + string.digits - token = ''.join(random.SystemRandom().choice(alpha) for _ in range(32)) + token = token_generator() with open(known_tokens, 'a') as stream: stream.write('{0},{1},{2}\n'.format(token, username, user)) +def get_password(csv_fname, user): + '''Get the password of user within the csv file provided.''' + root_cdk = '/root/cdk' + if not os.path.isdir(root_cdk): + return None + tokens_fname = os.path.join(root_cdk, csv_fname) + with open(tokens_fname, 'r') as stream: + for line in stream: + record = line.split(',') + if record[1] == user: + return record[0] + return None + + +def get_token(username): + """Grab a token from the static file if present. """ + return get_password('known_tokens.csv', username) + + +def set_token(password, save_salt): + ''' Store a token so it can be recalled later by token_generator. + + param: password - the password to be stored + param: save_salt - the key to store the value of the token.''' + db = unitdata.kv() + db.set(save_salt, password) + return db.get(save_salt) + + +def token_generator(length=32): + ''' Generate a random token for use in passwords and account tokens. + + param: length - the length of the token to generate''' + alpha = string.ascii_letters + string.digits + token = ''.join(random.SystemRandom().choice(alpha) for _ in range(length)) + return token + + @retry(times=3, delay_secs=10) def all_kube_system_pods_running(): ''' Check pod status in the kube-system namespace. Returns True if all @@ -866,7 +963,6 @@ def all_kube_system_pods_running(): return False result = json.loads(output) - for pod in result['items']: status = pod['status']['phase'] if status != 'Running': diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index d42c9241a6cae..0187fd4c80653 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -303,9 +303,10 @@ def watch_for_changes(kube_api, kube_control, cni): @when('kubernetes-worker.snaps.installed', 'kube-api-endpoint.available', 'tls_client.ca.saved', 'tls_client.client.certificate.saved', 'tls_client.client.key.saved', 'tls_client.server.certificate.saved', - 'tls_client.server.key.saved', 'kube-control.dns.available', + 'tls_client.server.key.saved', + 'kube-control.dns.available', 'kube-control.auth.available', 'cni.available', 'kubernetes-worker.restart-needed') -def start_worker(kube_api, kube_control, cni): +def start_worker(kube_api, kube_control, auth_control, cni): ''' Start kubelet using the provided API and DNS info.''' servers = get_kube_api_servers(kube_api) # Note that the DNS server doesn't necessarily exist at this point. We know @@ -320,10 +321,13 @@ def start_worker(kube_api, kube_control, cni): hookenv.log('Waiting for cluster cidr.') return + creds = kube_control.get_auth_credentials() + data_changed('kube-control.creds', creds) + # set --allow-privileged flag for kubelet set_privileged() - create_config(random.choice(servers)) + create_config(random.choice(servers), creds) configure_worker_services(servers, dns, cluster_cidr) set_state('kubernetes-worker.config.created') restart_unit_services() @@ -428,27 +432,25 @@ def arch(): return architecture -def create_config(server): +def create_config(server, creds): '''Create a kubernetes configuration for the worker unit.''' # Get the options from the tls-client layer. layer_options = layer.options('tls-client') # Get all the paths to the tls information required for kubeconfig. ca = layer_options.get('ca_certificate_path') - key = layer_options.get('client_key_path') - cert = layer_options.get('client_certificate_path') # Create kubernetes configuration in the default location for ubuntu. - create_kubeconfig('/home/ubuntu/.kube/config', server, ca, key, cert, - user='ubuntu') + create_kubeconfig('/home/ubuntu/.kube/config', server, ca, + token=creds['client_token'], user='ubuntu') # Make the config dir readable by the ubuntu users so juju scp works. cmd = ['chown', '-R', 'ubuntu:ubuntu', '/home/ubuntu/.kube'] check_call(cmd) # Create kubernetes configuration in the default location for root. - create_kubeconfig('/root/.kube/config', server, ca, key, cert, - user='root') + create_kubeconfig('/root/.kube/config', server, ca, + token=creds['client_token'], user='root') # Create kubernetes configuration for kubelet, and kube-proxy services. - create_kubeconfig(kubeconfig_path, server, ca, key, cert, - user='kubelet') + create_kubeconfig(kubeconfig_path, server, ca, + token=creds['kubelet_token'], user='kubelet') def configure_worker_services(api_servers, dns, cluster_cidr): @@ -463,16 +465,15 @@ def configure_worker_services(api_servers, dns, cluster_cidr): kubelet_opts.add('require-kubeconfig', 'true') kubelet_opts.add('kubeconfig', kubeconfig_path) kubelet_opts.add('network-plugin', 'cni') - kubelet_opts.add('logtostderr', 'true') kubelet_opts.add('v', '0') kubelet_opts.add('address', '0.0.0.0') kubelet_opts.add('port', '10250') kubelet_opts.add('cluster-dns', dns['sdn-ip']) kubelet_opts.add('cluster-domain', dns['domain']) - kubelet_opts.add('anonymous-auth', 'false') kubelet_opts.add('client-ca-file', ca_cert_path) kubelet_opts.add('tls-cert-file', server_cert_path) kubelet_opts.add('tls-private-key-file', server_key_path) + kubelet_opts.add('logtostderr', 'true') kube_proxy_opts = FlagManager('kube-proxy') kube_proxy_opts.add('cluster-cidr', cluster_cidr) @@ -487,19 +488,40 @@ def configure_worker_services(api_servers, dns, cluster_cidr): check_call(cmd) -def create_kubeconfig(kubeconfig, server, ca, key, certificate, user='ubuntu', - context='juju-context', cluster='juju-cluster'): +def create_kubeconfig(kubeconfig, server, ca, key=None, certificate=None, + user='ubuntu', context='juju-context', + cluster='juju-cluster', password=None, token=None): '''Create a configuration for Kubernetes based on path using the supplied arguments for values of the Kubernetes server, CA, key, certificate, user context and cluster.''' + if not key and not certificate and not password and not token: + raise ValueError('Missing authentication mechanism.') + + # token and password are mutually exclusive. Error early if both are + # present. The developer has requested an impossible situation. + # see: kubectl config set-credentials --help + if token and password: + raise ValueError('Token and Password are mutually exclusive.') # Create the config file with the address of the master server. cmd = 'kubectl config --kubeconfig={0} set-cluster {1} ' \ '--server={2} --certificate-authority={3} --embed-certs=true' check_call(split(cmd.format(kubeconfig, cluster, server, ca))) + # Delete old users + cmd = 'kubectl config --kubeconfig={0} unset users' + check_call(split(cmd.format(kubeconfig))) # Create the credentials using the client flags. - cmd = 'kubectl config --kubeconfig={0} set-credentials {1} ' \ - '--client-key={2} --client-certificate={3} --embed-certs=true' - check_call(split(cmd.format(kubeconfig, user, key, certificate))) + cmd = 'kubectl config --kubeconfig={0} ' \ + 'set-credentials {1} '.format(kubeconfig, user) + + if key and certificate: + cmd = '{0} --client-key={1} --client-certificate={2} '\ + '--embed-certs=true'.format(cmd, key, certificate) + if password: + cmd = "{0} --username={1} --password={2}".format(cmd, user, password) + # This is mutually exclusive from password. They will not work together. + if token: + cmd = "{0} --token={1}".format(cmd, token) + check_call(split(cmd)) # Create a default context with the cluster. cmd = 'kubectl config --kubeconfig={0} set-context {1} ' \ '--cluster={2} --user={3}' @@ -761,6 +783,26 @@ def notify_master_gpu_not_enabled(kube_control): kube_control.set_gpu(False) +@when('kube-control.connected') +def request_kubelet_and_proxy_credentials(kube_control): + """ Request kubelet node authorization with a well formed kubelet user. + This also implies that we are requesting kube-proxy auth. """ + + # The kube-cotrol interface is created to support RBAC. + # At this point we might as well do the right thing and return the hostname + # even if it will only be used when we enable RBAC + nodeuser = 'system:node:{}'.format(gethostname()) + kube_control.set_auth_request(nodeuser) + + +@when('kube-control.auth.available') +def catch_change_in_creds(kube_control): + """Request a service restart in case credential updates were detected.""" + creds = kube_control.get_auth_credentials() + if data_changed('kube-control.creds', creds): + set_state('kubernetes-worker.restart-needed') + + @when_not('kube-control.connected') def missing_kube_control(): """Inform the operator they need to add the kube-control relation. From 0525b84a45b521125aecb1c2cd50484c41aff262 Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Wed, 28 Jun 2017 10:47:45 +0300 Subject: [PATCH 2/2] Disable anonymous-auth --- .../juju/layers/kubernetes-worker/reactive/kubernetes_worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index 0187fd4c80653..e82a19e2ec1bd 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -470,6 +470,7 @@ def configure_worker_services(api_servers, dns, cluster_cidr): kubelet_opts.add('port', '10250') kubelet_opts.add('cluster-dns', dns['sdn-ip']) kubelet_opts.add('cluster-domain', dns['domain']) + kubelet_opts.add('anonymous-auth', 'false') kubelet_opts.add('client-ca-file', ca_cert_path) kubelet_opts.add('tls-cert-file', server_cert_path) kubelet_opts.add('tls-private-key-file', server_key_path)