WIP: Salt reconfiguration to get rid of nginx and pass a bearer token to

the apiserver. - install libcap2-bin w/ apiserver - use setcap to allow the apiserver to bind to low numbered ports - set secure port for the apiserver to 443 - Kubelet talk to port 443 on gce
kubernetes · Apr 9, 2015 · f12d810 · f12d810
1 parent 43ec88f
commit f12d810
Show file tree

Hide file tree

Showing 9 changed files with 100 additions and 52 deletions.
diff --git a/cluster/common.sh b/cluster/common.sh
@@ -26,6 +26,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 # Assumed vars:
 #   KUBE_USER
 #   KUBE_PASSWORD
+#   KUBE_BEARER_TOKEN
 #   KUBE_MASTER_IP
 #   KUBECONFIG
 #
@@ -44,11 +45,18 @@ function create-kubeconfig() {
   "${kubectl}" config set-cluster "${CONTEXT}" --server="https://${KUBE_MASTER_IP}" \
                                                --certificate-authority="${CA_CERT}" \
                                                --embed-certs=true
-  "${kubectl}" config set-credentials "${CONTEXT}" --username="${KUBE_USER}" \
-                                                --password="${KUBE_PASSWORD}" \
-                                                --client-certificate="${KUBE_CERT}" \
-                                                --client-key="${KUBE_KEY}" \
-                                                --embed-certs=true
+  if [[ -z "${KUBE_USER:-}" || -z "${KUBE_PASSWORD:-}" ]]; then
+    "${kubectl}" config set-credentials "${CONTEXT}" --token="${KUBE_BEARER_TOKEN}" \
+                                                     --client-certificate="${KUBE_CERT}" \
+                                                     --client-key="${KUBE_KEY}" \
+                                                     --embed-certs=true
+  else
+    "${kubectl}" config set-credentials "${CONTEXT}" --username="${KUBE_USER}" \
+                                                     --password="${KUBE_PASSWORD}" \
+                                                     --client-certificate="${KUBE_CERT}" \
+                                                     --client-key="${KUBE_KEY}" \
+                                                     --embed-certs=true
+  fi
   "${kubectl}" config set-context "${CONTEXT}" --cluster="${CONTEXT}" --user="${CONTEXT}"
   "${kubectl}" config use-context "${CONTEXT}"  --cluster="${CONTEXT}"
 
@@ -101,3 +109,27 @@ function get-kubeconfig-basicauth() {
     KUBE_PASSWORD=''
   fi
 }
+
+# Get the bearer token for the current-context in kubeconfig if one exists.
+# Assumed vars:
+#   KUBECONFIG  # if unset, defaults to global
+#
+# Vars set:
+#   KUBE_BEARER_TOKEN
+#
+# KUBE_BEARER_TOKEN will be empty if no current-context is set, or the
+# current-context user does not exist or contain a bearer token entry.
+function get-kubeconfig-bearertoken() {
+  # Template to safely extract the token for the current-context user.
+  # The long chain of 'with' commands avoids indexing nil if any of the
+  # entries ("current-context", "contexts"."current-context", "users", etc)
+  # is missing.
+  # Note: we save dot ('.') to $root because the 'with' action overrides it.
+  # See http://golang.org/pkg/text/template/.
+  local token='{{$root := .}}{{with index $root "current-context"}}{{with index $root "contexts" .}}{{with index . "user"}}{{with index $root "users" .}}{{index . "token"}}{{end}}{{end}}{{end}}{{end}}'
+  KUBE_BEARER_TOKEN=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o template --template="${token}")
+  # Handle empty/missing token
+  if [[ "${KUBE_BEARER_TOKEN}" == '<no value>' ]]; then
+    KUBE_BEARER_TOKEN=''
+  fi
+}
diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh
@@ -245,23 +245,17 @@ EOF
 }
 
 # This should only happen on cluster initialization. Uses
-# MASTER_HTPASSWORD to generate the nginx/htpasswd file, and the
-# KUBELET_TOKEN, plus /dev/urandom, to generate known_tokens.csv
-# (KNOWN_TOKENS_FILE). After the first boot and on upgrade, these
-# files exist on the master-pd and should never be touched again
-# (except perhaps an additional service account, see NB below.)
+# KUBE_BEARER_TOKEN, KUBELET_TOKEN, and /dev/urandom to generate
+# known_tokens.csv (KNOWN_TOKENS_FILE). After the first boot and
+# on upgrade, this file exists on the master-pd and should never
+# be touched again (except perhaps an additional service account,
+# see NB below.)
 function create-salt-auth() {
-  local -r htpasswd_file="/srv/salt-overlay/salt/nginx/htpasswd"
-
-  if [ ! -e "${htpasswd_file}" ]; then
-    mkdir -p /srv/salt-overlay/salt/nginx
-    echo "${MASTER_HTPASSWD}" > "${htpasswd_file}"
-  fi
-
   if [ ! -e "${KNOWN_TOKENS_FILE}" ]; then
     mkdir -p /srv/salt-overlay/salt/kube-apiserver
     (umask 077;
-      echo "${KUBELET_TOKEN},kubelet,kubelet" > "${KNOWN_TOKENS_FILE}")
+      echo "${KUBE_BEARER_TOKEN},admin,admin" > "${KNOWN_TOKENS_FILE}";
+      echo "${KUBELET_TOKEN},kubelet,kubelet" >> "${KNOWN_TOKENS_FILE}")
 
     mkdir -p /srv/salt-overlay/salt/kubelet
     kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"

diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh
@@ -261,30 +261,21 @@ function get-password {
   fi
 }
 
-# Set MASTER_HTPASSWD
-function set-master-htpasswd {
-  python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
-    -b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD"
-  local htpasswd
-  MASTER_HTPASSWD=$(cat "${KUBE_TEMP}/htpasswd")
-}
-
-# Generate authentication token for admin user. Will
-# read from $HOME/.kubernetes_auth if available.
+# Ensure that we have a bearer token created for validating to the master.
+# Will read from kubeconfig for the current context if available.
+#
+# Assumed vars
+#   KUBE_ROOT
 #
 # Vars set:
-#   KUBE_ADMIN_TOKEN
-function get-admin-token {
-  local file="$HOME/.kubernetes_auth"
-  if [[ -r "$file" ]]; then
-    KUBE_ADMIN_TOKEN=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["BearerToken"]')
-    return
+#   KUBE_BEARER_TOKEN
+function get-bearer-token() {
+  get-kubeconfig-bearertoken
+  if [[ -z "${KUBE_BEARER_TOKEN}" ]]; then
+    KUBE_BEARER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
   fi
-  KUBE_ADMIN_TOKEN=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(32))')
 }
 
-
-
 # Wait for background jobs to finish. Exit with
 # an error status if any of the jobs failed.
 function wait-for-jobs {
@@ -464,7 +455,7 @@ ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false})
 DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-})
 DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-})
 DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})
-MASTER_HTPASSWD: $(yaml-quote ${MASTER_HTPASSWD})
+KUBE_BEARER_TOKEN: $(yaml-quote ${KUBE_BEARER_TOKEN:-})
 ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-})
 MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE})
 EOF
@@ -498,8 +489,7 @@ function write-node-env {
 # variables are set:
 #   ensure-temp-dir
 #   detect-project
-#   get-password
-#   set-master-htpasswd
+#   get-bearer-token
 #
 function create-master-instance {
   local address_opt=""
@@ -532,8 +522,7 @@ function kube-up {
   ensure-temp-dir
   detect-project
 
-  get-password
-  set-master-htpasswd
+  get-bearer-token
 
   # Make sure we have the tar files staged on Google Storage
   find-release-tars
@@ -662,8 +651,9 @@ function kube-up {
   echo "  up."
   echo
 
-  until curl --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" --max-time 5 \
-          --fail --output /dev/null --silent "https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do
+  until curl --insecure -H "Authorization: Bearer ${KUBE_BEARER_TOKEN}" \
+          --max-time 5 --fail --output /dev/null --silent \
+          "https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do
       printf "."
       sleep 2
   done
@@ -844,8 +834,7 @@ function kube-push {
   detect-project
   detect-master
   detect-minion-names
-  get-password
-  set-master-htpasswd
+  get-bearer-token
 
   # Make sure we have the tar files staged on Google Storage
   find-release-tars

diff --git a/cluster/saltbase/salt/kube-apiserver/default b/cluster/saltbase/salt/kube-apiserver/default
@@ -43,6 +43,12 @@
 {% set secure_port = "--secure_port=6443" -%}
 {% set token_auth_file = "--token_auth_file=/dev/null" -%}
 
+{% if grains.cloud is defined -%}
+{% if grains.cloud in [ 'gce' ] -%}
+    {% set secure_port = "--secure_port=443" -%}
+{% endif -%}
+{% endif -%}
+
 {% if grains.cloud is defined -%}
 {% if grains.cloud in [ 'aws', 'gce', 'vagrant' ] -%}
     # TODO: generate and distribute tokens for other cloud providers.

diff --git a/cluster/saltbase/salt/kube-apiserver/init.sls b/cluster/saltbase/salt/kube-apiserver/init.sls
@@ -38,6 +38,15 @@
 
 {% endif %}
 
+{% if grains.cloud is defined %}
+{% if grains.cloud in ['gce'] %}
+# TODO: remove this once kube-apiserver runs in a container
+libcap2-bin:
+  pkg:
+    - installed
+{% endif %}
+{% endif %}
+
 {% if grains.cloud is defined %}
 {% if grains.cloud in ['aws', 'gce', 'vagrant'] %}
 # TODO: generate and distribute tokens on other cloud providers.

diff --git a/cluster/saltbase/salt/kube-apiserver/initd b/cluster/saltbase/salt/kube-apiserver/initd
@@ -41,6 +41,9 @@ do_start()
         # Raise the file descriptor limit - we expect to open a lot of sockets!
         ulimit -n 65536
 
+        # Make it possible to bind to low numbered ports.
+        /sbin/setcap 'cap_net_bind_service=+ep' $DAEMON
+
         # Return
         #   0 if daemon has been started
         #   1 if daemon was already running

diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default
@@ -4,15 +4,24 @@
 {% endif -%}
 
 {% if grains.api_servers is defined -%}
-  {% set api_servers = "--api_servers=https://" + grains.api_servers + ":6443" -%}
+  {% set api_servers = "--api_servers=https://" + grains.api_servers -%}
 {% elif grains.apiservers is defined -%} # TODO(remove after 0.16.0): Deprecated form
-  {% set api_servers = "--api_servers=https://" + grains.apiservers + ":6443" -%}
+  {% set api_servers = "--api_servers=https://" + grains.apiservers -%}
 {% elif grains['roles'][0] == 'kubernetes-master' -%}
   {% set master_ipv4 = salt['grains.get']('fqdn_ip4')[0] -%}
-  {% set api_servers = "--api_servers=https://" + master_ipv4 + ":6443" -%}
+  {% set api_servers = "--api_servers=https://" + master_ipv4 -%}
 {% else -%}
   {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
-  {% set api_servers = "--api_servers=https://" + ips[0][0] + ":6443" -%}
+  {% set api_servers = "--api_servers=https://" + ips[0][0] -%}
+{% endif -%}
+
+{% if grains.cloud is defined -%}
+  {% if grains.cloud in [ 'gce' ] -%}
+    # TODO: remove nginx for other cloud providers.
+    {% set api_servers_with_port = api_servers -%}
+  {% endif -%}
+{% else -%}
+  {% set api_servers_with_port = api_servers + ":6443" -%}
 {% endif -%}
 
 {% set config = "--config=/etc/kubernetes/manifests" -%}
@@ -33,4 +42,4 @@
   {% set docker_root = " --docker_root=" + grains.docker_root -%}
 {% endif -%}
 
-DAEMON_ARGS="{{daemon_args}} {{api_servers}} {{hostname_override}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}}"
+DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{hostname_override}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}}"
diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls
@@ -32,7 +32,9 @@ base:
     - kube-controller-manager
     - kube-scheduler
     - monit
+{% if grains['cloud'] is defined and grains['cloud'] != 'gce' %}
     - nginx
+{% endif %}
     - cadvisor
     - kube-client-tools
 {% if grains['cloud'] is defined and grains['cloud'] != 'vagrant' %}

diff --git a/cluster/validate-cluster.sh b/cluster/validate-cluster.sh
@@ -29,6 +29,7 @@ source "${KUBE_ROOT}/cluster/kube-env.sh"
 source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
 
 get-password
+get-bearer-token
 detect-master > /dev/null
 detect-minions > /dev/null
 
@@ -89,7 +90,10 @@ for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
     attempt=0
     while true; do
       echo -n "Attempt $((attempt+1)) at checking Kubelet installation on node ${MINION_NAMES[$i]} ..."
-      if [[ "$KUBERNETES_PROVIDER" != "libvirt-coreos" && "$KUBERNETES_PROVIDER" != "juju" ]]; then
+      if [[ "$KUBERNETES_PROVIDER" == "gce" ]]; then
+        curl_output=$(curl -s --insecure -H "Authorization: Bearer ${KUBE_BEARER_TOKEN}" \
+          "https://${KUBE_MASTER_IP}/api/v1beta1/proxy/minions/${name}/healthz")
+      elif [[ "$KUBERNETES_PROVIDER" != "libvirt-coreos" && "$KUBERNETES_PROVIDER" != "juju" ]]; then
         curl_output=$(curl -s --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" \
           "https://${KUBE_MASTER_IP}/api/v1beta1/proxy/minions/${name}/healthz")
       else