Skip to content

Commit

Permalink
WIP: Salt reconfiguration to remove nginx and pass a bearer token to
Browse files Browse the repository at this point in the history 
the apiserver (GCE only).

 - Configure the apiserver to listen securely on 443 instead of 6443.
 - Configure the kubelet to connect to 443 instead of 6443.
  • Loading branch information
@roberthbailey
roberthbailey committed Apr 17, 2015
1 parent 3450bfd commit 451d97b
Show file tree
Hide file tree
Showing 7 changed files with 93 additions and 59 deletions.
42 changes: 37 additions & 5 deletions cluster/common.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ DEFAULT_KUBECONFIG="${HOME}/.kube/config"
# Assumed vars:
# KUBE_USER
# KUBE_PASSWORD
# KUBE_BEARER_TOKEN
# KUBE_MASTER_IP
# KUBECONFIG
#
Expand All @@ -47,11 +48,18 @@ function create-kubeconfig() {
"${kubectl}" config set-cluster "${CONTEXT}" --server="https://${KUBE_MASTER_IP}" \
--certificate-authority="${CA_CERT}" \
--embed-certs=true
"${kubectl}" config set-credentials "${CONTEXT}" --username="${KUBE_USER}" \
--password="${KUBE_PASSWORD}" \
--client-certificate="${KUBE_CERT}" \
--client-key="${KUBE_KEY}" \
--embed-certs=true
if [[ -z "${KUBE_USER:-}" || -z "${KUBE_PASSWORD:-}" ]]; then
"${kubectl}" config set-credentials "${CONTEXT}" --token="${KUBE_BEARER_TOKEN}" \
--client-certificate="${KUBE_CERT}" \
--client-key="${KUBE_KEY}" \
--embed-certs=true
else
"${kubectl}" config set-credentials "${CONTEXT}" --username="${KUBE_USER}" \
--password="${KUBE_PASSWORD}" \
--client-certificate="${KUBE_CERT}" \
--client-key="${KUBE_KEY}" \
--embed-certs=true
fi
"${kubectl}" config set-context "${CONTEXT}" --cluster="${CONTEXT}" --user="${CONTEXT}"
"${kubectl}" config use-context "${CONTEXT}" --cluster="${CONTEXT}"

Expand Down Expand Up @@ -106,3 +114,27 @@ function get-kubeconfig-basicauth() {
KUBE_PASSWORD=''
fi
}

# Get the bearer token for the current-context in kubeconfig if one exists.
# Assumed vars:
# KUBECONFIG # if unset, defaults to global
#
# Vars set:
# KUBE_BEARER_TOKEN
#
# KUBE_BEARER_TOKEN will be empty if no current-context is set, or the
# current-context user does not exist or contain a bearer token entry.
function get-kubeconfig-bearertoken() {
# Template to safely extract the token for the current-context user.
# The long chain of 'with' commands avoids indexing nil if any of the
# entries ("current-context", "contexts"."current-context", "users", etc)
# is missing.
# Note: we save dot ('.') to $root because the 'with' action overrides it.
# See http://golang.org/pkg/text/template/.
local token='{{$dot := .}}{{with $ctx := index $dot "current-context"}}{{range $element := (index $dot "contexts")}}{{ if eq .name $ctx }}{{ with $user := .context.user }}{{range $element := (index $dot "users")}}{{ if eq .name $user }}{{ index . "user" "token" }}{{end}}{{end}}{{end}}{{end}}{{end}}{{end}}'
KUBE_BEARER_TOKEN=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o template --template="${token}")
# Handle empty/missing token
if [[ "${KUBE_BEARER_TOKEN}" == '<no value>' ]]; then
KUBE_BEARER_TOKEN=''
fi
}
20 changes: 7 additions & 13 deletions cluster/gce/configure-vm.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,23 +245,17 @@ EOF
}

# This should only happen on cluster initialization. Uses
# MASTER_HTPASSWORD to generate the nginx/htpasswd file, and the
# KUBELET_TOKEN, plus /dev/urandom, to generate known_tokens.csv
# (KNOWN_TOKENS_FILE). After the first boot and on upgrade, these
# files exist on the master-pd and should never be touched again
# (except perhaps an additional service account, see NB below.)
# KUBE_BEARER_TOKEN, KUBELET_TOKEN, and /dev/urandom to generate
# known_tokens.csv (KNOWN_TOKENS_FILE). After the first boot and
# on upgrade, this file exists on the master-pd and should never
# be touched again (except perhaps an additional service account,
# see NB below.)
function create-salt-auth() {
local -r htpasswd_file="/srv/salt-overlay/salt/nginx/htpasswd"

if [ ! -e "${htpasswd_file}" ]; then
mkdir -p /srv/salt-overlay/salt/nginx
echo "${MASTER_HTPASSWD}" > "${htpasswd_file}"
fi

if [ ! -e "${KNOWN_TOKENS_FILE}" ]; then
mkdir -p /srv/salt-overlay/salt/kube-apiserver
(umask 077;
echo "${KUBELET_TOKEN},kubelet,kubelet" > "${KNOWN_TOKENS_FILE}")
echo "${KUBE_BEARER_TOKEN},admin,admin" > "${KNOWN_TOKENS_FILE}";
echo "${KUBELET_TOKEN},kubelet,kubelet" >> "${KNOWN_TOKENS_FILE}")

mkdir -p /srv/salt-overlay/salt/kubelet
kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
Expand Down
45 changes: 17 additions & 28 deletions cluster/gce/util.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,30 +263,21 @@ function get-password {
fi
}

# Set MASTER_HTPASSWD
function set-master-htpasswd {
python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
-b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD"
local htpasswd
MASTER_HTPASSWD=$(cat "${KUBE_TEMP}/htpasswd")
}

# Generate authentication token for admin user. Will
# read from $HOME/.kubernetes_auth if available.
# Ensure that we have a bearer token created for validating to the master.
# Will read from kubeconfig for the current context if available.
#
# Assumed vars
# KUBE_ROOT
#
# Vars set:
# KUBE_ADMIN_TOKEN
function get-admin-token {
local file="$HOME/.kubernetes_auth"
if [[ -r "$file" ]]; then
KUBE_ADMIN_TOKEN=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["BearerToken"]')
return
# KUBE_BEARER_TOKEN
function get-bearer-token() {
get-kubeconfig-bearertoken
if [[ -z "${KUBE_BEARER_TOKEN}" ]]; then
KUBE_BEARER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
fi
KUBE_ADMIN_TOKEN=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(32))')
}



# Wait for background jobs to finish. Exit with
# an error status if any of the jobs failed.
function wait-for-jobs {
Expand Down Expand Up @@ -466,7 +457,7 @@ ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false})
DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-})
DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-})
DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})
MASTER_HTPASSWD: $(yaml-quote ${MASTER_HTPASSWD})
KUBE_BEARER_TOKEN: $(yaml-quote ${KUBE_BEARER_TOKEN:-})
ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-})
MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE})
EOF
Expand Down Expand Up @@ -500,8 +491,7 @@ function write-node-env {
# variables are set:
# ensure-temp-dir
# detect-project
# get-password
# set-master-htpasswd
# get-bearer-token
#
function create-master-instance {
local address_opt=""
Expand Down Expand Up @@ -534,8 +524,7 @@ function kube-up {
ensure-temp-dir
detect-project

get-password
set-master-htpasswd
get-bearer-token

# Make sure we have the tar files staged on Google Storage
find-release-tars
Expand Down Expand Up @@ -663,8 +652,9 @@ function kube-up {
echo " up."
echo

until curl --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" --max-time 5 \
--fail --output /dev/null --silent "https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do
until curl --insecure -H "Authorization: Bearer ${KUBE_BEARER_TOKEN}" \
--max-time 5 --fail --output /dev/null --silent \
"https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do
printf "."
sleep 2
done
Expand Down Expand Up @@ -844,8 +834,7 @@ function kube-push {
detect-project
detect-master
detect-minion-names
get-password
set-master-htpasswd
get-bearer-token

# Make sure we have the tar files staged on Google Storage
find-release-tars
Expand Down
18 changes: 11 additions & 7 deletions cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@
{% set cert_file = "--tls_cert_file=/srv/kubernetes/server.cert" -%}
{% set key_file = "--tls_private_key_file=/srv/kubernetes/server.key" -%}

{% set secure_port = "--secure_port=6443" -%}
{% set secure_port = "6443" -%}
{% if grains['cloud'] is defined and grains['cloud'] == 'gce' %}
{% set secure_port = "443" -%}
{% endif -%}

{% set token_auth_file = "--token_auth_file=/dev/null" -%}

{% if grains.cloud is defined -%}
Expand Down Expand Up @@ -77,24 +81,24 @@
"/kube-apiserver",
"{{address}}",
"{{etcd_servers}}",
"{{ cloud_provider }}",
"{{ cloud_config }}",
"{{ runtime_config }}",
"{{cloud_provider}}",
"{{cloud_config}}",
"{{runtime_config}}",
"{{admission_control}}",
"--allow_privileged={{pillar['allow_privileged']}}",
"{{portal_net}}",
"{{cluster_name}}",
"{{cert_file}}",
"{{key_file}}",
"{{secure_port}}",
"--secure_port={{secure_port}}",
"{{token_auth_file}}",
"{{publicAddressOverride}}",
"{{pillar['log_level']}}"
],
"ports":[
{ "name": "https",
"containerPort": 6443,
"hostPort": 6443},{
"containerPort": {{secure_port}},
"hostPort": {{secure_port}}},{
"name": "http",
"containerPort": 7080,
"hostPort": 7080},{
Expand Down
19 changes: 14 additions & 5 deletions cluster/saltbase/salt/kubelet/default
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,24 @@
{% endif -%}

{% if grains.api_servers is defined -%}
{% set api_servers = "--api_servers=https://" + grains.api_servers + ":6443" -%}
{% set api_servers = "--api_servers=https://" + grains.api_servers -%}
{% elif grains.apiservers is defined -%} # TODO(remove after 0.16.0): Deprecated form
{% set api_servers = "--api_servers=https://" + grains.apiservers + ":6443" -%}
{% set api_servers = "--api_servers=https://" + grains.apiservers -%}
{% elif grains['roles'][0] == 'kubernetes-master' -%}
{% set master_ipv4 = salt['grains.get']('fqdn_ip4')[0] -%}
{% set api_servers = "--api_servers=https://" + master_ipv4 + ":6443" -%}
{% set api_servers = "--api_servers=https://" + master_ipv4 -%}
{% else -%}
{% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
{% set api_servers = "--api_servers=https://" + ips[0][0] + ":6443" -%}
{% set api_servers = "--api_servers=https://" + ips[0][0] -%}
{% endif -%}

{% if grains.cloud is defined -%}
{% if grains.cloud in [ 'gce' ] -%}
# TODO: remove nginx for other cloud providers.
{% set api_servers_with_port = api_servers -%}
{% endif -%}
{% else -%}
{% set api_servers_with_port = api_servers + ":6443" -%}
{% endif -%}

{% set config = "--config=/etc/kubernetes/manifests" -%}
Expand All @@ -33,4 +42,4 @@
{% set docker_root = " --docker_root=" + grains.docker_root -%}
{% endif -%}

DAEMON_ARGS="{{daemon_args}} {{api_servers}} {{hostname_override}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}}"
DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{hostname_override}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}}"
2 changes: 2 additions & 0 deletions cluster/saltbase/salt/top.sls
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ base:
- kube-controller-manager
- kube-scheduler
- monit
{% if grains['cloud'] is defined and grains['cloud'] != 'gce' %}
- nginx
{% endif %}
- cadvisor
- kube-client-tools
- kube-master-addons
Expand Down
6 changes: 5 additions & 1 deletion cluster/validate-cluster.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ source "${KUBE_ROOT}/cluster/kube-env.sh"
source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"

get-password
get-bearer-token
detect-master > /dev/null
detect-minions > /dev/null

Expand Down Expand Up @@ -89,7 +90,10 @@ for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
attempt=0
while true; do
echo -n "Attempt $((attempt+1)) at checking Kubelet installation on node ${MINION_NAMES[$i]} ..."
if [[ "$KUBERNETES_PROVIDER" != "libvirt-coreos" && "$KUBERNETES_PROVIDER" != "juju" ]]; then
if [[ "$KUBERNETES_PROVIDER" == "gce" ]]; then
curl_output=$(curl -s --insecure -H "Authorization: Bearer ${KUBE_BEARER_TOKEN}" \
"https://${KUBE_MASTER_IP}/api/v1beta1/proxy/minions/${name}/healthz")
elif [[ "$KUBERNETES_PROVIDER" != "libvirt-coreos" && "$KUBERNETES_PROVIDER" != "juju" ]]; then
curl_output=$(curl -s --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" \
"https://${KUBE_MASTER_IP}/api/v1beta1/proxy/minions/${name}/healthz")
else
Expand Down

0 comments on commit 451d97b

Please sign in to comment.