Skip to content

Commit

Permalink
Remove nginx and replace basic auth with bearer token auth for GCE.
Browse files Browse the repository at this point in the history 
 - Configure the apiserver to listen securely on 443 instead of 6443.
 - Configure the kubelet to connect to 443 instead of 6443.
 - Update documentation to refer to bearer tokens instead of basic auth.
  • Loading branch information
@roberthbailey
roberthbailey committed Apr 21, 2015
1 parent 9d5da50 commit 049e4ac
Show file tree
Hide file tree
Showing 8 changed files with 127 additions and 95 deletions.
37 changes: 35 additions & 2 deletions cluster/common.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ DEFAULT_KUBECONFIG="${HOME}/.kube/config"
# Assumed vars:
# KUBE_USER
# KUBE_PASSWORD
# KUBE_BEARER_TOKEN
# KUBE_MASTER_IP
# KUBECONFIG
# CONTEXT
Expand Down Expand Up @@ -56,10 +57,17 @@ function create-kubeconfig() {
"--embed-certs=true"
)
fi
local user_args=(
local user_args=()
if [[ -z "${KUBE_USER:-}" || -z "${KUBE_PASSWORD:-}" ]]; then
user_args+=(
"--token=${KUBE_BEARER_TOKEN}"
)
else
user_args+=(
"--username=${KUBE_USER}"
"--password=${KUBE_PASSWORD}"
)
)
fi
if [[ ! -z "${KUBE_CERT:-}" && ! -z "${KUBE_KEY:-}" ]]; then
user_args+=(
"--client-certificate=${KUBE_CERT}"
Expand Down Expand Up @@ -124,3 +132,28 @@ function get-kubeconfig-basicauth() {
KUBE_PASSWORD=''
fi
}

# Get the bearer token for the current-context in kubeconfig if one exists.
# Assumed vars:
# KUBECONFIG # if unset, defaults to global
#
# Vars set:
# KUBE_BEARER_TOKEN
#
# KUBE_BEARER_TOKEN will be empty if no current-context is set, or the
# current-context user does not exist or contain a bearer token entry.
function get-kubeconfig-bearertoken() {
export KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG}
# Template to safely extract the token for the current-context user.
# The long chain of 'with' commands avoids indexing nil if any of the
# entries ("current-context", "contexts"."current-context", "users", etc)
# is missing.
# Note: we save dot ('.') to $root because the 'with' action overrides it.
# See http://golang.org/pkg/text/template/.
local token='{{$dot := .}}{{with $ctx := index $dot "current-context"}}{{range $element := (index $dot "contexts")}}{{ if eq .name $ctx }}{{ with $user := .context.user }}{{range $element := (index $dot "users")}}{{ if eq .name $user }}{{ index . "user" "token" }}{{end}}{{end}}{{end}}{{end}}{{end}}{{end}}'
KUBE_BEARER_TOKEN=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o template --template="${token}")
# Handle empty/missing token
if [[ "${KUBE_BEARER_TOKEN}" == '<no value>' ]]; then
KUBE_BEARER_TOKEN=''
fi
}
20 changes: 7 additions & 13 deletions cluster/gce/configure-vm.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,23 +251,17 @@ EOF
}

# This should only happen on cluster initialization. Uses
# MASTER_HTPASSWORD to generate the nginx/htpasswd file, and the
# KUBELET_TOKEN, plus /dev/urandom, to generate known_tokens.csv
# (KNOWN_TOKENS_FILE). After the first boot and on upgrade, these
# files exist on the master-pd and should never be touched again
# (except perhaps an additional service account, see NB below.)
# KUBE_BEARER_TOKEN, KUBELET_TOKEN, and /dev/urandom to generate
# known_tokens.csv (KNOWN_TOKENS_FILE). After the first boot and
# on upgrade, this file exists on the master-pd and should never
# be touched again (except perhaps an additional service account,
# see NB below.)
function create-salt-auth() {
local -r htpasswd_file="/srv/salt-overlay/salt/nginx/htpasswd"

if [ ! -e "${htpasswd_file}" ]; then
mkdir -p /srv/salt-overlay/salt/nginx
echo "${MASTER_HTPASSWD}" > "${htpasswd_file}"
fi

if [ ! -e "${KNOWN_TOKENS_FILE}" ]; then
mkdir -p /srv/salt-overlay/salt/kube-apiserver
(umask 077;
echo "${KUBELET_TOKEN},kubelet,kubelet" > "${KNOWN_TOKENS_FILE}")
echo "${KUBE_BEARER_TOKEN},admin,admin" > "${KNOWN_TOKENS_FILE}";
echo "${KUBELET_TOKEN},kubelet,kubelet" >> "${KNOWN_TOKENS_FILE}")

mkdir -p /srv/salt-overlay/salt/kubelet
kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
Expand Down
45 changes: 17 additions & 28 deletions cluster/gce/util.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,30 +263,21 @@ function get-password {
fi
}

# Set MASTER_HTPASSWD
function set-master-htpasswd {
python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
-b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD"
local htpasswd
MASTER_HTPASSWD=$(cat "${KUBE_TEMP}/htpasswd")
}

# Generate authentication token for admin user. Will
# read from $HOME/.kubernetes_auth if available.
# Ensure that we have a bearer token created for validating to the master.
# Will read from kubeconfig for the current context if available.
#
# Assumed vars
# KUBE_ROOT
#
# Vars set:
# KUBE_ADMIN_TOKEN
function get-admin-token {
local file="$HOME/.kubernetes_auth"
if [[ -r "$file" ]]; then
KUBE_ADMIN_TOKEN=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["BearerToken"]')
return
# KUBE_BEARER_TOKEN
function get-bearer-token() {
get-kubeconfig-bearertoken
if [[ -z "${KUBE_BEARER_TOKEN:-}" ]]; then
KUBE_BEARER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
fi
KUBE_ADMIN_TOKEN=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(32))')
}



# Wait for background jobs to finish. Exit with
# an error status if any of the jobs failed.
function wait-for-jobs {
Expand Down Expand Up @@ -473,7 +464,7 @@ ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false})
DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-})
DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-})
DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})
MASTER_HTPASSWD: $(yaml-quote ${MASTER_HTPASSWD})
KUBE_BEARER_TOKEN: $(yaml-quote ${KUBE_BEARER_TOKEN})
ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-})
MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE})
EOF
Expand Down Expand Up @@ -507,8 +498,7 @@ function write-node-env {
# variables are set:
# ensure-temp-dir
# detect-project
# get-password
# set-master-htpasswd
# get-bearer-token
#
function create-master-instance {
local address_opt=""
Expand Down Expand Up @@ -541,8 +531,7 @@ function kube-up {
ensure-temp-dir
detect-project

get-password
set-master-htpasswd
get-bearer-token

# Make sure we have the tar files staged on Google Storage
find-release-tars
Expand Down Expand Up @@ -670,8 +659,9 @@ function kube-up {
echo " up."
echo

until curl --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" --max-time 5 \
--fail --output /dev/null --silent "https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do
until curl --insecure -H "Authorization: Bearer ${KUBE_BEARER_TOKEN}" \
--max-time 5 --fail --output /dev/null --silent \
"https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do
printf "."
sleep 2
done
Expand Down Expand Up @@ -850,8 +840,7 @@ function kube-push {
detect-project
detect-master
detect-minion-names
get-password
set-master-htpasswd
get-bearer-token

# Make sure we have the tar files staged on Google Storage
find-release-tars
Expand Down
18 changes: 11 additions & 7 deletions cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@
{% set cert_file = "--tls_cert_file=/srv/kubernetes/server.cert" -%}
{% set key_file = "--tls_private_key_file=/srv/kubernetes/server.key" -%}

{% set secure_port = "--secure_port=6443" -%}
{% set secure_port = "6443" -%}
{% if grains['cloud'] is defined and grains['cloud'] == 'gce' %}
{% set secure_port = "443" -%}
{% endif -%}

{% set token_auth_file = "--token_auth_file=/dev/null" -%}

{% if grains.cloud is defined -%}
Expand Down Expand Up @@ -77,24 +81,24 @@
"/kube-apiserver",
"{{address}}",
"{{etcd_servers}}",
"{{ cloud_provider }}",
"{{ cloud_config }}",
"{{ runtime_config }}",
"{{cloud_provider}}",
"{{cloud_config}}",
"{{runtime_config}}",
"{{admission_control}}",
"--allow_privileged={{pillar['allow_privileged']}}",
"{{portal_net}}",
"{{cluster_name}}",
"{{cert_file}}",
"{{key_file}}",
"{{secure_port}}",
"--secure_port={{secure_port}}",
"{{token_auth_file}}",
"{{publicAddressOverride}}",
"{{pillar['log_level']}}"
],
"ports":[
{ "name": "https",
"containerPort": 6443,
"hostPort": 6443},{
"containerPort": {{secure_port}},
"hostPort": {{secure_port}}},{
"name": "http",
"containerPort": 7080,
"hostPort": 7080},{
Expand Down
17 changes: 12 additions & 5 deletions cluster/saltbase/salt/kubelet/default
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,22 @@
{% endif -%}

{% if grains.api_servers is defined -%}
{% set api_servers = "--api_servers=https://" + grains.api_servers + ":6443" -%}
{% set api_servers = "--api_servers=https://" + grains.api_servers -%}
{% elif grains.apiservers is defined -%} # TODO(remove after 0.16.0): Deprecated form
{% set api_servers = "--api_servers=https://" + grains.apiservers + ":6443" -%}
{% set api_servers = "--api_servers=https://" + grains.apiservers -%}
{% elif grains['roles'][0] == 'kubernetes-master' -%}
{% set master_ipv4 = salt['grains.get']('fqdn_ip4')[0] -%}
{% set api_servers = "--api_servers=https://" + master_ipv4 + ":6443" -%}
{% set api_servers = "--api_servers=https://" + master_ipv4 -%}
{% else -%}
{% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
{% set api_servers = "--api_servers=https://" + ips[0][0] + ":6443" -%}
{% set api_servers = "--api_servers=https://" + ips[0][0] -%}
{% endif -%}

# TODO: remove nginx for other cloud providers.
{% if grains['cloud'] is defined and grains['cloud'] == 'gce' -%}
{% set api_servers_with_port = api_servers -%}
{% else -%}
{% set api_servers_with_port = api_servers + ":6443" -%}
{% endif -%}

{% set config = "--config=/etc/kubernetes/manifests" -%}
Expand All @@ -33,4 +40,4 @@
{% set docker_root = " --docker_root=" + grains.docker_root -%}
{% endif -%}

DAEMON_ARGS="{{daemon_args}} {{api_servers}} {{hostname_override}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}}"
DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{hostname_override}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}}"
2 changes: 2 additions & 0 deletions cluster/saltbase/salt/top.sls
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ base:
- kube-controller-manager
- kube-scheduler
- monit
{% if grains['cloud'] is defined and grains['cloud'] != 'gce' %}
- nginx
{% endif %}
- cadvisor
- kube-client-tools
- kube-master-addons
Expand Down
25 changes: 14 additions & 11 deletions docs/accessing_the_api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,20 @@ HTTP on 3 ports:
- only GET requests are allowed.
- requests are rate limited
3. Secure Port
- default is port 6443, change with `-secure_port`
- default is port 443, change with `-secure_port`
- default IP is first non-localhost network interface, change with `-public_address_override`
- serves HTTPS. Set cert with `-tls_cert_file` and key with `-tls_private_key_file`.
- uses token-file based [authentication](./authentication.md).
- uses token-file or client-certificate based [authentication](./authentication.md).
- uses policy-based [authorization](./authorization.md).

## Proxies and Firewall rules

Additionally, in typical configurations (i.e. GCE), there is a proxy (nginx) running
Additionally, in some configurations there is a proxy (nginx) running
on the same machine as the apiserver process. The proxy serves HTTPS protected
by Basic Auth on port 443, and proxies to the apiserver on localhost:8080.
Typically, firewall rules will allow HTTPS access to port 443.
by Basic Auth on port 443, and proxies to the apiserver on localhost:8080. In
these configurations the secure port is typically set to 6443.

A firewall rule is typically configured to allow external HTTPS access to port 443.

The above are defaults and reflect how Kubernetes is deployed to GCE using
kube-up.sh. Other cloud providers may vary.
Expand All @@ -42,15 +44,15 @@ There are three differently configured serving ports because there are a
variety of uses cases:
1. Clients outside of a Kubernetes cluster, such as human running `kubectl`
on desktop machine. Currently, accesses the Localhost Port via a proxy (nginx)
running on the `kubernetes-master` machine. Proxy uses Basic Auth.
running on the `kubernetes-master` machine. Proxy uses bearer token authentication.
2. Processes running in Containers on Kubernetes that need to do read from
the apiserver. Currently, these can use Readonly Port.
3. Scheduler and Controller-manager processes, which need to do read-write
API operations. Currently, these have to run on the
API operations. Currently, these have to run on the
operations on the apiserver. Currently, these have to run on the same
host as the apiserver and use the Localhost Port.
4. Kubelets, which need to do read-write API operations and are necessarily
on different machines than the apiserver. Kubelet uses the Secure Port
4. Kubelets, which need to do read-write API operations and are necessarily
on different machines than the apiserver. Kubelet uses the Secure Port
to get their pods, to find the services that a pod can see, and to
write events. Credentials are distributed to kubelets at cluster
setup time.
Expand All @@ -59,13 +61,14 @@ variety of uses cases:
- Policy will limit the actions kubelets can do via the authed port.
- Kube-proxy currently uses the readonly port to read services and endpoints,
but will eventually use the auth port.
- Kubelets may change from token-based authentication to cert-based-auth.
- Kubelets will change from token-based authentication to cert-based-auth.
- Scheduler and Controller-manager will use the Secure Port too. They
will then be able to run on different machines than the apiserver.
- A general mechanism will be provided for [giving credentials to
pods](
https://github.com/GoogleCloudPlatform/kubernetes/issues/1907).
- The Readonly Port will no longer be needed and will be removed.
- The Readonly Port will no longer be needed and [will be removed](
https://github.com/GoogleCloudPlatform/kubernetes/issues/5921).
- Clients, like kubectl, will all support token-based auth, and the
Localhost will no longer be needed, and will not be the default.
However, the localhost port may continue to be an option for
Expand Down
Loading

0 comments on commit 049e4ac

Please sign in to comment.