Skip to content

Commit

Permalink
Merge pull request #6427 from zmerlynn/upgrade_master
Browse files Browse the repository at this point in the history 
First hack at upgrade script for GCE
  • Loading branch information
@rjnagal
rjnagal committed Apr 6, 2015
2 parents fe735d3 + 616c6be commit 1ecb926
Show file tree
Hide file tree
Showing 3 changed files with 300 additions and 63 deletions.
81 changes: 50 additions & 31 deletions cluster/gce/configure-vm.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ set -o pipefail
# If we have any arguments at all, this is a push and not just setup.
is_push=$@

readonly KNOWN_TOKENS_FILE="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"

function ensure-install-dir() {
INSTALL_DIR="/var/cache/kubernetes-install"
mkdir -p ${INSTALL_DIR}
Expand Down Expand Up @@ -55,17 +57,6 @@ for k,v in yaml.load(sys.stdin).iteritems():
print "readonly {var}={value}".format(var = k, value = pipes.quote(str(v)))
''' < "${kube_env_yaml}")

# We bake the KUBELET_TOKEN in separately to avoid auth information
# having to be re-communicated on kube-push. (Otherwise the client
# has to keep the bearer token around to handle generating a valid
# kube-env.)
if [[ -z "${KUBELET_TOKEN:-}" ]]; then
until KUBELET_TOKEN=$(curl-metadata kube-token); do
echo 'Waiting for metadata KUBELET_TOKEN...'
sleep 3
done
fi

# Infer master status from presence in node pool
if [[ $(hostname) = ${NODE_INSTANCE_PREFIX}* ]]; then
KUBERNETES_MASTER="false"
Expand All @@ -82,6 +73,19 @@ for k,v in yaml.load(sys.stdin).iteritems():
fi
}

function ensure-kube-token() {
# We bake the KUBELET_TOKEN in separately to avoid auth information
# having to be re-communicated on kube-push. (Otherwise the client
# has to keep the bearer token around to handle generating a valid
# kube-env.)
if [[ -z "${KUBELET_TOKEN:-}" ]] && [[ ! -e "${KNOWN_TOKENS_FILE}" ]]; then
until KUBELET_TOKEN=$(curl-metadata kube-token); do
echo 'Waiting for metadata KUBELET_TOKEN...'
sleep 3
done
fi
}

function remove-docker-artifacts() {
echo "== Deleting docker0 =="
# Forcibly install bridge-utils (options borrowed from Salt logs).
Expand Down Expand Up @@ -240,27 +244,41 @@ admission_control: '$(echo "$ADMISSION_CONTROL" | sed -e "s/'/''/g")'
EOF
}

# This should only happen on cluster initialization
# This should only happen on cluster initialization. Uses
# MASTER_HTPASSWORD to generate the nginx/htpasswd file, and the
# KUBELET_TOKEN, plus /dev/urandom, to generate known_tokens.csv
# (KNOWN_TOKENS_FILE). After the first boot and on upgrade, these
# files exist on the master-pd and should never be touched again
# (except perhaps an additional service account, see NB below.)
function create-salt-auth() {
mkdir -p /srv/salt-overlay/salt/nginx
echo "${MASTER_HTPASSWD}" > /srv/salt-overlay/salt/nginx/htpasswd

mkdir -p /srv/salt-overlay/salt/kube-apiserver
known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
(umask 077;
echo "${KUBELET_TOKEN},kubelet,kubelet" > "${known_tokens_file}")

mkdir -p /srv/salt-overlay/salt/kubelet
kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
(umask 077;
echo "{\"BearerToken\": \"${KUBELET_TOKEN}\", \"Insecure\": true }" > "${kubelet_auth_file}")

# Generate tokens for other "service accounts". Append to known_tokens.
local -r service_accounts=("system:scheduler" "system:controller_manager" "system:logging" "system:monitoring" "system:dns")
for account in "${service_accounts[@]}"; do
token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
echo "${token},${account},${account}" >> "${known_tokens_file}"
done
local -r htpasswd_file="/srv/salt-overlay/salt/nginx/htpasswd"

if [ ! -e "${htpasswd_file}" ]; then
mkdir -p /srv/salt-overlay/salt/nginx
echo "${MASTER_HTPASSWD}" > "${htpasswd_file}"
fi

if [ ! -e "${KNOWN_TOKENS_FILE}" ]; then
mkdir -p /srv/salt-overlay/salt/kube-apiserver
(umask 077;
echo "${KUBELET_TOKEN},kubelet,kubelet" > "${KNOWN_TOKENS_FILE}")

mkdir -p /srv/salt-overlay/salt/kubelet
kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
(umask 077;
echo "{\"BearerToken\": \"${KUBELET_TOKEN}\", \"Insecure\": true }" > "${kubelet_auth_file}")

# Generate tokens for other "service accounts". Append to known_tokens.
#
# NB: If this list ever changes, this script actually has to
# change to detect the existence of this file, kill any deleted
# old tokens and add any new tokens (to handle the upgrade case).
local -r service_accounts=("system:scheduler" "system:controller_manager" "system:logging" "system:monitoring" "system:dns")
for account in "${service_accounts[@]}"; do
token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
echo "${token},${account},${account}" >> "${KNOWN_TOKENS_FILE}"
done
fi
}

function download-release() {
Expand Down Expand Up @@ -390,6 +408,7 @@ if [[ -z "${is_push}" ]]; then
ensure-install-dir
set-kube-env
[[ "${KUBERNETES_MASTER}" == "true" ]] && mount-master-pd
ensure-kube-token
create-salt-pillar
create-salt-auth
download-release
Expand Down
175 changes: 175 additions & 0 deletions cluster/gce/upgrade.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
#!/bin/bash

# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# !!!EXPERIMENTAL !!! Upgrade script for GCE. Expect this to get
# rewritten in Go in relatively short order, but it allows us to start
# testing the concepts.

set -o errexit
set -o nounset
set -o pipefail

# VERSION_REGEX matches things like "v0.13.1"
readonly VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$"

# CI_VERSION_REGEX matches things like "v0.14.1-341-ge0c9d9e"
readonly CI_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-(.*)$"

if [[ "${KUBERNETES_PROVIDER:-gce}" != "gce" ]]; then
echo "!!! ${1} only works on GCE" >&2
exit 1
fi

KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
source "${KUBE_ROOT}/cluster/kube-env.sh"
source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"

function usage() {
echo "!!! EXPERIMENTAL !!!"
echo ""
echo "${0} [-M|-N] <release or continuous integration version>"
echo " Upgrades master and nodes by default"
echo " -M: Upgrade master only"
echo " -N: Upgrade nodes only"
echo ""
echo "(... Fetching current release versions ...)"
echo ""

local latest_release
local latest_stable
local latest_ci

latest_stable=$(gsutil cat gs://kubernetes-release/release/stable.txt)
latest_release=$(gsutil cat gs://kubernetes-release/release/latest.txt)
latest_ci=$(gsutil cat gs://kubernetes-release/ci/latest.txt)

echo "To upgrade to:"
echo " latest stable: ${0} ${latest_stable}"
echo " latest release: ${0} ${latest_release}"
echo " latest ci: ${0} ${latest_ci}"
}

function upgrade-master() {
echo "== Upgrading master to ${SERVER_BINARY_TAR_URL}. Do not interrupt, deleting master instance. =="

ensure-temp-dir
detect-project
detect-master
get-password
set-master-htpasswd

# Delete the master instance. Note that the master-pd is created
# with auto-delete=no, so it should not be deleted.
gcloud compute instances delete \
--project "${PROJECT}" \
--quiet \
--zone "${ZONE}" \
"${MASTER_NAME}"

create-master-instance "${MASTER_NAME}-ip"
wait-for-master
}

function wait-for-master() {
echo "== Waiting for new master to respond to API requests =="

until curl --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" --max-time 5 \
--fail --output /dev/null --silent "https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do
printf "."
sleep 2
done

echo "== Done =="
}

function upgrade-nodes() {
echo "== Upgrading nodes to ${SERVER_BINARY_TAR_URL}. =="
ensure-temp-dir
detect-project
detect-minion-names
get-password
set-master-htpasswd
kube-update-nodes upgrade
echo "== Done =="
}

function tars_from_version() {
version=${1-}

if [[ ${version} =~ ${VERSION_REGEX} ]]; then
SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${version}/kubernetes-server-linux-amd64.tar.gz"
SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${version}/kubernetes-salt.tar.gz"
elif [[ ${version} =~ ${CI_VERSION_REGEX} ]]; then
SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${version}/kubernetes-server-linux-amd64.tar.gz"
SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${version}/kubernetes-salt.tar.gz"
else
echo "!!! Version not provided or version doesn't match regexp" >&2
exit 1
fi

if ! curl -Ss --range 0-1 ${SERVER_BINARY_TAR_URL} >&/dev/null; then
echo "!!! Can't find release at ${SERVER_BINARY_TAR_URL}" >&2
exit 1
fi

echo "== Release ${version} validated =="
}

master_upgrade=true
node_upgrade=true

while getopts ":MNh" opt; do
case ${opt} in
M)
node_upgrade=false
;;
N)
master_upgrade=false
;;
h)
usage
exit 0
;;
\?)
echo "Invalid option: -$OPTARG" >&2
usage
exit 1
;;
esac
done
shift $((OPTIND-1))

if [[ $# -lt 1 ]]; then
usage
exit 1
fi

if [[ "${master_upgrade}" == "false" ]] && [[ "${node_upgrade}" == "false" ]]; then
echo "Can't specify both -M and -N" >&2
exit 1
fi

tars_from_version ${1}

if [[ "${master_upgrade}" == "true" ]]; then
upgrade-master
fi

if [[ "${node_upgrade}" == "true" ]]; then
upgrade-nodes
fi

"${KUBE_ROOT}/cluster/validate-cluster.sh"
Loading

0 comments on commit 1ecb926

Please sign in to comment.