Skip to content

Commit

Permalink
Rework vagrant cluster set up.
Browse files Browse the repository at this point in the history 
* Have a single config file that mirrors other cluster providers
* Warn users not to use 'vagrant up' directly
* Allow 'extra' parameters to the docker daemon.  Fixes kubernetes#2685
* Renumbers things so that they are more sane.  Master/minions are 10.245.1.x, container subnets are 10.246.x.1/24, portal is 10.247.0.0/16
  • Loading branch information
@jbeda
jbeda committed Dec 17, 2014
1 parent ff30500 commit 428aeac
Show file tree
Hide file tree
Showing 10 changed files with 212 additions and 166 deletions.
26 changes: 17 additions & 9 deletions Vagrantfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,22 @@ VAGRANTFILE_API_VERSION = "2"
# Require a recent version of vagrant otherwise some have reported errors setting host names on boxes
Vagrant.require_version ">= 1.6.2"

if ARGV.first == "up" && ENV['USING_KUBE_SCRIPTS'] != 'true'
raise Vagrant::Errors::VagrantError.new, <<END
Calling 'vagrant up' directly is not supported. Instead, please run the following:
export KUBERNETES_PROVIDER=vagrant
./cluster/kube-up.sh
END
end

# The number of minions to provision
$num_minion = (ENV['KUBERNETES_NUM_MINIONS'] || 3).to_i
$num_minion = (ENV['NUM_MINIONS'] || 3).to_i

# ip configuration
$master_ip = "10.245.1.2"
$minion_ip_base = "10.245.2."
$minion_ips = $num_minion.times.collect { |n| $minion_ip_base + "#{n+2}" }
$minion_ips_str = $minion_ips.join(",")
$master_ip = ENV['MASTER_IP']
$minion_ip_base = ENV['MINION_IP_BASE'] || ""
$minion_ips = $num_minion.times.collect { |n| $minion_ip_base + "#{n+3}" }

# Determine the OS platform to use
$kube_os = ENV['KUBERNETES_OS'] || "fedora"
Expand Down Expand Up @@ -64,9 +72,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.define "master" do |config|
customize_vm config

config.vm.provision "shell", inline: "/vagrant/cluster/vagrant/provision-master.sh #{$master_ip} #{$num_minion} #{$minion_ips_str}"
config.vm.provision "shell", run: "always", path: "#{ENV['KUBE_TEMP']}/master-start.sh"
config.vm.network "private_network", ip: "#{$master_ip}"
config.vm.hostname = "kubernetes-master"
config.vm.hostname = ENV['MASTER_NAME']
end

# Kubernetes minion
Expand All @@ -76,9 +84,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

minion_index = n+1
minion_ip = $minion_ips[n]
minion.vm.provision "shell", inline: "/vagrant/cluster/vagrant/provision-minion.sh #{$master_ip} #{$num_minion} #{$minion_ips_str} #{minion_ip} #{minion_index}"
minion.vm.provision "shell", run: "always", path: "#{ENV['KUBE_TEMP']}/minion-start-#{n}.sh"
minion.vm.network "private_network", ip: "#{minion_ip}"
minion.vm.hostname = "kubernetes-minion-#{minion_index}"
minion.vm.hostname = "#{ENV['INSTANCE_PREFIX']}-minion-#{minion_index}"
end
end

Expand Down
44 changes: 28 additions & 16 deletions cluster/vagrant/config-default.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,31 +16,43 @@

## Contains configuration values for interacting with the Vagrant cluster

# NUMBER OF MINIONS IN THE CLUSTER
NUM_MINIONS=${KUBERNETES_NUM_MINIONS-"3"}
# Number of minions in the cluster
NUM_MINIONS=${NUM_MINIONS-"3"}
export NUM_MINIONS

# IP LOCATIONS FOR INTERACTING WITH THE MASTER
export KUBE_MASTER_IP="10.245.1.2"
# The IP of the master
export MASTER_IP="10.245.1.2"

INSTANCE_PREFIX=kubernetes
MASTER_NAME="${INSTANCE_PREFIX}-master"
MASTER_TAG="${INSTANCE_PREFIX}-master"
MINION_TAG="${INSTANCE_PREFIX}-minion"
# Unable to use hostnames yet because DNS is not in cluster, so we revert external look-up name to use the minion IP
#MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
export INSTANCE_PREFIX=kubernetes
export MASTER_NAME="${INSTANCE_PREFIX}-master"

# IP LOCATIONS FOR INTERACTING WITH THE MINIONS
MINION_IP_BASE="10.245.2."
# Map out the IPs, names and container subnets of each minion
export MINION_IP_BASE="10.245.1."
MINION_CONTAINER_SUBNET_BASE="10.246"
CONTAINER_SUBNET="${MINION_CONTAINER_SUBNET_BASE}.0.0/16"
for ((i=0; i < NUM_MINIONS; i++)) do
KUBE_MINION_IP_ADDRESSES[$i]="${MINION_IP_BASE}$[$i+2]"
MINION_IP[$i]="${MINION_IP_BASE}$[$i+2]"
MINION_NAMES[$i]="${MINION_IP[$i]}"
VAGRANT_MINION_NAMES[$i]="minion-$[$i+1]"
MINION_IPS[$i]="${MINION_IP_BASE}$((i+3))"
MINION_NAMES[$i]="${INSTANCE_PREFIX}-minion-$((i+1))"
MINION_CONTAINER_SUBNETS[$i]="${MINION_CONTAINER_SUBNET_BASE}.${i}.1/24"
MINION_CONTAINER_ADDRS[$i]="${MINION_CONTAINER_SUBNET_BASE}.${i}.1"
MINION_CONTAINER_NETMASKS[$i]="255.255.255.0"
VAGRANT_MINION_NAMES[$i]="minion-$((i+1))"
done

PORTAL_NET=10.247.0.0/16

# Since this isn't exposed on the network, default to a simple user/passwd
MASTER_USER=vagrant
MASTER_PASSWD=vagrant


# Optional: Install node monitoring.
ENABLE_NODE_MONITORING=true

# Optional: Enable node logging.
ENABLE_NODE_LOGGING=true
LOGGING_DESTINATION=elasticsearch

# Extra options to set on the Docker command line. This is useful for setting
# --insecure-registry for local registries.
DOCKER_OPTS=""
26 changes: 13 additions & 13 deletions cluster/vagrant/pod-ip-test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,47 +59,47 @@ cd "${KUBE_ROOT}"
echo All verbose output will be redirected to $logfile, use --logfile option to change.

printf "Start the cluster with 2 minions .. "
export KUBERNETES_NUM_MINIONS=2
export NUM_MINIONS=2
export KUBERNETES_PROVIDER=vagrant

(cluster/kube-up.sh &>> $logfile) || true
(cluster/kube-up.sh >>"$logfile" 2>&1) || true
echoOK $?

printf "Check if minion-1 can reach kubernetes master .. "
vagrant ssh minion-1 -- ping -c 10 kubernetes-master &>> $logfile
vagrant ssh minion-1 -- ping -c 10 kubernetes-master >>"$logfile" 2>&1
echoOK $?
printf "Check if minion-2 can reach kubernetes master .. "
vagrant ssh minion-2 -- ping -c 10 kubernetes-master &>> $logfile
vagrant ssh minion-2 -- ping -c 10 kubernetes-master >>"$logfile" 2>&1
echoOK $?

printf "Pull an image that runs a web server on minion-1 .. "
vagrant ssh minion-1 -- 'sudo docker pull dockerfile/nginx' &>> $logfile
vagrant ssh minion-1 -- 'sudo docker pull kubernetes/serve_hostname' >>"$logfile" 2>&1
echoOK $?
printf "Pull an image that runs a web server on minion-2 .. "
vagrant ssh minion-2 -- 'sudo docker pull dockerfile/nginx' &>> $logfile
vagrant ssh minion-2 -- 'sudo docker pull kubernetes/serve_hostname' >>"$logfile" 2>&1
echoOK $?

printf "Run the server on minion-1 .. "
vagrant ssh minion-1 -- sudo docker run -d dockerfile/nginx &>> $logfile
vagrant ssh minion-1 -- sudo docker run -d kubernetes/serve_hostname >>"$logfile" 2>&1
echoOK $?
printf "Run the server on minion-2 .. "
vagrant ssh minion-2 -- sudo docker run -d dockerfile/nginx &>> $logfile
vagrant ssh minion-2 -- sudo docker run -d kubernetes/serve_hostname >>"$logfile" 2>&1
echoOK $?

printf "Run ping from minion-1 to docker bridges and to the containers on both minions .. "
vagrant ssh minion-1 -- 'ping -c 20 10.244.1.1 && ping -c 20 10.244.2.1 && ping -c 20 10.244.1.3 && ping -c 20 10.244.2.3' &>> $logfile
vagrant ssh minion-1 -- 'ping -c 20 10.246.0.1 && ping -c 20 10.246.1.1 && ping -c 20 10.246.0.2 && ping -c 20 10.246.1.2' >>"$logfile" 2>&1
echoOK $?
printf "Same pinch from minion-2 .. "
vagrant ssh minion-2 -- 'ping -c 20 10.244.1.1 && ping -c 20 10.244.2.1 && ping -c 20 10.244.1.3 && ping -c 20 10.244.2.3' &>> $logfile
vagrant ssh minion-2 -- 'ping -c 20 10.246.0.1 && ping -c 20 10.246.1.1 && ping -c 20 10.246.0.2 && ping -c 20 10.246.1.2' >>"$logfile" 2>&1
echoOK $?

printf "tcp check, curl to both the running webservers from minion-1 .. "
vagrant ssh minion-1 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80' &>> $logfile
vagrant ssh minion-1 -- 'curl -sS 10.246.0.2:9376 && curl -sS 10.246.1.2:9376' >>"$logfile" 2>&1
echoOK $?
printf "tcp check, curl to both the running webservers from minion-2 .. "
vagrant ssh minion-2 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80' &>> $logfile
vagrant ssh minion-2 -- 'curl -sS 10.246.0.2:9376 && curl -sS 10.246.1.2:9376' >>"$logfile" 2>&1
echoOK $?

printf "All good, destroy the cluster .. "
vagrant destroy -f &>> $logfile
vagrant destroy -f >>"$logfile" 2>&1
echoOK $?
37 changes: 0 additions & 37 deletions cluster/vagrant/provision-config.sh
View file

This file was deleted.

45 changes: 26 additions & 19 deletions cluster/vagrant/provision-master.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@
# exit on any error
set -e

KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
source "${KUBE_ROOT}/cluster/vagrant/provision-config.sh"

function release_not_found() {
echo "It looks as if you don't have a compiled version of Kubernetes. If you" >&2
echo "are running from a clone of the git repo, please run ./build/release.sh." >&2
Expand Down Expand Up @@ -50,10 +47,9 @@ fi


# Setup hosts file to support ping by hostname to each minion in the cluster from apiserver
minion_ip_array=(${MINION_IPS//,/ })
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
minion=${MINION_NAMES[$i]}
ip=${minion_ip_array[$i]}
ip=${MINION_IPS[$i]}
if [ ! "$(cat /etc/hosts | grep $minion)" ]; then
echo "Adding $minion to hosts file"
echo "$ip $minion" >> /etc/hosts
Expand Down Expand Up @@ -109,27 +105,39 @@ cat <<EOF >/etc/salt/master.d/salt-output.conf
# Minimize the amount of output to terminal
state_verbose: False
state_output: mixed
log_level: debug
log_level_logfile: debug
EOF

cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
log_level: debug
log_level_logfile: debug
EOF


# Generate and distribute a shared secret (bearer token) to
# apiserver and kubelet so that kubelet can authenticate to
# apiserver to send events.
kubelet_token=$(cat /dev/urandom | base64 | tr -d "=+/" | dd bs=32 count=1 2> /dev/null)

mkdir -p /srv/salt-overlay/salt/kube-apiserver
known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
(umask u=rw,go= ; echo "$kubelet_token,kubelet,kubelet" > $known_tokens_file)
if [[ ! -f "${known_tokens_file}" ]]; then
kubelet_token=$(cat /dev/urandom | base64 | tr -d "=+/" | dd bs=32 count=1 2> /dev/null)

mkdir -p /srv/salt-overlay/salt/kubelet
kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
(umask u=rw,go= ; echo "{\"BearerToken\": \"$kubelet_token\", \"Insecure\": true }" > $kubelet_auth_file)
mkdir -p /srv/salt-overlay/salt/kube-apiserver
known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
(umask u=rw,go= ; echo "$kubelet_token,kubelet,kubelet" > $known_tokens_file)

mkdir -p /srv/salt-overlay/salt/kubelet
kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
(umask u=rw,go= ; echo "{\"BearerToken\": \"$kubelet_token\", \"Insecure\": true }" > $kubelet_auth_file)
fi

# Configure nginx authorization
mkdir -p "$KUBE_TEMP"
mkdir -p /srv/salt-overlay/salt/nginx
python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" -b -c "${KUBE_TEMP}/htpasswd" "$MASTER_USER" "$MASTER_PASSWD"
MASTER_HTPASSWD=$(cat "${KUBE_TEMP}/htpasswd")
echo $MASTER_HTPASSWD > /srv/salt-overlay/salt/nginx/htpasswd
if [[ ! -f /srv/salt-overlay/salt/nginx/htpasswd ]]; then
python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
-b -c "/srv/salt-overlay/salt/nginx/htpasswd" \
"$MASTER_USER" "$MASTER_PASSWD"
fi

echo "Running release install script"
rm -rf /kube-install
Expand All @@ -141,7 +149,7 @@ pushd /kube-install
popd

# we will run provision to update code each time we test, so we do not want to do salt installs each time
if ! which salt-master >/dev/null 2>&1; then
if ! which salt-master &>/dev/null; then

# Configure the salt-api
cat <<EOF >/etc/salt/master.d/salt-api.conf
Expand Down Expand Up @@ -173,7 +181,6 @@ EOF
# enabling the service (which is not an error) from being printed to stderr.
SYSTEMD_LOG_LEVEL=notice systemctl enable salt-api
systemctl start salt-api

fi

if ! which salt-minion >/dev/null 2>&1; then
Expand All @@ -186,5 +193,5 @@ else
# set up to run highstate as new minions join for the first time.
echo "Executing configuration"
salt '*' mine.update
salt --force-color '*' state.highstate
salt --show-timeout --force-color '*' state.highstate
fi
21 changes: 11 additions & 10 deletions cluster/vagrant/provision-minion.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,6 @@

# exit on any error
set -e
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
source "${KUBE_ROOT}/cluster/vagrant/provision-config.sh"

MINION_IP=$4

# Setup hosts file to support ping by hostname to master
if [ ! "$(cat /etc/hosts | grep $MASTER_NAME)" ]; then
Expand All @@ -28,10 +24,9 @@ if [ ! "$(cat /etc/hosts | grep $MASTER_NAME)" ]; then
fi

# Setup hosts file to support ping by hostname to each minion in the cluster
minion_ip_array=(${MINION_IPS//,/ })
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
minion=${MINION_NAMES[$i]}
ip=${minion_ip_array[$i]}
ip=${MINION_IPS[$i]}
if [ ! "$(cat /etc/hosts | grep $minion)" ]; then
echo "Adding $minion to hosts file"
echo "$ip $minion" >> /etc/hosts
Expand All @@ -44,6 +39,11 @@ cat <<EOF >/etc/salt/minion.d/master.conf
master: '$(echo "$MASTER_NAME" | sed -e "s/'/''/g")'
EOF

cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
log_level: debug
log_level_logfile: debug
EOF

# Our minions will have a pool role to distinguish them from the master.
cat <<EOF >/etc/salt/minion.d/grains.conf
grains:
Expand All @@ -56,15 +56,16 @@ grains:
roles:
- kubernetes-pool
- kubernetes-pool-vagrant
cbr-cidr: '$(echo "$MINION_IP_RANGE" | sed -e "s/'/''/g")'
cbr-cidr: '$(echo "$CONTAINER_SUBNET" | sed -e "s/'/''/g")'
minion_ip: '$(echo "$MINION_IP" | sed -e "s/'/''/g")'
EOF

# we will run provision to update code each time we test, so we do not want to do salt install each time
if ! which salt-minion >/dev/null 2>&1; then
# Install Salt
curl -sS -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s
else
# Sometimes the minion gets wedged when it comes up along with the master.
# Restarting it here un-wedges it.
systemctl restart salt-minion.service
fi

# run the networking setup
"${KUBE_ROOT}/cluster/vagrant/provision-network.sh" $@
Loading

0 comments on commit 428aeac

Please sign in to comment.