Merge pull request kubernetes#1747 from pietern/vsphere-binary-deploy

vSphere support for binary deploys
duda · Oct 29, 2014 · 78df011 · 78df011
2 parents 2c10dd8 + b656f2b
commit 78df011
Show file tree

Hide file tree

Showing 23 changed files with 593 additions and 387 deletions.
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -12,7 +12,7 @@ least one maintainer on relevant issues and PRs.
 
 * GCE: [Brendan Burns](https://github.com/brendandburns), [Joe Beda](https://github.com/jbeda), [Daniel Smith](https://github.com/lavalamp), [Tim Hockin](https://github.com/thockin)
 * Azure: [Jeff Mendoza](https://github.com/jeffmendoza)
-* VSphere: [Pieter Noordhuis](https://github.com/pietern)
+* vSphere: [Pieter Noordhuis](https://github.com/pietern)
 * Rackspace: [Ryan Richard](https://github.com/doublerr)
 * oVirt: [Federico Simoncelli](https://github.com/simon3z)
 * Local: [Derek Carr](https://github.com/derekwaynecarr)

diff --git a/README.md b/README.md
@@ -24,10 +24,10 @@ While the concepts and architecture in Kubernetes represent years of experience
   * [OpenStack](https://developer.rackspace.com/blog/running-coreos-and-kubernetes/)
   * [CloudStack](docs/getting-started-guides/cloudstack.md)
   * [Rackspace](docs/getting-started-guides/rackspace.md)
+  * [vSphere](docs/getting-started-guides/vsphere.md)
 
 * The following clouds are currently broken at Kubernetes head.  Please sync your client to `v0.3` (`git checkout v0.3`) to use these:
   * [Locally](docs/getting-started-guides/locally.md)
-  * [vSphere](docs/getting-started-guides/vsphere.md)
   * [Microsoft Azure](docs/getting-started-guides/azure.md)
 * [Kubernetes 101](https://github.com/GoogleCloudPlatform/kubernetes/tree/master/examples/walkthrough)
 * [kubecfg command line tool](https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/cli.md)

diff --git a/cluster/gce/templates/salt-master.sh b/cluster/gce/templates/salt-master.sh
@@ -38,7 +38,7 @@ cat <<EOF >/etc/salt/master.d/reactor.conf
 # React to new minions starting by running highstate on them.
 reactor:
   - 'salt/minion/*/start':
-    - /srv/reactor/start.sls
+    - /srv/reactor/highstate-new.sls
 EOF
 
 # Install Salt

diff --git a/cluster/saltbase/reactor/highstate-masters.sls b/cluster/saltbase/reactor/highstate-masters.sls
@@ -0,0 +1,10 @@
+# This runs highstate on the master node(s).
+#
+# Some of the cluster deployment scripts pass the list of minion addresses to
+# the apiserver as a command line argument. This list needs to be updated if a
+# new minion is started, so run highstate on the master(s) when this happens.
+#
+highstate_master:
+  cmd.state.highstate:
+    - tgt: 'roles:kubernetes-master'
+    - expr_form: grain
diff --git a/cluster/saltbase/reactor/highstate-minions.sls b/cluster/saltbase/reactor/highstate-minions.sls
@@ -0,0 +1,11 @@
+# This runs highstate on the minion nodes.
+#
+# Some of the cluster deployment scripts use the list of minions on the minions
+# themselves (for example: every minion is configured with static routes to
+# every other minion on a vSphere deployment). To propagate changes throughout
+# the pool, run highstate on all minions whenever a single minion starts.
+#
+highstate_minions:
+  cmd.state.highstate:
+    - tgt: 'roles:kubernetes-pool'
+    - expr_form: grain
diff --git a/cluster/saltbase/reactor/highstate-new.sls b/cluster/saltbase/reactor/highstate-new.sls
@@ -0,0 +1,4 @@
+# This runs highstate only on the NEW node, regardless of type.
+highstate_new:
+  cmd.state.highstate:
+    - tgt: {{ data['id'] }}
diff --git a/cluster/saltbase/reactor/start.sls b/cluster/saltbase/reactor/start.sls
diff --git a/cluster/saltbase/salt/base.sls b/cluster/saltbase/salt/base.sls
@@ -1,6 +1,7 @@
 pkg-core:
   pkg.installed:
     - names:
+      - curl
 {% if grains['os_family'] == 'RedHat' %}
       - python
       - git

diff --git a/cluster/saltbase/salt/docker/init.sls b/cluster/saltbase/salt/docker/init.sls
@@ -17,12 +17,16 @@ docker-repo:
     - require:
       - pkg: pkg-core
 
+{% if grains.cloud is defined %}
+{% if grains.cloud == 'gce' %}
 # The default GCE images have ip_forwarding explicitly set to 0.
 # Here we take care of commenting that out.
 /etc/sysctl.d/11-gce-network-security.conf:
   file.replace:
     - pattern: '^net.ipv4.ip_forward=0'
     - repl: '# net.ipv4.ip_forward=0'
+{% endif %}
+{% endif %}
 
 net.ipv4.ip_forward:
   sysctl.present:

diff --git a/cluster/saltbase/salt/nginx/init.sls b/cluster/saltbase/salt/nginx/init.sls
@@ -17,6 +17,9 @@ nginx:
   {% if grains.cloud == 'vagrant' %}
     {% set cert_ip=grains.fqdn_ip4 %}
   {% endif %}
+  {% if grains.cloud == 'vsphere' %}
+    {% set cert_ip=grains.ip_interfaces.eth0[0] %}
+  {% endif %}
 {% endif %}
 # If there is a pillar defined, override any defaults.
 {% if pillar['cert_ip'] is defined %}
@@ -34,6 +37,8 @@ nginx:
     - source: salt://nginx/{{certgen}} 
 {% if cert_ip is defined %}
     - args: {{cert_ip}}
+    - require:
+      - pkg: curl
 {% endif %}
     - cwd: /
     - user: root

diff --git a/cluster/vagrant/provision-master.sh b/cluster/vagrant/provision-master.sh
@@ -92,7 +92,7 @@ cat <<EOF >/etc/salt/master.d/reactor.conf
 # React to new minions starting by running highstate on them.
 reactor:
   - 'salt/minion/*/start':
-    - /srv/reactor/start.sls
+    - /srv/reactor/highstate-new.sls
 EOF
 
 cat <<EOF >/etc/salt/master.d/salt-output.conf

diff --git a/cluster/validate-cluster.sh b/cluster/validate-cluster.sh
@@ -36,7 +36,7 @@ MINIONS_FILE=/tmp/minions
 "${KUBE_ROOT}/cluster/kubecfg.sh" -template $'{{range.Items}}{{.Name}}\n{{end}}' list minions > ${MINIONS_FILE}
 
 # On vSphere, use minion IPs as their names
-if [ "$KUBERNETES_PROVIDER" == "vsphere" ]; then
+if [[ "${KUBERNETES_PROVIDER}" == "vsphere" ]]; then
   for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
     MINION_NAMES[i]=${KUBE_MINION_IP_ADDRESSES[i]}
   done

diff --git a/icebox/cluster/vsphere/config-common.sh → cluster/vsphere/config-common.sh b/icebox/cluster/vsphere/config-common.sh → cluster/vsphere/config-common.sh
@@ -14,24 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-function public-key {
-  local dir=${HOME}/.ssh
-
-  for f in $HOME/.ssh/{id_{rsa,dsa},*}.pub; do
-    if [ -r $f ]; then
-      echo $f
-      return
-    fi
-  done
-
-  echo "Can't find public key file..." 1>&2
-  exit 1
-}
-
-DISK=./kube/kube.vmdk
-GUEST_ID=debian7_64Guest
-PUBLIC_KEY_FILE=${PUBLIC_KEY_FILE-$(public-key)}
-SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null"
+SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=ERROR"
 
 # These need to be set
 #export GOVC_URL=

diff --git a/icebox/cluster/vsphere/config-default.sh → cluster/vsphere/config-default.sh b/icebox/cluster/vsphere/config-default.sh → cluster/vsphere/config-default.sh
@@ -14,10 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-source $(dirname ${BASH_SOURCE})/config-common.sh
-
 NUM_MINIONS=4
+DISK=./kube/kube.vmdk
+GUEST_ID=debian7_64Guest
+
 INSTANCE_PREFIX=kubernetes
+MASTER_TAG="${INSTANCE_PREFIX}-master"
+MINION_TAG="${INSTANCE_PREFIX}-minion"
 
 MASTER_NAME="${INSTANCE_PREFIX}-master"
 MASTER_MEMORY_MB=1024
@@ -27,3 +30,5 @@ MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
 MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
 MINION_MEMORY_MB=2048
 MINION_CPU=1
+
+PORTAL_NET="10.244.240.0/20"
diff --git a/icebox/cluster/vsphere/config-test.sh → cluster/vsphere/config-test.sh b/icebox/cluster/vsphere/config-test.sh → cluster/vsphere/config-test.sh
@@ -14,10 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-source $(dirname ${BASH_SOURCE})/config-common.sh
-
 NUM_MINIONS=2
+DISK=./kube/kube.vmdk
+GUEST_ID=debian7_64Guest
+
 INSTANCE_PREFIX="e2e-test-${USER}"
+MASTER_TAG="${INSTANCE_PREFIX}-master"
+MINION_TAG="${INSTANCE_PREFIX}-minion"
 
 MASTER_NAME="${INSTANCE_PREFIX}-master"
 MASTER_MEMORY_MB=1024
@@ -27,3 +30,5 @@ MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
 MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
 MINION_MEMORY_MB=1024
 MINION_CPU=1
+
+PORTAL_NET="10.244.240.0/20"
diff --git a/cluster/vsphere/templates/create-dynamic-salt-files.sh b/cluster/vsphere/templates/create-dynamic-salt-files.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Create the overlay files for the salt tree.  We create these in a separate
+# place so that we can blow away the rest of the salt configs on a kube-push and
+# re-apply these.
+
+mkdir -p /srv/salt-overlay/pillar
+cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
+node_instance_prefix: $NODE_INSTANCE_PREFIX
+portal_net: $PORTAL_NET
+EOF
+
+mkdir -p /srv/salt-overlay/salt/nginx
+echo $MASTER_HTPASSWD > /srv/salt-overlay/salt/nginx/htpasswd
diff --git a/icebox/cluster/vsphere/templates/hostname.sh → cluster/vsphere/templates/hostname.sh b/icebox/cluster/vsphere/templates/hostname.sh → cluster/vsphere/templates/hostname.sh
diff --git a/...ster/vsphere/templates/install-release.sh → cluster/vsphere/templates/install-release.sh b/...ster/vsphere/templates/install-release.sh → cluster/vsphere/templates/install-release.sh
@@ -14,11 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Install release
+# This script assumes that the environment variable SERVER_BINARY_TAR contains
+# the release tar to download and unpack. It is meant to be pushed to the
+# master and run.
 
-echo "Unpacking release"
-rm -rf master-release || false
-tar xzf master-release.tgz
+echo "Unpacking Salt tree"
+rm -rf kubernetes
+tar xzf "${SALT_TAR}"
 
 echo "Running release install script"
-sudo master-release/src/scripts/master-release-install.sh
+sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR}"
diff --git a/.../cluster/vsphere/templates/salt-master.sh → cluster/vsphere/templates/salt-master.sh b/.../cluster/vsphere/templates/salt-master.sh → cluster/vsphere/templates/salt-master.sh
@@ -38,23 +38,19 @@ cat <<EOF >/etc/salt/master.d/reactor.conf
 # React to new minions starting by running highstate on them.
 reactor:
   - 'salt/minion/*/start':
-    - /srv/reactor/start.sls
+    - /srv/reactor/highstate-new.sls
+    - /srv/reactor/highstate-masters.sls
+    - /srv/reactor/highstate-minions.sls
 EOF
 
-mkdir -p /srv/salt/nginx
-echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd
-
 # Install Salt
 #
 # We specify -X to avoid a race condition that can cause minion failure to
 # install.  See https://github.com/saltstack/salt-bootstrap/issues/270
 #
 # -M installs the master
-if [ ! -x /etc/init.d/salt-master ]; then
-  wget -q -O - https://bootstrap.saltstack.com | sh -s -- -M -X
-else
-  /etc/init.d/salt-master restart
-  /etc/init.d/salt-minion restart
-fi
+set +x
+wget -q -O - https://bootstrap.saltstack.com | sh -s -- -M -X
+set -x
 
 echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd
diff --git a/.../cluster/vsphere/templates/salt-minion.sh → cluster/vsphere/templates/salt-minion.sh b/.../cluster/vsphere/templates/salt-minion.sh → cluster/vsphere/templates/salt-minion.sh
@@ -18,14 +18,14 @@
 sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list
 
 # Resolve hostname of master
-if ! grep -q $MASTER_NAME /etc/hosts; then
-  echo "Adding host entry for $MASTER_NAME"
-  echo "$MASTER_IP $MASTER_NAME" >> /etc/hosts
+if ! grep -q $KUBE_MASTER /etc/hosts; then
+  echo "Adding host entry for $KUBE_MASTER"
+  echo "$KUBE_MASTER_IP $KUBE_MASTER" >> /etc/hosts
 fi
 
 # Prepopulate the name of the Master
 mkdir -p /etc/salt/minion.d
-echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
+echo "master: $KUBE_MASTER" > /etc/salt/minion.d/master.conf
 
 # Turn on debugging for salt-minion
 # echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion
@@ -48,8 +48,4 @@ EOF
 #
 # We specify -X to avoid a race condition that can cause minion failure to
 # install.  See https://github.com/saltstack/salt-bootstrap/issues/270
-if [ ! -x /etc/init.d/salt-minion ]; then
-  wget -q -O - https://bootstrap.saltstack.com | sh -s -- -X
-else
-  /etc/init.d/salt-minion restart
-fi
+wget -q -O - https://bootstrap.saltstack.com | sh -s -- -X