Improve vagrant hostname support across cluster

duda · Sep 5, 2014 · 7f75aae · 7f75aae
1 parent acb3e39
commit 7f75aae
Show file tree

Hide file tree

Showing 7 changed files with 61 additions and 13 deletions.
diff --git a/cluster/vagrant/config-default.sh b/cluster/vagrant/config-default.sh
@@ -23,10 +23,16 @@ NUM_MINIONS=${KUBERNETES_NUM_MINIONS-"3"}
 export KUBE_MASTER_IP="10.245.1.2"
 export KUBERNETES_MASTER="https://10.245.1.2"
 
+INSTANCE_PREFIX=kubernetes
+MASTER_NAME="${INSTANCE_PREFIX}-master"
+MASTER_TAG="${INSTANCE_PREFIX}-master"
+MINION_TAG="${INSTANCE_PREFIX}-minion"
+MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
+
 # IP LOCATIONS FOR INTERACTING WITH THE MINIONS
 MINION_IP_BASE="10.245.2."
 for (( i=0; i <${NUM_MINIONS}; i++)) do
-	KUBE_MINION_IP_ADDRESSES[$i]="${MINION_IP_BASE}$[$i+2]"
-	MINION_NAMES[$i]="${MINION_IP_BASE}$[$i+2]"
-	VAGRANT_MINION_NAMES[$i]="minion-$[$i+1]"
+  KUBE_MINION_IP_ADDRESSES[$i]="${MINION_IP_BASE}$[$i+2]"
+  MINION_IP[$i]="${MINION_IP_BASE}$[$i+2]"
+  VAGRANT_MINION_NAMES[$i]="minion-$[$i+1]"
 done
diff --git a/cluster/vagrant/provision-master.sh b/cluster/vagrant/provision-master.sh
@@ -18,12 +18,24 @@
 set -e
 source $(dirname $0)/provision-config.sh
 
+# Setup hosts file to support ping by hostname to each minion in the cluster from apiserver
+minion_ip_array=(${MINION_IPS//,/ })
+for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+  minion=${MINION_NAMES[$i]}
+  ip=${minion_ip_array[$i]}  
+  if [ ! "$(cat /etc/hosts | grep $minion)" ]; then
+    echo "Adding $minion to hosts file"
+    echo "$ip $minion" >> /etc/hosts
+  fi  
+done
+
 # Update salt configuration
 mkdir -p /etc/salt/minion.d
 echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
 
 cat <<EOF >/etc/salt/minion.d/grains.conf
 grains:
+  node_ip: $MASTER_IP
   master_ip: $MASTER_IP
   etcd_servers: $MASTER_IP
   cloud_provider: vagrant

diff --git a/cluster/vagrant/provision-minion.sh b/cluster/vagrant/provision-minion.sh
@@ -20,20 +20,31 @@ source $(dirname $0)/provision-config.sh
 
 MINION_IP=$4
 
-# make sure each minion has an entry in hosts file for master
+# Setup hosts file to support ping by hostname to master
 if [ ! "$(cat /etc/hosts | grep $MASTER_NAME)" ]; then
-  echo "Adding host entry for $MASTER_NAME"
+  echo "Adding $MASTER_NAME to hosts file"
   echo "$MASTER_IP $MASTER_NAME" >> /etc/hosts
 fi
 
+# Setup hosts file to support ping by hostname to each minion in the cluster
+minion_ip_array=(${MINION_IPS//,/ })
+for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+  minion=${MINION_NAMES[$i]}
+  ip=${minion_ip_array[$i]}  
+  if [ ! "$(cat /etc/hosts | grep $minion)" ]; then
+    echo "Adding $minion to hosts file"
+    echo "$ip $minion" >> /etc/hosts
+  fi  
+done
+
 # Let the minion know who its master is
 mkdir -p /etc/salt/minion.d
 echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
 
 # Our minions will have a pool role to distinguish them from the master.
 cat <<EOF >/etc/salt/minion.d/grains.conf
 grains:
-  minion_ip: $MINION_IP
+  node_ip: $MINION_IP
   etcd_servers: $MASTER_IP
   roles:
     - kubernetes-pool

diff --git a/cluster/vagrant/util.sh b/cluster/vagrant/util.sh
@@ -40,6 +40,7 @@ function verify-prereqs {
 
 # Instantiate a kubernetes cluster
 function kube-up {		
+
   get-password
   vagrant up
 
@@ -73,10 +74,11 @@ function kube-up {
       done
     done
   done
-
+  
   echo
   echo "Waiting for each minion to be registered with cloud provider"
   for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+    MACHINE="${MINION_NAMES[$i]}"
     COUNT="0"
     until [ "$COUNT" -eq "1" ]; do
       $(dirname $0)/kubecfg.sh -template '{{range.Items}}{{.ID}}:{{end}}' list minions > /tmp/minions

diff --git a/docs/salt.md b/docs/salt.md
@@ -57,6 +57,8 @@ cloud_provider | (Optional) The cloud_provider used by apiserver: *gce*, *azure*
 etcd_servers | (Required) Comma-delimited list of IP addresses the apiserver and kubelet use to reach etcd
 hostnamef | (Optional) The full host name of the machine, i.e. hostname -f
 master_ip | (Optional) The IP address that the apiserver will bind against
+node_ip | (Optional) The IP address to use to address this node
+minion_ip | (Optional) Mapped to the kubelet hostname_override, K8S TODO - change this name
 roles | (Required) 1. **kubernetes-master** means this machine is the master in the kubernetes cluster.  2. **kubernetes-pool** means this machine is a kubernetes-minion.  Depending on the role, the Salt scripts will provision different resources on the machine.
 
 These keys may be leveraged by the Salt sls files to branch behavior.

diff --git a/pkg/cloudprovider/vagrant/vagrant.go b/pkg/cloudprovider/vagrant/vagrant.go
@@ -19,6 +19,7 @@ package vagrant_cloud
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -55,7 +56,7 @@ type SaltLoginResponse struct {
 // SaltMinion is a machine managed by the Salt service.
 type SaltMinion struct {
 	Roles []string `json:"roles"`
-	IP    string   `json:"minion_ip"`
+	IP    string   `json:"node_ip"`
 	Host  string   `json:"host"`
 }
 
@@ -94,8 +95,22 @@ func (v *VagrantCloud) Zones() (cloudprovider.Zones, bool) {
 
 // IPAddress returns the address of a particular machine instance.
 func (v *VagrantCloud) IPAddress(instance string) (net.IP, error) {
-	// since the instance now is the IP in the vagrant env, this is trivial no-op
-	return net.ParseIP(instance), nil
+	token, err := v.saltLogin()
+	if err != nil {
+		return nil, err
+	}
+	minions, err := v.saltMinions(token)
+	if err != nil {
+		return nil, err
+	}
+	filteredMinions := v.saltMinionsByRole(minions, "kubernetes-pool")
+	for _, minion := range filteredMinions {
+		fmt.Println("Minion: ", minion.Host, " , ", instance, " IP: ", minion.IP)
+		if minion.Host == instance {
+			return net.ParseIP(minion.IP), nil
+		}
+	}
+	return nil, fmt.Errorf("Unable to find IP address for instance:", instance)
 }
 
 // saltMinionsByRole filters a list of minions that have a matching role.
@@ -187,7 +202,7 @@ func (v *VagrantCloud) List(filter string) ([]string, error) {
 	filteredMinions := v.saltMinionsByRole(minions, "kubernetes-pool")
 	var instances []string
 	for _, instance := range filteredMinions {
-		instances = append(instances, instance.IP)
+		instances = append(instances, instance.Host)
 	}
 
 	return instances, nil

diff --git a/pkg/cloudprovider/vagrant/vagrant_test.go b/pkg/cloudprovider/vagrant/vagrant_test.go
@@ -27,7 +27,7 @@ func startSaltTestServer() *httptest.Server {
 
 	// mock responses
 	var (
-		testSaltMinionsResponse = []byte(`{ "return": [{"kubernetes-minion-1": {"kernel": "Linux", "domain": "", "zmqversion": "3.2.4", "kernelrelease": "3.11.10-301.fc20.x86_64", "pythonpath": ["/usr/bin", "/usr/lib64/python27.zip", "/usr/lib64/python2.7", "/usr/lib64/python2.7/plat-linux2", "/usr/lib64/python2.7/lib-tk", "/usr/lib64/python2.7/lib-old", "/usr/lib64/python2.7/lib-dynload", "/usr/lib64/python2.7/site-packages", "/usr/lib/python2.7/site-packages"], "etcd_servers": "10.245.1.2", "ip_interfaces": {"lo": ["127.0.0.1"], "docker0": ["172.17.42.1"], "enp0s8": ["10.245.2.2"], "p2p1": ["10.0.2.15"]}, "shell": "/bin/sh", "mem_total": 491, "saltversioninfo": [2014, 1, 7], "osmajorrelease": ["20"], "minion_ip": "10.245.2.2", "id": "kubernetes-minion-1", "osrelease": "20", "ps": "ps -efH", "server_id": 1005530826, "num_cpus": 1, "hwaddr_interfaces": {"lo": "00:00:00:00:00:00", "docker0": "56:84:7a:fe:97:99", "enp0s8": "08:00:27:17:c5:0f", "p2p1": "08:00:27:96:96:e1"}, "virtual": "VirtualBox", "osfullname": "Fedora", "master": "kubernetes-master", "ipv4": ["10.0.2.15", "10.245.2.2", "127.0.0.1", "172.17.42.1"], "ipv6": ["::1", "fe80::a00:27ff:fe17:c50f", "fe80::a00:27ff:fe96:96e1"], "cpu_flags": ["fpu", "vme", "de", "pse", "tsc", "msr", "pae", "mce", "cx8", "apic", "sep", "mtrr", "pge", "mca", "cmov", "pat", "pse36", "clflush", "mmx", "fxsr", "sse", "sse2", "syscall", "nx", "rdtscp", "lm", "constant_tsc", "rep_good", "nopl", "pni", "monitor", "ssse3", "lahf_lm"], "localhost": "kubernetes-minion-1", "lsb_distrib_id": "Fedora", "fqdn_ip4": ["127.0.0.1"], "fqdn_ip6": [], "nodename": "kubernetes-minion-1", "saltversion": "2014.1.7", "saltpath": "/usr/lib/python2.7/site-packages/salt", "pythonversion": [2, 7, 5, "final", 0], "host": "kubernetes-minion-1", "os_family": "RedHat", "oscodename": "Heisenbug", "defaultencoding": "UTF-8", "osfinger": "Fedora-20", "roles": ["kubernetes-pool"], "num_gpus": 1, "cpu_model": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", "fqdn": "kubernetes-minion-1", "osarch": "x86_64", "cpuarch": "x86_64", "gpus": [{"model": "VirtualBox Graphics Adapter", "vendor": "unknown"}], "path": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin", "os": "Fedora", "defaultlanguage": "en_US"}}]}`)
+		testSaltMinionsResponse = []byte(`{ "return": [{"kubernetes-minion-1": {"kernel": "Linux", "domain": "", "zmqversion": "3.2.4", "kernelrelease": "3.11.10-301.fc20.x86_64", "pythonpath": ["/usr/bin", "/usr/lib64/python27.zip", "/usr/lib64/python2.7", "/usr/lib64/python2.7/plat-linux2", "/usr/lib64/python2.7/lib-tk", "/usr/lib64/python2.7/lib-old", "/usr/lib64/python2.7/lib-dynload", "/usr/lib64/python2.7/site-packages", "/usr/lib/python2.7/site-packages"], "etcd_servers": "10.245.1.2", "ip_interfaces": {"lo": ["127.0.0.1"], "docker0": ["172.17.42.1"], "enp0s8": ["10.245.2.2"], "p2p1": ["10.0.2.15"]}, "shell": "/bin/sh", "mem_total": 491, "saltversioninfo": [2014, 1, 7], "osmajorrelease": ["20"], "node_ip": "10.245.2.2", "id": "kubernetes-minion-1", "osrelease": "20", "ps": "ps -efH", "server_id": 1005530826, "num_cpus": 1, "hwaddr_interfaces": {"lo": "00:00:00:00:00:00", "docker0": "56:84:7a:fe:97:99", "enp0s8": "08:00:27:17:c5:0f", "p2p1": "08:00:27:96:96:e1"}, "virtual": "VirtualBox", "osfullname": "Fedora", "master": "kubernetes-master", "ipv4": ["10.0.2.15", "10.245.2.2", "127.0.0.1", "172.17.42.1"], "ipv6": ["::1", "fe80::a00:27ff:fe17:c50f", "fe80::a00:27ff:fe96:96e1"], "cpu_flags": ["fpu", "vme", "de", "pse", "tsc", "msr", "pae", "mce", "cx8", "apic", "sep", "mtrr", "pge", "mca", "cmov", "pat", "pse36", "clflush", "mmx", "fxsr", "sse", "sse2", "syscall", "nx", "rdtscp", "lm", "constant_tsc", "rep_good", "nopl", "pni", "monitor", "ssse3", "lahf_lm"], "localhost": "kubernetes-minion-1", "lsb_distrib_id": "Fedora", "fqdn_ip4": ["127.0.0.1"], "fqdn_ip6": [], "nodename": "kubernetes-minion-1", "saltversion": "2014.1.7", "saltpath": "/usr/lib/python2.7/site-packages/salt", "pythonversion": [2, 7, 5, "final", 0], "host": "kubernetes-minion-1", "os_family": "RedHat", "oscodename": "Heisenbug", "defaultencoding": "UTF-8", "osfinger": "Fedora-20", "roles": ["kubernetes-pool"], "num_gpus": 1, "cpu_model": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", "fqdn": "kubernetes-minion-1", "osarch": "x86_64", "cpuarch": "x86_64", "gpus": [{"model": "VirtualBox Graphics Adapter", "vendor": "unknown"}], "path": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin", "os": "Fedora", "defaultlanguage": "en_US"}}]}`)
 		testSaltLoginResponse   = []byte(`{ "return": [{"perms": [".*"], "start": 1407355696.564397, "token": "ca74fa1c48ce40e204a1e820d2fa14b7cf033137", "expire": 1407398896.564398, "user": "vagrant", "eauth": "pam"}]}`)
 		testSaltFailure         = []byte(`failure`)
 	)
@@ -73,7 +73,7 @@ func TestVagrantCloud(t *testing.T) {
 		t.Fatalf("Incorrect number of instances returned")
 	}
 
-	if instances[0] != "10.245.2.2" {
+	if instances[0] != "kubernetes-minion-1" {
 		t.Fatalf("Invalid instance returned")
 	}