add ip per pod across vagrant minions

duda · Aug 26, 2014 · 2dd5789 · 2dd5789
1 parent c92b7d5
commit 2dd5789
Show file tree

Hide file tree

Showing 8 changed files with 182 additions and 4 deletions.
diff --git a/.gitignore b/.gitignore
@@ -29,6 +29,7 @@ Session.vim
 
 # Vagrant
 .vagrant
+network_closure.sh
 
 # compiled binaries in third_party
 /third_party/pkg
diff --git a/Vagrantfile b/Vagrantfile
@@ -45,7 +45,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       minion_ip = minion_ips[n]
       minion.vm.box = kube_box[kube_os]["name"]
       minion.vm.box_url = kube_box[kube_os]["box_url"]
-      minion.vm.provision "shell", inline: "/vagrant/cluster/vagrant/provision-minion.sh #{master_ip} #{num_minion} #{minion_ips_str} #{minion_ip}"
+      minion.vm.provision "shell", inline: "/vagrant/cluster/vagrant/provision-minion.sh #{master_ip} #{num_minion} #{minion_ips_str} #{minion_ip} #{minion_index}"
       minion.vm.network "private_network", ip: "#{minion_ip}"
       minion.vm.hostname = "kubernetes-minion-#{minion_index}"
     end

diff --git a/cluster/saltbase/salt/docker/init.sls b/cluster/saltbase/salt/docker/init.sls
@@ -4,6 +4,9 @@
 {% set environment_file = '/etc/default/docker' %}
 {% endif %}
 
+bridge-utils:
+  pkg.installed
+
 {% if grains['os_family'] != 'RedHat' %}
 
 docker-repo:
@@ -25,9 +28,6 @@ net.ipv4.ip_forward:
   sysctl.present:
     - value: 1
 
-bridge-utils:
-  pkg.installed
-
 cbr0:
   container_bridge.ensure:
     - cidr: {{ grains['cbr-cidr'] }}

diff --git a/cluster/saltbase/salt/sdn/init.sls b/cluster/saltbase/salt/sdn/init.sls
@@ -0,0 +1,16 @@
+{% if grains['os_family'] == 'RedHat' %}
+
+openvswitch:
+  pkg:
+    - installed
+  service.running:
+    - enable: True
+
+sdn:
+  cmd.wait:
+    - name: /vagrant/network_closure.sh
+    - watch:
+      - pkg: docker-io
+      - pkg: openvswitch
+
+{% endif %}
diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls
@@ -12,6 +12,8 @@ base:
     - nsinit
 {% if grains['cloud'] is defined and grains['cloud'] == 'azure' %}
     - openvpn-client
+{% else %}
+    - sdn
 {% endif %}
 
   'roles:kubernetes-master':

diff --git a/cluster/vagrant/pod-ip-test.sh b/cluster/vagrant/pod-ip-test.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+cd $(dirname ${BASH_SOURCE})/../../
+
+# start the cluster with 2 minions
+export KUBERNETES_NUM_MINIONS=2
+export KUBERNETES_PROVIDER=vagrant
+cluster/kube-up.sh
+
+echo "Pull an image that runs a web server"
+vagrant ssh minion-1 -- sudo docker pull dockerfile/nginx
+vagrant ssh minion-2 -- sudo docker pull dockerfile/nginx
+
+echo "Run the servers"
+vagrant ssh minion-1 -- sudo docker run -d dockerfile/nginx
+vagrant ssh minion-2 -- sudo docker run -d dockerfile/nginx
+
+echo "Run ping from minion-1 to docker bridges and to the containers on both minions"
+vagrant ssh minion-1 -- 'ping -c 10 10.244.1.1 && ping -c 10 10.244.2.1 && ping -c 10 10.244.1.3 && ping -c 10 10.244.2.3'
+echo "Same pinch from minion-2"
+vagrant ssh minion-2 -- 'ping -c 10 10.244.1.1 && ping -c 10 10.244.2.1 && ping -c 10 10.244.1.3 && ping -c 10 10.244.2.3'
+
+echo "tcp check, curl to both the running webservers from both machines"
+vagrant ssh minion-1 -- 'curl 10.244.1.3:80  && curl 10.244.2.3:80'
+vagrant ssh minion-2 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80'
+
+echo "All good, destroy the cluster"
+vagrant destroy -f
diff --git a/cluster/vagrant/provision-minion.sh b/cluster/vagrant/provision-minion.sh
@@ -52,3 +52,6 @@ if [ ! $(which salt-minion) ]; then
   systemctl enable salt-minion
   systemctl start salt-minion
 fi
+
+# run the networking setup
+$(dirname $0)/provision-network.sh $@
diff --git a/cluster/vagrant/provision-network.sh b/cluster/vagrant/provision-network.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# exit on any error
+set -e
+source $(dirname $0)/provision-config.sh
+
+MINION_IP=$4
+MINION_ID=$5
+DOCKER_BRIDGE=kbr0
+OVS_SWITCH=obr0
+GRE_TUNNEL_BASE=gre
+BRIDGE_BASE=10.244
+BRIDGE_ADDRESS=${BRIDGE_BASE}.${MINION_ID}.1
+BRIDGE_NETWORK=${BRIDGE_ADDRESS}/24
+BRIDGE_NETMASK=255.255.255.0
+NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/
+POST_NETWORK_SCRIPT=/vagrant/network_closure.sh
+
+# add docker bridge ifcfg file
+cat <<EOF > ${NETWORK_CONF_PATH}ifcfg-${DOCKER_BRIDGE}
+# Generated by yours truly
+DEVICE=${DOCKER_BRIDGE}
+ONBOOT=yes
+TYPE=Bridge
+BOOTPROTO=static
+IPADDR=${BRIDGE_ADDRESS}
+NETMASK=${BRIDGE_NETMASK}
+STP=yes
+EOF
+
+# add the ovs bridge ifcfg file
+cat <<EOF > ${NETWORK_CONF_PATH}ifcfg-${OVS_SWITCH}
+DEVICE=${OVS_SWITCH}
+ONBOOT=yes
+DEVICETYPE=ovs
+TYPE=OVSBridge
+BOOTPROTO=static
+HOTPLUG=no
+BRIDGE=${DOCKER_BRIDGE}
+EOF
+
+# now loop through all other minions and create persistent gre tunnels
+MINION_IPS=$3
+MINION_IP_ARRAY=(`echo ${MINION_IPS} | tr "," "\n"`)
+GRE_NUM=0
+for remote_ip in "${MINION_IP_ARRAY[@]}"
+do
+    if [ "${remote_ip}" == "${MINION_IP}" ]; then
+         continue
+    fi
+    ((GRE_NUM++)) || echo
+    GRE_TUNNEL=${GRE_TUNNEL_BASE}${GRE_NUM}
+    # ovs-vsctl add-port ${OVS_SWITCH} ${GRE_TUNNEL} -- set interface ${GRE_TUNNEL} type=gre options:remote_ip=${remote_ip}
+    cat <<EOF >  ${NETWORK_CONF_PATH}ifcfg-${GRE_TUNNEL}
+DEVICE=${GRE_TUNNEL}
+ONBOOT=yes
+DEVICETYPE=ovs
+TYPE=OVSTunnel
+OVS_BRIDGE=${OVS_SWITCH}
+OVS_TUNNEL_TYPE=gre
+OVS_TUNNEL_OPTIONS="options:remote_ip=${remote_ip}"
+EOF
+done
+
+# add ip route rules such that all pod traffic flows through docker bridge and consequently to the gre tunnels
+cat <<EOF > /${NETWORK_CONF_PATH}route-${DOCKER_BRIDGE}
+${BRIDGE_BASE}.0.0/16 dev ${DOCKER_BRIDGE} scope link src ${BRIDGE_ADDRESS}
+EOF
+
+
+# generate the post-configure script to be called by salt as cmd.wait
+cat <<EOF > ${POST_NETWORK_SCRIPT}
+#!/bin/bash
+
+set -e
+# NAT interface fails to revive on network restart, so OR-gate to true
+systemctl restart network.service || true
+
+# set docker bridge up, and set stp on the ovs bridge
+ip link set dev ${DOCKER_BRIDGE} up
+ovs-vsctl set Bridge ${OVS_SWITCH} stp_enable=true
+
+# modify the docker service file such that it uses the kube docker bridge and not its own
+sed -ie "s/ExecStart=\/usr\/bin\/docker -d/ExecStart=\/usr\/bin\/docker -d -b=${DOCKER_BRIDGE} --iptables=false/g" /usr/lib/systemd/system/docker.service
+systemctl daemon-reload
+systemctl restart docker.service
+
+# setup iptables masquerade rules so the pods can reach the internet
+iptables -t nat -A POSTROUTING -s ${BRIDGE_BASE}.0.0/16 ! -d ${BRIDGE_BASE}.0.0/16 -j MASQUERADE
+
+# persist please
+iptables-save >& /etc/sysconfig/iptables
+
+# self-destruct after doing the job
+#rm -f ${POST_NETWORK_SCRIPT}
+EOF
+
+chmod +x ${POST_NETWORK_SCRIPT}