Fix vagrant networking. Include the master as part of overlay. And re…

…move STP for efficiency.
duda · Feb 6, 2015 · b61ea1b · b61ea1b
1 parent 617e1ca
commit b61ea1b
Show file tree

Hide file tree

Showing 6 changed files with 71 additions and 76 deletions.
diff --git a/Vagrantfile b/Vagrantfile
@@ -17,7 +17,7 @@ END
 end
 
 # The number of minions to provision
-$num_minion = (ENV['NUM_MINIONS'] || 1).to_i
+$num_minion = (ENV['NUM_MINIONS'] || 2).to_i
 
 # ip configuration
 $master_ip = ENV['MASTER_IP']

diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls
@@ -40,6 +40,9 @@ base:
     - kube-addons
 {% if grains['cloud'] is defined and grains['cloud'] == 'azure' %}
     - openvpn
+{% else %}
+    - docker
+    - sdn
 {% endif %}
 
   'roles:kubernetes-pool-vsphere':

diff --git a/cluster/vagrant/config-default.sh b/cluster/vagrant/config-default.sh
@@ -17,7 +17,7 @@
 ## Contains configuration values for interacting with the Vagrant cluster
 
 # Number of minions in the cluster
-NUM_MINIONS=${NUM_MINIONS-"1"}
+NUM_MINIONS=${NUM_MINIONS-"2"}
 export NUM_MINIONS
 
 # The IP of the master
@@ -29,12 +29,15 @@ export MASTER_NAME="${INSTANCE_PREFIX}-master"
 # Map out the IPs, names and container subnets of each minion
 export MINION_IP_BASE="10.245.1."
 MINION_CONTAINER_SUBNET_BASE="10.246"
+MASTER_CONTAINER_NETMASK="255.255.255.0"
+MASTER_CONTAINER_ADDR="${MINION_CONTAINER_SUBNET_BASE}.0.1"
+MASTER_CONTAINER_SUBNET="${MINION_CONTAINER_SUBNET_BASE}.0.1/24"
 CONTAINER_SUBNET="${MINION_CONTAINER_SUBNET_BASE}.0.0/16"
 for ((i=0; i < NUM_MINIONS; i++)) do
   MINION_IPS[$i]="${MINION_IP_BASE}$((i+3))"
   MINION_NAMES[$i]="${INSTANCE_PREFIX}-minion-$((i+1))"
-  MINION_CONTAINER_SUBNETS[$i]="${MINION_CONTAINER_SUBNET_BASE}.${i}.1/24"
-  MINION_CONTAINER_ADDRS[$i]="${MINION_CONTAINER_SUBNET_BASE}.${i}.1"
+  MINION_CONTAINER_SUBNETS[$i]="${MINION_CONTAINER_SUBNET_BASE}.$((i+1)).1/24"
+  MINION_CONTAINER_ADDRS[$i]="${MINION_CONTAINER_SUBNET_BASE}.$((i+1)).1"
   MINION_CONTAINER_NETMASKS[$i]="255.255.255.0"
   VAGRANT_MINION_NAMES[$i]="minion-$((i+1))"
 done
@@ -69,4 +72,4 @@ DNS_REPLICAS=1
 
 # Optional: Enable setting flags for kube-apiserver to turn on behavior in active-dev
 RUNTIME_CONFIG=""
-#RUNTIME_CONFIG="api/v1beta3"
+#RUNTIME_CONFIG="api/v1beta3"
diff --git a/cluster/vagrant/provision-network.sh b/cluster/vagrant/provision-network.sh
@@ -16,63 +16,15 @@
 
 DOCKER_BRIDGE=kbr0
 OVS_SWITCH=obr0
-GRE_TUNNEL_BASE=gre
+DOCKER_OVS_TUN=tun0
+TUNNEL_BASE=gre
 NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/
 POST_NETWORK_SCRIPT_DIR=/kubernetes-vagrant
 POST_NETWORK_SCRIPT=${POST_NETWORK_SCRIPT_DIR}/network_closure.sh
 
 # ensure location of POST_NETWORK_SCRIPT exists
 mkdir -p $POST_NETWORK_SCRIPT_DIR
 
-# add docker bridge ifcfg file
-cat <<EOF > ${NETWORK_CONF_PATH}ifcfg-${DOCKER_BRIDGE}
-# Generated by yours truly
-DEVICE=${DOCKER_BRIDGE}
-ONBOOT=yes
-TYPE=Bridge
-BOOTPROTO=static
-IPADDR=${MINION_CONTAINER_ADDR}
-NETMASK=${MINION_CONTAINER_NETMASK}
-STP=yes
-EOF
-
-# add the ovs bridge ifcfg file
-cat <<EOF > ${NETWORK_CONF_PATH}ifcfg-${OVS_SWITCH}
-DEVICE=${OVS_SWITCH}
-ONBOOT=yes
-DEVICETYPE=ovs
-TYPE=OVSBridge
-BOOTPROTO=static
-HOTPLUG=no
-BRIDGE=${DOCKER_BRIDGE}
-EOF
-
-# now loop through all other minions and create persistent gre tunnels
-GRE_NUM=0
-for remote_ip in "${MINION_IPS[@]}"
-do
-    if [ "${remote_ip}" == "${MINION_IP}" ]; then
-         continue
-    fi
-    ((GRE_NUM++)) || echo
-    GRE_TUNNEL=${GRE_TUNNEL_BASE}${GRE_NUM}
-    # ovs-vsctl add-port ${OVS_SWITCH} ${GRE_TUNNEL} -- set interface ${GRE_TUNNEL} type=gre options:remote_ip=${remote_ip}
-    cat <<EOF >  ${NETWORK_CONF_PATH}ifcfg-${GRE_TUNNEL}
-DEVICE=${GRE_TUNNEL}
-ONBOOT=yes
-DEVICETYPE=ovs
-TYPE=OVSTunnel
-OVS_BRIDGE=${OVS_SWITCH}
-OVS_TUNNEL_TYPE=gre
-OVS_TUNNEL_OPTIONS="options:remote_ip=${remote_ip}"
-EOF
-done
-
-# add ip route rules such that all pod traffic flows through docker bridge and consequently to the gre tunnels
-cat <<EOF > ${NETWORK_CONF_PATH}route-${DOCKER_BRIDGE}
-${CONTAINER_SUBNET} dev ${DOCKER_BRIDGE} scope link src ${MINION_CONTAINER_ADDR}
-EOF
-
 # generate the post-configure script to be called by salt as cmd.wait
 cat <<EOF > ${POST_NETWORK_SCRIPT}
 #!/bin/bash
@@ -81,27 +33,58 @@ set -e
 
 # Only do this operation once, otherwise, we get docker.service files output on disk, and the command line arguments get applied multiple times
 grep -q kbr0 /etc/sysconfig/docker || {
+  CONTAINER_SUBNETS=(${MASTER_CONTAINER_SUBNET} ${MINION_CONTAINER_SUBNETS[@]})
+  CONTAINER_IPS=(${MASTER_IP} ${MINION_IPS[@]})
+
   # Stop docker before making these updates
   systemctl stop docker
 
-  # NAT interface fails to revive on network restart, so OR-gate to true
-  systemctl restart network.service || true
-
-  # set docker bridge up, and set stp on the ovs bridge
+  # create new docker bridge
+  ip link set dev ${DOCKER_BRIDGE} down || true
+  brctl delbr ${DOCKER_BRIDGE} || true
+  brctl addbr ${DOCKER_BRIDGE}
   ip link set dev ${DOCKER_BRIDGE} up
-  ovs-vsctl set Bridge ${OVS_SWITCH} stp_enable=true
+  ifconfig ${DOCKER_BRIDGE} ${CONTAINER_ADDR} netmask ${CONTAINER_NETMASK} up
+
+  # add ovs bridge
+  ovs-vsctl del-br ${OVS_SWITCH} || true
+  ovs-vsctl add-br ${OVS_SWITCH} -- set Bridge ${OVS_SWITCH} fail-mode=secure
+  ovs-vsctl set bridge ${OVS_SWITCH} protocols=OpenFlow13
+  ovs-vsctl del-port ${OVS_SWITCH} ${TUNNEL_BASE}0 || true
+  ovs-vsctl add-port ${OVS_SWITCH} ${TUNNEL_BASE}0 -- set Interface ${TUNNEL_BASE}0 type=${TUNNEL_BASE} options:remote_ip="flow" options:key="flow" ofport_request=10
+
+  # add tun device
+  ovs-vsctl del-port ${OVS_SWITCH} ${DOCKER_OVS_TUN} || true
+  ovs-vsctl add-port ${OVS_SWITCH} ${DOCKER_OVS_TUN} -- set Interface ${DOCKER_OVS_TUN} type=internal ofport_request=9
+  brctl addif ${DOCKER_BRIDGE} ${DOCKER_OVS_TUN}
+  ip link set ${DOCKER_OVS_TUN} up
+
+
+  # add oflow rules, because we do not want to use stp
+  ovs-ofctl -O OpenFlow13 del-flows ${OVS_SWITCH}
+
+  # now loop through all other minions and create persistent gre tunnels
+  NODE_INDEX=0
+  for remote_ip in "\${CONTAINER_IPS[@]}"
+  do
+      if [ "\${remote_ip}" == "${NODE_IP}" ]; then
+           ovs-ofctl -O OpenFlow13 add-flow ${OVS_SWITCH} "table=0,ip,in_port=10,nw_dst=\${CONTAINER_SUBNETS[\${NODE_INDEX}]},actions=output:9"
+           ovs-ofctl -O OpenFlow13 add-flow ${OVS_SWITCH} "table=0,arp,in_port=10,nw_dst=\${CONTAINER_SUBNETS[\${NODE_INDEX}]},actions=output:9"
+      else
+           ovs-ofctl -O OpenFlow13 add-flow ${OVS_SWITCH} "table=0,in_port=9,ip,nw_dst=\${CONTAINER_SUBNETS[\${NODE_INDEX}]},actions=set_field:\${remote_ip}->tun_dst,output:10"
+           ovs-ofctl -O OpenFlow13 add-flow ${OVS_SWITCH} "table=0,in_port=9,arp,nw_dst=\${CONTAINER_SUBNETS[\${NODE_INDEX}]},actions=set_field:\${remote_ip}->tun_dst,output:10"
+      fi
+      ((NODE_INDEX++)) || true
+  done
+
+  # add ip route rules such that all pod traffic flows through docker bridge and consequently to the gre tunnels
+  ip route add ${CONTAINER_SUBNET} dev ${DOCKER_BRIDGE} scope link src ${CONTAINER_ADDR}
+
 
   # modify the docker service file such that it uses the kube docker bridge and not its own
-  #echo "OPTIONS=-b=kbr0 --iptables=false --selinux-enabled" > /etc/sysconfig/docker
-  echo "OPTIONS='-b=kbr0 --iptables=false --selinux-enabled ${DOCKER_OPTS}'" >/etc/sysconfig/docker
+  echo "OPTIONS='-b=kbr0 --selinux-enabled ${DOCKER_OPTS}'" >/etc/sysconfig/docker
   systemctl daemon-reload
-  systemctl restart docker.service
-
-  # setup iptables masquerade rules so the pods can reach the internet
-  iptables -t nat -A POSTROUTING -s ${CONTAINER_SUBNET} ! -d ${CONTAINER_SUBNET} -j MASQUERADE
-
-  # persist please
-  iptables-save >& /etc/sysconfig/iptables
+  systemctl start docker
 }
 EOF
 

diff --git a/cluster/vagrant/util.sh b/cluster/vagrant/util.sh
@@ -69,6 +69,13 @@ function create-provision-scripts {
     echo "MASTER_IP='${MASTER_IP}'"
     echo "MINION_NAMES=(${MINION_NAMES[@]})"
     echo "MINION_IPS=(${MINION_IPS[@]})"
+    echo "NODE_IP='${MASTER_IP}'"
+    echo "CONTAINER_SUBNET='${CONTAINER_SUBNET}'"
+    echo "CONTAINER_NETMASK='${MASTER_CONTAINER_NETMASK}'"
+    echo "MASTER_CONTAINER_SUBNET='${MASTER_CONTAINER_SUBNET}'"
+    echo "CONTAINER_ADDR='${MASTER_CONTAINER_ADDR}'"
+    echo "MINION_CONTAINER_NETMASKS='${MINION_CONTAINER_NETMASKS[@]}'"
+    echo "MINION_CONTAINER_SUBNETS=(${MINION_CONTAINER_SUBNETS[@]})"
     echo "PORTAL_NET='${PORTAL_NET}'"
     echo "MASTER_USER='${MASTER_USER}'"
     echo "MASTER_PASSWD='${MASTER_PASSWD}'"
@@ -80,6 +87,7 @@ function create-provision-scripts {
     echo "DNS_DOMAIN='${DNS_DOMAIN:-}'"
     echo "RUNTIME_CONFIG='${RUNTIME_CONFIG:-}'"
     grep -v "^#" "${KUBE_ROOT}/cluster/vagrant/provision-master.sh"
+    grep -v "^#" "${KUBE_ROOT}/cluster/vagrant/provision-network.sh"
   ) > "${KUBE_TEMP}/master-start.sh"
 
   for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
@@ -91,8 +99,11 @@ function create-provision-scripts {
       echo "MINION_IPS=(${MINION_IPS[@]})"
       echo "MINION_IP='${MINION_IPS[$i]}'"
       echo "MINION_ID='$i'"
-      echo "MINION_CONTAINER_ADDR='${MINION_CONTAINER_ADDRS[$i]}'"
-      echo "MINION_CONTAINER_NETMASK='${MINION_CONTAINER_NETMASKS[$i]}'"
+      echo "NODE_IP='${MINION_IPS[$i]}'"
+      echo "MASTER_CONTAINER_SUBNET='${MASTER_CONTAINER_SUBNET}'"
+      echo "CONTAINER_ADDR='${MINION_CONTAINER_ADDRS[$i]}'"
+      echo "CONTAINER_NETMASK='${MINION_CONTAINER_NETMASKS[$i]}'"
+      echo "MINION_CONTAINER_SUBNETS=(${MINION_CONTAINER_SUBNETS[@]})"
       echo "CONTAINER_SUBNET='${CONTAINER_SUBNET}'"
       echo "DOCKER_OPTS='${EXTRA_DOCKER_OPTS-}'"
       grep -v "^#" "${KUBE_ROOT}/cluster/vagrant/provision-minion.sh"

diff --git a/hack/e2e-suite/services.sh b/hack/e2e-suite/services.sh
@@ -33,11 +33,6 @@ source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
 
 prepare-e2e
 
-if [[ "$KUBERNETES_PROVIDER" == "vagrant" ]]; then
-  echo "WARNING: Skipping services.sh for ${KUBERNETES_PROVIDER}.  See https://github.com/GoogleCloudPlatform/kubernetes/issues/3655"
-  exit 0
-fi
-
 function error() {
   echo "$@" >&2
   exit 1