run kube-proxy in a static pod

nikhiljindal · Nov 19, 2015 · 1d9d11c · 1d9d11c
1 parent 885134a
commit 1d9d11c
Show file tree

Hide file tree

Showing 15 changed files with 272 additions and 257 deletions.
diff --git a/build/common.sh b/build/common.sh
@@ -99,6 +99,7 @@ readonly KUBE_DOCKER_WRAPPED_BINARIES=(
   kube-apiserver,busybox
   kube-controller-manager,busybox
   kube-scheduler,busybox
+  kube-proxy,gcr.io/google_containers/debian-iptables:v1
 )
 
 # The set of addons images that should be prepopulated

diff --git a/cluster/saltbase/install.sh b/cluster/saltbase/install.sh
@@ -25,9 +25,10 @@ SALT_ROOT=$(dirname "${BASH_SOURCE}")
 readonly SALT_ROOT
 
 readonly KUBE_DOCKER_WRAPPED_BINARIES=(
-      kube-apiserver
-      kube-controller-manager
-      kube-scheduler
+  kube-apiserver
+  kube-controller-manager
+  kube-scheduler
+  kube-proxy
 )
 
 readonly SERVER_BIN_TAR=${1-}

diff --git a/cluster/saltbase/pillar/docker-images.sls b/cluster/saltbase/pillar/docker-images.sls
@@ -2,3 +2,4 @@
 kube-apiserver_docker_tag: #kube-apiserver_docker_tag_value#
 kube-controller-manager_docker_tag: #kube-controller-manager_docker_tag_value#
 kube-scheduler_docker_tag: #kube-scheduler_docker_tag_value#
+kube-proxy_docker_tag: #kube-proxy_docker_tag_value#
diff --git a/cluster/saltbase/salt/kube-node-unpacker/init.sls b/cluster/saltbase/salt/kube-node-unpacker/init.sls
@@ -0,0 +1,43 @@
+/etc/kubernetes/kube-node-unpacker.sh:
+  file.managed:
+    - source: salt://kube-node-unpacker/kube-node-unpacker.sh
+    - user: root
+    - group: root
+    - mode: 755
+
+node-docker-image-tags:
+  file.touch:
+    - name: /srv/pillar/docker-images.sls
+
+{% if pillar.get('is_systemd') %}
+
+{{ pillar.get('systemd_system_path') }}/kube-node-unpacker.service:
+  file.managed:
+    - source: salt://kube-node-unpacker/kube-node-unpacker.service
+    - user: root
+    - group: root
+  cmd.wait:
+    - name: /opt/kubernetes/helpers/services bounce kube-node-unpacker
+    - watch:
+      - file: node-docker-image-tags
+      - file: /etc/kubernetes/kube-node-unpacker.sh
+      - file: {{ pillar.get('systemd_system_path') }}/kube-node-unpacker.service
+
+{% else %}
+
+/etc/init.d/kube-node-unpacker:
+  file.managed:
+    - source: salt://kube-node-unpacker/initd
+    - user: root
+    - group: root
+    - mode: 755
+
+kube-node-unpacker:
+  service.running:
+    - enable: True
+    - restart: True
+    - watch:
+      - file: node-docker-image-tags
+      - file: /etc/kubernetes/kube-node-unpacker.sh
+
+{% endif %}
diff --git a/cluster/saltbase/salt/kube-node-unpacker/initd b/cluster/saltbase/salt/kube-node-unpacker/initd
@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:   kube-node-unpacker
+# Required-Start:    $local_fs $network $syslog docker
+# Required-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Kubernetes Node Unpacker
+# Description:
+#   Unpacks docker images on Kubernetes nodes
+### END INIT INFO
+
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="Kubernetes Node Unpacker"
+NAME=kube-node-unpacker
+DAEMON_LOG_FILE=/var/log/$NAME.log
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+KUBE_MASTER_ADDONS_SH=/etc/kubernetes/kube-node-unpacker.sh
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
+# and status_of_proc is working.
+. /lib/lsb/init-functions
+
+
+
+
+#
+# Function that starts the daemon/service
+#
+do_start()
+{
+    ${KUBE_MASTER_ADDONS_SH} </dev/null >>${DAEMON_LOG_FILE} 2>&1 &
+    echo $! > ${PIDFILE}
+    disown
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+    kill $(cat ${PIDFILE})
+    rm ${PIDFILE}
+    return
+}
+
+case "$1" in
+  start)
+        log_daemon_msg "Starting $DESC" "$NAME"
+        do_start
+        case "$?" in
+                0|1) log_end_msg 0 || exit 0 ;;
+                2) log_end_msg 1 || exit 1 ;;
+        esac
+        ;;
+  stop)
+        log_daemon_msg "Stopping $DESC" "$NAME"
+        do_stop
+        case "$?" in
+                0|1) log_end_msg 0 ;;
+                2) exit 1 ;;
+        esac
+        ;;
+  status)
+        status_of_proc -p $PIDFILE $KUBE_MASTER_ADDONS_SH $NAME
+        ;;
+
+  restart|force-reload)
+        log_daemon_msg "Restarting $DESC" "$NAME"
+        do_stop
+        case "$?" in
+          0|1)
+                do_start
+                case "$?" in
+                        0) log_end_msg 0 ;;
+                        1) log_end_msg 1 ;; # Old process is still running
+                        *) log_end_msg 1 ;; # Failed to start
+                esac
+                ;;
+          *)
+                # Failed to stop
+                log_end_msg 1
+                ;;
+        esac
+        ;;
+  *)
+        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+        exit 3
+        ;;
+esac
diff --git a/cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.service b/cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=Kubernetes Node Unpacker
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+
+[Service]
+ExecStart=/etc/kubernetes/kube-node-unpacker.sh
+
+[Install]
+WantedBy=multi-user.target
diff --git a/cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.sh b/cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# loadedImageFlags is a bit-flag to track which docker images loaded successfully.
+let loadedImageFlags=0
+
+while true; do
+  restart_docker=false
+
+  if which docker 1>/dev/null 2>&1; then
+
+    timeout 30 docker load -i /srv/salt/kube-bins/kube-proxy.tar 1>/dev/null 2>&1
+    rc=$?
+    if [[ "${rc}" == 0 ]]; then
+      let loadedImageFlags="${loadedImageFlags}|1"
+    elif [[ "${rc}" == 124 ]]; then
+      restart_docker=true
+    fi
+  fi
+
+  # required docker images got installed. exit while loop.
+  if [[ "${loadedImageFlags}" == 1 ]]; then break; fi
+
+  # Sometimes docker load hang, restart docker daemon resolve the issue
+  if [[ "${restart_docker}" ]]; then service docker restart; fi
+
+  # sleep for 15 seconds before attempting to load docker images again
+  sleep 15
+
+done
+
+# Now exit. After kube-push, salt will notice that the service is down and it
+# will start it and new docker images will be loaded.
diff --git a/cluster/saltbase/salt/kube-proxy/default b/cluster/saltbase/salt/kube-proxy/default
diff --git a/cluster/saltbase/salt/kube-proxy/init.sls b/cluster/saltbase/salt/kube-proxy/init.sls
@@ -1,77 +1,33 @@
-{% if pillar.get('is_systemd') %}
-{% set environment_file = '/etc/sysconfig/kube-proxy' %}
-{% else %}
-{% set environment_file = '/etc/default/kube-proxy' %}
-{% endif %}
-
-/usr/local/bin/kube-proxy:
+/var/lib/kube-proxy/kubeconfig:
   file.managed:
-    - source: salt://kube-bins/kube-proxy
+    - source: salt://kube-proxy/kubeconfig
     - user: root
     - group: root
-    - mode: 755
+    - mode: 400
+    - makedirs: true
 
-{{ environment_file }}:
+# kube-proxy in a static pod
+/etc/kubernetes/manifests/kube-proxy.manifest:
   file.managed:
-    - source: salt://kube-proxy/default
+    - source: salt://kube-proxy/kube-proxy.manifest
     - template: jinja
     - user: root
     - group: root
     - mode: 644
-
-kube-proxy:
-  group.present:
-    - system: True
-  user.present:
-    - system: True
-    - gid_from_name: True
-    - shell: /sbin/nologin
-    - home: /var/kube-proxy
+    - makedirs: true
+    - dir_mode: 755
     - require:
-      - group: kube-proxy
-
-{% if pillar.get('is_systemd') %}
+      - service: docker
+      - service: kubelet
 
-{{ pillar.get('systemd_system_path') }}/kube-proxy.service:
+/var/log/kube-proxy.log:
   file.managed:
-    - source: salt://kube-proxy/kube-proxy.service
     - user: root
     - group: root
-  cmd.wait:
-    - name: /opt/kubernetes/helpers/services bounce kube-proxy
-    - watch:
-      - file: {{ environment_file }}
-      - file: {{ pillar.get('systemd_system_path') }}/kube-proxy.service
-      - file: /var/lib/kube-proxy/kubeconfig
-
-{% else %}
-
-/etc/init.d/kube-proxy:
-  file.managed:
-    - source: salt://kube-proxy/initd
-    - user: root
-    - group: root
-    - mode: 755
-
-{% endif %}
+    - mode: 644
 
-kube-proxy-service:
-  service.running:
+#stop legacy kube-proxy service 
+stop_kube-proxy:
+  service.dead:
     - name: kube-proxy
-    - enable: True
-    - watch:
-      - file: {{ environment_file }}
-{% if pillar.get('is_systemd') %}
-      - file: {{ pillar.get('systemd_system_path') }}/kube-proxy.service
-{% else %}
-      - file: /etc/init.d/kube-proxy
-{% endif %}
-      - file: /var/lib/kube-proxy/kubeconfig
-
-/var/lib/kube-proxy/kubeconfig:
-  file.managed:
-    - source: salt://kube-proxy/kubeconfig
-    - user: root
-    - group: root
-    - mode: 400
-    - makedirs: true
+    - enable: None