libvirt-coreos cluster provider

libvirt-coreos is a cluster provider for kubernetes that starts local VMs and runs kubernetes on it. Its goal is to provide a multi-machines environment to develop and test kubernetes. The purpose is mostly the same as the vagrant provider but with a big focus on efficiency. The vagrant cluster takes a long time to boot and consumes a huge amount of disk space. libvirt-coreos aims at being cheaper. As a consequence, libvirt-coreos allows to start bigger clusters with more minions than vagrant.
satnam6502 · Feb 19, 2015 · ce7a993 · ce7a993
1 parent c9657ca
commit ce7a993
Show file tree

Hide file tree

Showing 14 changed files with 828 additions and 3 deletions.
diff --git a/cluster/kube-env.sh b/cluster/kube-env.sh
@@ -18,7 +18,7 @@
 # You can override the default provider by exporting the KUBERNETES_PROVIDER
 # variable in your bashrc
 #
-# The valid values: 'gce', 'gke', 'aws', 'azure', 'vagrant', 'vsphere'
+# The valid values: 'gce', 'gke', 'aws', 'azure', 'vagrant', 'vsphere', 'libvirt-coreos'
 
 KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER:-gce}
 

diff --git a/cluster/kubectl.sh b/cluster/kubectl.sh
@@ -120,6 +120,11 @@ elif [[ "$KUBERNETES_PROVIDER" == "vagrant" ]]; then
   config=(
     "--kubeconfig=$HOME/.kubernetes_vagrant_kubeconfig"
   )
+elif [[ "$KUBERNETES_PROVIDER" == "libvirt-coreos" ]]; then
+  detect-master > /dev/null
+  config=(
+    "--server=http://${KUBE_MASTER_IP}:8080"
+  )
 fi
 
 echo "current-context: \"$(${kubectl} "${config[@]:+${config[@]}}" config view -o template --template='{{index . "current-context"}}')\"" >&2

diff --git a/cluster/libvirt-coreos/.gitignore b/cluster/libvirt-coreos/.gitignore
@@ -0,0 +1,2 @@
+/libvirt_storage_pool/
+/coreos_production_qemu_image.img.bz2
diff --git a/cluster/libvirt-coreos/config-default.sh b/cluster/libvirt-coreos/config-default.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Contains configuration values for interacting with the libvirt CoreOS cluster
+
+# Number of minions in the cluster
+NUM_MINIONS=${NUM_MINIONS:-3}
+export NUM_MINIONS
diff --git a/cluster/libvirt-coreos/coreos.xml b/cluster/libvirt-coreos/coreos.xml
@@ -0,0 +1,71 @@
+<domain type='kvm'>
+  <name>${name}</name>
+  <memory unit='MiB'>512</memory>
+  <currentMemory unit='MiB'>512</currentMemory>
+  <vcpu placement='static'>2</vcpu>
+  <os>
+    <type arch='x86_64' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <features>
+    <acpi/>
+    <apic/>
+    <pae/>
+  </features>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>restart</on_crash>
+  <devices>
+    <emulator>$(which qemu-system-$(uname -m))</emulator>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2'/>
+      <source file='${POOL_PATH}/${image}'/>
+      <target dev='vda' bus='virtio'/>
+    </disk>
+    <controller type='usb' index='0'>
+    </controller>
+    <filesystem type='mount' accessmode='squash'>
+      <source dir='${POOL_PATH}/${config}'/>
+      <target dir='config-2'/>
+      <readonly/>
+    </filesystem>
+    <filesystem type='mount' accessmode='squash'>
+      <source dir='${kubernetes_dir}'/>
+      <target dir='kubernetes'/>
+      <readonly/>
+    </filesystem>
+    <interface type='network'>
+       <mac address='52:54:00:00:00:${i}'/>
+       <source network='kubernetes_global'/>
+       <model type='virtio'/>
+    </interface>
+    <interface type='network'>
+       <mac address='52:54:00:00:01:${i}'/>
+       <source network='kubernetes_pods'/>
+       <model type='virtio'/>
+    </interface>
+    <serial type='pty'>
+      <target port='0'/>
+    </serial>
+    <console type='pty'>
+      <target type='serial' port='0'/>
+    </console>
+    <channel type='spicevmc'>
+      <target type='virtio' name='com.redhat.spice.0'/>
+    </channel>
+    <input type='tablet' bus='usb'/>
+    <input type='mouse' bus='ps2'/>
+    <input type='keyboard' bus='ps2'/>
+    <graphics type='spice' autoport='yes'/>
+    <sound model='ich6'>
+    </sound>
+    <video>
+      <model type='qxl' vram='9216' heads='1'/>
+    </video>
+    <redirdev bus='usb' type='spicevmc'>
+    </redirdev>
+    <memballoon model='virtio'>
+    </memballoon>
+  </devices>
+</domain>
diff --git a/cluster/libvirt-coreos/network_kubernetes_global.xml b/cluster/libvirt-coreos/network_kubernetes_global.xml
@@ -0,0 +1,11 @@
+<network>
+  <name>kubernetes_global</name>
+  <forward mode='nat'>
+    <nat>
+      <port start='1024' end='65535'/>
+    </nat>
+  </forward>
+  <bridge name='virbr_kub_gl' stp='off' delay='0'/>
+  <ip address='192.168.10.254' netmask='255.255.255.0'>
+  </ip>
+</network>
diff --git a/cluster/libvirt-coreos/network_kubernetes_pods.xml b/cluster/libvirt-coreos/network_kubernetes_pods.xml
@@ -0,0 +1,6 @@
+<network>
+  <name>kubernetes_pods</name>
+  <bridge name='virbr_kub_pods' stp='off' delay='0'/>
+  <ip address='10.10.0.1' netmask='255.255.0.0'>
+  </ip>
+</network>
diff --git a/cluster/libvirt-coreos/user_data.yml b/cluster/libvirt-coreos/user_data.yml
@@ -0,0 +1,111 @@
+#cloud-config
+
+hostname: ${name}
+
+ssh_authorized_keys:
+${ssh_keys}
+
+write_files:
+  - path: /etc/systemd/journald.conf
+    permissions: 0644
+    content: |
+      [Journal]
+      SystemMaxUse=50M
+      RuntimeMaxUse=50M
+
+coreos:
+  etcd:
+    name: ${name}
+    addr: 192.168.10.$(($i+1)):4001
+    bind-addr: 0.0.0.0
+    peer-addr: 192.168.10.$(($i+1)):7001
+    # peers: {etcd_peers}
+    discovery: ${discovery}
+  units:
+    - name: static.network
+      command: start
+      content: |
+        [Match]
+        # Name=eth0
+        MACAddress=52:54:00:00:00:${i}
+
+        [Network]
+        Address=192.168.10.$(($i+1))/24
+        DNS=192.168.10.254
+        Gateway=192.168.10.254
+    - name: cbr0.netdev
+      command: start
+      content: |
+        [NetDev]
+        Kind=bridge
+        Name=cbr0
+    - name: cbr0.network
+      command: start
+      content: |
+        [Match]
+        Name=cbr0
+
+        [Network]
+        Address=10.10.$(($i+1)).1/24
+
+        [Route]
+        Destination=10.10.0.0/16
+    - name: cbr0-interface.network
+      command: start
+      content: |
+        [Match]
+        # Name=eth1
+        MACAddress=52:54:00:00:01:${i}
+
+        [Network]
+        Bridge=cbr0
+    - name: nat.service
+      command: start
+      content: |
+        [Unit]
+        Description=NAT non container traffic
+
+        [Service]
+        ExecStart=/usr/sbin/iptables -w -t nat -A POSTROUTING -o eth0 -j MASQUERADE ! -d 10.10.0.0/16
+        RemainAfterExit=yes
+        Type=oneshot
+    - name: etcd.service
+      command: start
+    - name: docker.service
+      command: start
+      drop-ins:
+        - name: 50-opts.conf
+          content: |
+            [Service]
+            Environment=DOCKER_OPTS='--bridge=cbr0 --iptables=false'
+    - name: docker-tcp.socket
+      command: start
+      enable: yes
+      content: |
+        [Unit]
+        Description=Docker Socket for the API
+
+        [Socket]
+        ListenStream=2375
+        BindIPv6Only=both
+        Service=docker.service
+
+        [Install]
+        WantedBy=sockets.target
+    - name: opt-kubernetes.mount
+      command: start
+      content: |
+        [Unit]
+        ConditionVirtualization=|vm
+
+        [Mount]
+        What=kubernetes
+        Where=/opt/kubernetes
+        Options=ro,trans=virtio,version=9p2000.L
+        Type=9p
+  update:
+    group: ${COREOS_CHANNEL:-alpha}
+    reboot-strategy: off
+
+$( [[ ${type} =~ "master" ]] && render-template "$ROOT/user_data_master.yml" )
+$( [[ ${type} =~ "minion" ]] && render-template "$ROOT/user_data_minion.yml" )
diff --git a/cluster/libvirt-coreos/user_data_master.yml b/cluster/libvirt-coreos/user_data_master.yml
@@ -0,0 +1,63 @@
+#cloud-config
+
+coreos:
+  units:
+    - name: kube-apiserver.service
+      command: start
+      content: |
+        [Unit]
+        After=opt-kubernetes.mount etcd.service
+        ConditionFileIsExecutable=/opt/kubernetes/bin/kube-apiserver
+        Description=Kubernetes API Server
+        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+        Requires=opt-kubernetes.mount etcd.service
+
+        [Service]
+        ExecStart=/opt/kubernetes/bin/kube-apiserver \
+        --address=0.0.0.0 \
+        --port=8080 \
+        --etcd_servers=http://127.0.0.1:4001 \
+        --kubelet_port=10250 \
+        --portal_net=10.10.254.0/24
+        Restart=always
+        RestartSec=2
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: kube-controller-manager.service
+      command: start
+      content: |
+        [Unit]
+        After=opt-kubernetes.mount kube-apiserver.service
+        ConditionFileIsExecutable=/opt/kubernetes/bin/kube-controller-manager
+        Description=Kubernetes Controller Manager
+        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+        Requires=opt-kubernetes.mount kube-apiserver.service
+
+        [Service]
+        ExecStart=/opt/kubernetes/bin/kube-controller-manager \
+        --master=127.0.0.1:8080 \
+        --machines=${machines}
+        Restart=always
+        RestartSec=2
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: kube-scheduler.service
+      command: start
+      content: |
+        [Unit]
+        After=opt-kubernetes.mount kube-apiserver.service
+        ConditionFileIsExecutable=/opt/kubernetes/bin/kube-scheduler
+        Description=Kubernetes Scheduler
+        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+        Requires=opt-kubernetes.mount kube-apiserver.service
+
+        [Service]
+        ExecStart=/opt/kubernetes/bin/kube-scheduler \
+        --master=127.0.0.1:8080
+        Restart=always
+        RestartSec=2
+
+        [Install]
+        WantedBy=multi-user.target
diff --git a/cluster/libvirt-coreos/user_data_minion.yml b/cluster/libvirt-coreos/user_data_minion.yml
@@ -0,0 +1,43 @@
+#cloud-config
+
+coreos:
+  units:
+    - name: kubelet.service
+      command: start
+      content: |
+        [Unit]
+        After=opt-kubernetes.mount etcd.service docker.socket
+        ConditionFileIsExecutable=/opt/kubernetes/bin/kubelet
+        Description=Kubernetes Kubelet
+        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+        Requires=opt-kubernetes.mount etcd.service docker.socket
+
+        [Service]
+        ExecStart=/opt/kubernetes/bin/kubelet \
+        --address=0.0.0.0 \
+        --hostname_override=192.168.10.$(($i+1)) \
+        --etcd_servers=http://127.0.0.1:4001
+        Restart=always
+        RestartSec=2
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: kube-proxy.service
+      command: start
+      content: |
+        [Unit]
+        After=opt-kubernetes.mount etcd.service
+        ConditionFileIsExecutable=/opt/kubernetes/bin/kube-proxy
+        Description=Kubernetes Proxy
+        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+        Requires=opt-kubernetes.mount etcd.service
+
+        [Service]
+        ExecStart=/opt/kubernetes/bin/kube-proxy \
+        --etcd_servers=http://127.0.0.1:4001 \
+        --master=http://192.168.10.1:7080
+        Restart=always
+        RestartSec=2
+
+        [Install]
+        WantedBy=multi-user.target
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		/libvirt_storage_pool/
		/coreos_production_qemu_image.img.bz2