Initial k8s support in libplayground (play-with-docker#216)

* Initial k8s support in libplayground * Make tasks never give up + custom event for k8s cluster status
stackpoet · Nov 16, 2017 · 3481442 · 3481442
1 parent dcb2bc7
commit 3481442
Show file tree

Hide file tree

Showing 24 changed files with 1,004 additions and 56 deletions.
diff --git a/api.go b/api.go
@@ -10,6 +10,7 @@ import (
 	"github.com/play-with-docker/play-with-docker/event"
 	"github.com/play-with-docker/play-with-docker/handlers"
 	"github.com/play-with-docker/play-with-docker/id"
+	"github.com/play-with-docker/play-with-docker/k8s"
 	"github.com/play-with-docker/play-with-docker/provisioner"
 	"github.com/play-with-docker/play-with-docker/pwd"
 	"github.com/play-with-docker/play-with-docker/pwd/types"
@@ -23,18 +24,21 @@ func main() {
 
 	e := initEvent()
 	s := initStorage()
-	f := initFactory(s)
+	df := initDockerFactory(s)
+	kf := initK8sFactory(s)
 
-	ipf := provisioner.NewInstanceProvisionerFactory(provisioner.NewWindowsASG(f, s), provisioner.NewDinD(id.XIDGenerator{}, f, s))
-	sp := provisioner.NewOverlaySessionProvisioner(f)
+	ipf := provisioner.NewInstanceProvisionerFactory(provisioner.NewWindowsASG(df, s), provisioner.NewDinD(id.XIDGenerator{}, df, s))
+	sp := provisioner.NewOverlaySessionProvisioner(df)
 
-	core := pwd.NewPWD(f, e, s, sp, ipf)
+	core := pwd.NewPWD(df, e, s, sp, ipf)
 
 	tasks := []scheduler.Task{
-		task.NewCheckPorts(e, f),
-		task.NewCheckSwarmPorts(e, f),
-		task.NewCheckSwarmStatus(e, f),
-		task.NewCollectStats(e, f, s),
+		task.NewCheckPorts(e, df),
+		task.NewCheckSwarmPorts(e, df),
+		task.NewCheckSwarmStatus(e, df),
+		task.NewCollectStats(e, df, s),
+		task.NewCheckK8sClusterStatus(e, kf),
+		task.NewCheckK8sClusterExposedPorts(e, kf),
 	}
 	sch, err := scheduler.NewScheduler(tasks, s, e, core)
 	if err != nil {
@@ -69,6 +73,10 @@ func initEvent() event.EventApi {
 	return event.NewLocalBroker()
 }
 
-func initFactory(s storage.StorageApi) docker.FactoryApi {
+func initDockerFactory(s storage.StorageApi) docker.FactoryApi {
 	return docker.NewLocalCachedFactory(s)
 }
+
+func initK8sFactory(s storage.StorageApi) k8s.FactoryApi {
+	return k8s.NewLocalCachedFactory(s)
+}
diff --git a/docker/factory.go b/docker/factory.go
@@ -8,8 +8,8 @@ import (
 	"net/url"
 	"time"
 
-	"docker.io/go-docker/api"
 	client "docker.io/go-docker"
+	"docker.io/go-docker/api"
 	"github.com/docker/go-connections/tlsconfig"
 	"github.com/play-with-docker/play-with-docker/pwd/types"
 	"github.com/play-with-docker/play-with-docker/router"

diff --git a/dockerfiles/k8s/Dockerfile b/dockerfiles/k8s/Dockerfile
@@ -0,0 +1,32 @@
+FROM centos:7
+
+COPY ./systemctl /usr/bin/systemctl
+COPY ./kubernetes.repo /etc/yum.repos.d/
+
+
+
+RUN yum install -y kubelet kubeadm \
+    && mv -f /etc/systemd/system/kubelet.service.d/10-kubeadm.conf /etc/systemd/system/kubelet.service \
+    && yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo \
+    && yum install -y docker-ce \
+    && sed -i -e '4d;5d;8d' /lib/systemd/system/docker.service \
+    && yum clean all
+
+RUN curl -Lf -o jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 \
+    && chmod +x ./jq && mv jq /usr/bin
+
+COPY ./kube* /etc/systemd/system/
+COPY ./wrapkubeadm.sh /usr/local/bin/kubeadm
+COPY ./tokens.csv /etc/pki/tokens.csv
+COPY ./daemon.json /etc/docker/
+
+COPY motd /etc/motd
+
+RUN echo $'cat /etc/motd \n\
+export PS1="[\h \W]$ "' >> /root/.bash_profile
+
+RUN mkdir -p /root/.kube && ln -s /etc/kubernetes/admin.conf /root/.kube/config \
+    && rm -f /etc/machine-id
+
+CMD systemctl start docker && systemctl start kubelet \
+    && while true; do bash -l; done
diff --git a/dockerfiles/k8s/Dockerfile.final b/dockerfiles/k8s/Dockerfile.final
@@ -0,0 +1,8 @@
+FROM franela/kind_builder
+
+COPY motd /etc/motd
+RUN echo $'cat /etc/motd \n\
+export PS1="[\h \W]$ "' >> /root/.bash_profile
+
+CMD systemctl start docker && systemctl start kubelet \
+    && while true; do bash -l; done
diff --git a/dockerfiles/k8s/daemon.json b/dockerfiles/k8s/daemon.json
@@ -0,0 +1,7 @@
+{
+    "experimental": true,
+    "debug": true,
+    "log-level": "info",
+    "insecure-registries": ["127.0.0.1"],
+    "hosts": ["unix:///var/run/docker.sock", "tcp://0.0.0.0:2375"]
+}
diff --git a/dockerfiles/k8s/kubelet.env b/dockerfiles/k8s/kubelet.env
@@ -0,0 +1,8 @@
+KUBELET_KUBECONFIG_ARGS=" --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
+KUBELET_SYSTEM_PODS_ARGS="--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
+KUBELET_NETWORK_ARGS="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
+KUBELET_DNS_ARGS="--cluster-dns=10.96.0.10 --cluster-domain=cluster.local"
+KUBELET_AUTHZ_ARGS="--authorization-mode=Webhook --client-ca-file=/etc/kubernetes/pki/ca.crt"
+KUBELET_CADVISOR_ARGS="--cadvisor-port=0"
+KUBELET_CGROUP_ARGS="--cgroup-driver=cgroupfs"
+KUBELET_EXTRA_ARGS="--fail-swap-on=false"
diff --git a/dockerfiles/k8s/kubelet.service b/dockerfiles/k8s/kubelet.service
@@ -0,0 +1,4 @@
+[Service]
+Restart=always
+EnvironmentFile=/etc/systemd/system/kubelet.env
+ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_NETWORK_ARGS $KUBELET_DNS_ARGS $KUBELET_AUTHZ_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS $KUBELET_EXTRA_ARGS
diff --git a/dockerfiles/k8s/kubernetes.repo b/dockerfiles/k8s/kubernetes.repo
@@ -0,0 +1,8 @@
+[kubernetes]
+name=Kubernetes
+baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
+        https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
diff --git a/dockerfiles/k8s/motd b/dockerfiles/k8s/motd
@@ -0,0 +1,30 @@
+
+                          WARNING!!!!
+
+ This is a sandbox environment. Using personal credentials
+ is HIGHLY! discouraged. Any consequences of doing so, are
+ completely the user's responsibilites.
+
+ You can bootstrap a cluster as follows:
+
+ 1. Initializes cluster master node:
+
+ kubeadm init --apiserver-advertise-address $(hostname -i)
+
+
+ 2. Initialize cluster networking:
+
+ kubectl apply -n kube-system -f \
+    "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"
+
+
+ 3. (Optional) Initialize kube-dashboard:
+
+ curl -L -s https://git.io/kube-dashboard  | sed 's/targetPort: 9090/targetPort: 9090\n  type: LoadBalancer/' | \
+    kubectl apply -f -
+
+
+                          The PWK team.
+
+
+