Added first k8s config

bravecobra · Jul 20, 2020 · 9018904 · 9018904
1 parent f9e7d83
commit 9018904
Show file tree

Hide file tree

Showing 14 changed files with 644 additions and 1 deletion.
diff --git a/.gitignore b/.gitignore
@@ -551,4 +551,5 @@ $RECYCLE.BIN/
 # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option)
 
 src.old/
-certs/
+certs/
+k8s/services/google-data.yml
diff --git a/k8s/infrastructure/coredns.yml b/k8s/infrastructure/coredns.yml
diff --git a/k8s/infrastructure/metallb.yaml b/k8s/infrastructure/metallb.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: my-ip-space
+      protocol: layer2
+      addresses:
+      - 127.0.0.240/28
diff --git a/k8s/infrastructure/namespace.yaml b/k8s/infrastructure/namespace.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    app: identityserver-ui
+  annotations:
+    app: identityserver-ui
+  name: identityserver-ui
diff --git a/k8s/infrastructure/proxy/nginx-ingress.yaml b/k8s/infrastructure/proxy/nginx-ingress.yaml
@@ -0,0 +1,67 @@
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
+metadata:
+  annotations:
+  #   # Enable client certificate authentication
+  #   nginx.ingress.kubernetes.io/auth-tls-verify-client: "on"
+  #   # Create the secret containing the trusted ca certificates
+  #   nginx.ingress.kubernetes.io/auth-tls-secret: "default/ca-secret"
+  #   # Specify the verification depth in the client certificates chain
+  #   nginx.ingress.kubernetes.io/auth-tls-verify-depth: "1"
+  #   # Specify an error page to be redirected to verification errors
+  #   #nginx.ingress.kubernetes.io/auth-tls-error-page: "http://www.mysite.com/error-cert.html"
+  #   # Specify if certificates are passed to upstream server
+  #   nginx.ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream: "true"
+    #nginx.ingress.kubernetes.io/server-alias: "sts.localhost.com api.localhost.com jsclient.localhost.com mvcclient.localhost.com"
+    #kubernetes.io/ingress.serverAlias: "sts.localhost.com api.localhost.com jsclient.localhost.com mvcclient.localhost.com"
+    nginx.ingress.kubernetes.io/proxy-buffer-size: "128k"
+    nginx.ingress.kubernetes.io/proxy-buffering: "on"
+    nginx.ingress.kubernetes.io/proxy-buffers-number: "4256k"
+  name: nginx-proxy
+  namespace: identityserver-ui
+spec:
+  tls:
+    - hosts:
+      - sts.localhost.com
+      - api.localhost.com
+      - jsclient.localhost.com
+      - mvcclient.localhost.com
+      secretName: tls-secret
+  rules:
+    - host: sts.localhost.com
+      http:
+        paths:
+        - path: /
+          backend:
+            serviceName: sts-svc
+            servicePort: 80
+    - host: api.localhost.com
+      http:
+        paths:
+        - path: /
+          backend:
+            serviceName: api-svc
+            servicePort: 80
+    - host: jsclient.localhost.com
+      http:
+        paths:
+        - path: /
+          backend:
+            serviceName: jsclient-svc
+            servicePort: 80
+    - host: mvcclient.localhost.com
+      http:
+        paths:
+        - path: /
+          backend:
+            serviceName: mvcclient-svc
+            servicePort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: sts-alias
+  namespace: identityserver-ui
+spec:
+  type: ExternalName
+  externalName: sts.localhost.com
diff --git a/k8s/infrastructure/proxy/test.yml b/k8s/infrastructure/proxy/test.yml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dnsutils
+  namespace: identityserver-ui
+spec:
+  containers:
+  - name: dnsutils
+    image: gcr.io/kubernetes-e2e-test-images/dnsutils:1.3
+    command:
+      - sleep
+      - "3600"
+    imagePullPolicy: IfNotPresent
+  restartPolicy: Always
diff --git a/k8s/infrastructure/tiller/rbac-config.yaml b/k8s/infrastructure/tiller/rbac-config.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: tiller
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: tiller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: tiller
+    namespace: kube-system
diff --git a/k8s/infrastructure/tiller/tiller.yaml b/k8s/infrastructure/tiller/tiller.yaml
diff --git a/k8s/kubernetes.md b/k8s/kubernetes.md
@@ -0,0 +1,164 @@
+# Setup Kubernetes
+
+## Prerequisites
+
+### MetalLB
+
+If you're not on a hosted cluster (like GCE, AKS, EKS or DigitialOcean), you need to provide load-balancing implementation in order to have externalip assigned automatically. I used MetalLB for this on my 1-node local cluster
+
+```powershell
+kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.9.3/manifests/namespace.yaml
+kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.9.3/manifests/metallb.yaml
+```
+
+### Tiller (2.x)
+
+As we use helm to install the nginx-ingress, we'll need to have it installed on our cluster.
+
+```powershell
+helm repo add stable https://kubernetes-charts.storage.googleapis.com/
+helm repo update
+kubectl apply -f ./k8s/infrastructure/rbac-config.yaml
+kubectl apply -f ./k8s/infrastructure/tiller.yaml
+```
+
+## Deploy
+
+### Create a new namespace
+
+Run the following command to create a new namespace called `identityserver-ui`
+
+```bash
+kubctl apply -f ./k8s/namespace.yml
+```
+
+### SQL Server
+
+Create a k8s secret for SQL server by first base64 encoding the actual password in powershell or bash, depending on you OS:
+
+```powershell
+[Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("Password_123"))
+```
+
+or
+
+```bash
+echo -n 'Password_123' | base64
+```
+
+> Storing a secret as a base64-encoded string is not secure! This is neither hashing or encryption.
+
+Edit the `sqlserverdb-deployment.yaml` if you want another password or override it with:
+
+```bash
+kubectl create secret generic sqlserverdb --namespace=identityserver-ui --from-literal=sa_password=Password_123
+```
+
+Now deploy the SQL Server onto your kubernetes cluster in the new namespace
+
+```powershell
+kubectl apply -f ./k8s/services/sqlserverdb-deployment.yaml
+```
+
+### CA Certificate
+
+First add the CA Root certificate as a ConfigMap, we made during the `docker-compose` setup
+
+```powershell
+kubectl create configmap ca-pemstore --from-file=./compose/nginx/certs/cacerts.pem --namespace=identityserver-ui
+```
+
+That way we can mount the CA Root certificate into our services as a configmap (as adding an extra file)
+
+### STS
+
+Create a config map from the `SeedData.json` file so we can use in the sts service.
+
+```powershell
+kubectl create configmap seeddata --from-file=./compose/sts/SeedData.json --namespace=identityserver-ui
+```
+
+Next we want to store the db connection string as a secret as well:
+
+```powershell
+[Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("Server=sqlserverdb-svc;Database=IdentityUI;User Id=sa;Password=Password_123;MultipleActiveResultSets=true"))
+```
+
+> Storing a secret as a base64-encoded string is not secure. This is neither hashing or encryption.
+
+Grab the output and store that in `sts-deployment.yml` file as a secret called `sts-dbconnection` (already done with the above connectionstring). The secret is referenced in the deployment later.
+
+The google credentials are referenced from yet another secret
+Create a file called `google-data.yml` in `./k8s/services` that looks like this:
+
+```yaml
+kind: Secret
+apiVersion: v1
+metadata:
+  name: sts-google
+  namespace: identityserver-ui
+data:
+  Google_ClientId: <output from your base64 encoded ClientID. Same technique as the DBConnectionstring>
+  Google_ClientSecret: <output from your base64 encoded ClientSecret. Same technique as the DBConnectionstring>
+type: Opaque
+```
+
+```powershell
+kubectl apply -f ./k8s/services/google-data.yaml
+kubectl apply -f ./k8s/services/sts-deployment.yaml
+```
+
+### API
+
+```powershell
+kubectl apply -f ./k8s/services/api-deployment.yaml
+```
+
+### JSClient
+
+```powershell
+kubectl apply -f ./k8s/services/jsclient-deployment.yaml
+```
+
+### MVCCLient
+
+```powershell
+kubectl apply -f ./k8s/services/mvcclient-deployment.yaml
+```
+
+### Ingress controller
+
+There are several ingress controllers to choose from according to [kubernetes.io](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/).
+To keep it fairly simple, I choose the plain nginx one. You could choose any other like [Istio](https://istio.io/) or [Traefik](https://github.com/containous/traefik). They are bit more involved configuration wise and offer a bit more options.
+The ingress controller will replace the nginx-proxy , we used in `docker-compose`.
+
+#### Certificates mount
+
+First add the certificate for all our hosts we want SSL for. We generated this certificate during the `docker-compose` setup.
+
+```powershell
+kubectl create secret tls tls-secret --key ./compose/nginx/certs/localhost.com.key --cert ./compose/nginx/certs/localhost.com.crt --namespace=identityserver-ui
+```
+
+#### DNS
+
+in order to make the pods be able to resolve the `localhost.com` when they want to verify the certificate, we need to make sure that the internal DNS service of the cluster is able to resolve the A- and CNAME records to the service `nginx-ingress-controller` that'll be created by helm in the next step. In order to do so we can change the CoreDNS configuration to rewrite incoming DNS queries and resolve them to that service.
+
+```powershell
+kubectl replace -n kube-system -f coredns_cm.yaml
+```
+
+#### Ingress
+
+```powershell
+helm install --name nginx-ingress stable/nginx-ingress --namespace=identityserver-ui
+kubectl apply -f ./k8s/infrastructure/nginx-ingress.yaml
+```
+
+### Testing it out
+
+Surf to your deployment:
+
+* [STS](https://sts.localhost.com)
+* [JSCLient](https://jsclient.localhost.com)
+* [MVCClient](https://mvcclient.localhost.com)
diff --git a/k8s/services/api-deployment.yaml b/k8s/services/api-deployment.yaml
@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    service: api
+  name: api
+  namespace: identityserver-ui
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      service: api
+  strategy: {}
+  template:
+    metadata:
+      annotations:
+      labels:
+        service: api
+    spec:
+      containers:
+      - env:
+        - name: Auth__AllowCORS__0
+          value: https://jsclient.localhost.com
+        - name: Auth__Authority
+          value: https://sts.localhost.com
+        - name: VIRTUAL_HOST
+          value: api.localhost.com
+        - name: ASPNETCORE_ENVIRONMENT
+          value: Development
+        image: api
+        imagePullPolicy : IfNotPresent
+        name: api
+        resources: {}
+        volumeMounts:
+        - name: ca-pemstore
+          mountPath: /etc/ssl/certs/cacerts.pem
+          subPath: cacerts.pem
+          readOnly: false
+      restartPolicy: Always
+      volumes:
+        - name: ca-pemstore
+          configMap:
+            name: ca-pemstore
+status: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: api-svc
+  namespace: identityserver-ui
+spec:
+  ports:
+  - name: "80"
+    port: 80
+    targetPort: 80
+  selector:
+    service: api
+---