Setup liveness and readiness k8 probes (#1418)

* /heathz and /readyz endpoints * Configure k8 to use readinessProbes * Use readyz in docker_compose * Use readyz in docker_compose * Use readyz in k8 test * add parallel build * add parallel build * Add '/' handler to metrics addr * Serve http 200 at '/' GCE Ingress Controllers require HTTP 200 to be served from '/' kubernetes/ingress-gce#241 (comment) * Set k8 probe scheme to https * Monitor dial logs * Set monitor's KT url
google · Jan 6, 2020 · 926eef1 · 926eef1
1 parent 214c0cf
commit 926eef1
Show file tree

Hide file tree

Showing 13 changed files with 119 additions and 17 deletions.
diff --git a/cmd/keytransparency-monitor/main.go b/cmd/keytransparency-monitor/main.go
@@ -50,7 +50,7 @@ var (
 
 	signingKey         = flag.String("sign-key", "genfiles/monitor_sign-key.pem", "Path to private key PEM for SMH signing")
 	signingKeyPassword = flag.String("password", "towel", "Password of the private key PEM file for SMH signing")
-	ktURL              = flag.String("kt-url", "localhost:8080", "URL of key-server.")
+	ktURL              = flag.String("kt-url", "localhost:443", "URL of key-server.")
 	insecure           = flag.Bool("insecure", false, "Skip TLS checks")
 	directoryID        = flag.String("directoryid", "", "KT Directory identifier to monitor")
 
@@ -80,6 +80,9 @@ func main() {
 	var config *pb.Directory
 	if err := b.Retry(cctx, func() (err error) {
 		config, err = ktClient.GetDirectory(ctx, &pb.GetDirectoryRequest{DirectoryId: *directoryID})
+		if err != nil {
+			glog.Errorf("GetDirectory(%v/%v): %v", *ktURL, *directoryID, err)
+		}
 		return
 	}, codes.Unavailable); err != nil {
 		glog.Exitf("Could not read directory info %v:", err)
@@ -138,7 +141,9 @@ func main() {
 	// Insert handlers for other http paths here.
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.Handler())
-	mux.Handle("/", gwmux)
+	mux.Handle("/healthz", serverutil.Healthz())
+	mux.Handle("/readyz", serverutil.Healthz())
+	mux.Handle("/", serverutil.RootHealthHandler(gwmux))
 
 	// Serve HTTP2 server over TLS.
 	glog.Infof("Listening on %v", *addr)

diff --git a/cmd/keytransparency-sequencer/main.go b/cmd/keytransparency-sequencer/main.go
@@ -185,7 +185,7 @@ func main() {
 	glog.Infof("Signer starting")
 
 	// Run servers
-	go serveHTTPMetric(*metricsAddr)
+	go serveHTTPMetrics(*metricsAddr, sqldb)
 	go serveHTTPGateway(ctx, lis, dopts, grpcServer,
 		pb.RegisterKeyTransparencyAdminHandlerFromEndpoint,
 	)

diff --git a/cmd/keytransparency-sequencer/server.go b/cmd/keytransparency-sequencer/server.go
@@ -16,6 +16,7 @@ package main
 
 import (
 	"context"
+	"database/sql"
 	"net"
 	"net/http"
 
@@ -26,12 +27,15 @@ import (
 	"google.golang.org/grpc"
 )
 
-func serveHTTPMetric(addr string) {
-	metricMux := http.NewServeMux()
-	metricMux.Handle("/metrics", promhttp.Handler())
+func serveHTTPMetrics(addr string, sqldb *sql.DB) {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+	mux.Handle("/healthz", serverutil.Healthz())
+	mux.Handle("/readyz", serverutil.Readyz(sqldb))
+	mux.Handle("/", serverutil.Healthz())
 
-	glog.Infof("Hosting metrics on %v", addr)
-	if err := http.ListenAndServe(addr, metricMux); err != nil {
+	glog.Infof("Hosting server status and metrics on %v", addr)
+	if err := http.ListenAndServe(addr, mux); err != nil {
 		glog.Fatalf("ListenAndServeTLS(%v): %v", addr, err)
 	}
 }
@@ -45,7 +49,7 @@ func serveHTTPGateway(ctx context.Context, lis net.Listener, dopts []grpc.DialOp
 	}
 
 	mux := http.NewServeMux()
-	mux.Handle("/", gwmux)
+	mux.Handle("/", serverutil.RootHealthHandler(gwmux))
 
 	server := &http.Server{Handler: serverutil.GrpcHandlerFunc(grpcServer, mux)}
 	if err := server.ServeTLS(lis, *certFile, *keyFile); err != nil {

diff --git a/cmd/keytransparency-server/main.go b/cmd/keytransparency-server/main.go
@@ -152,10 +152,13 @@ func main() {
 
 	// Insert handlers for other http paths here.
 	mux := http.NewServeMux()
-	mux.Handle("/", gwmux)
+	mux.Handle("/", serverutil.RootHealthHandler(gwmux))
 
 	metricMux := http.NewServeMux()
+	metricMux.Handle("/healthz", serverutil.Healthz())
+	metricMux.Handle("/readyz", serverutil.Readyz(sqldb))
 	metricMux.Handle("/metrics", promhttp.Handler())
+	metricMux.Handle("/", serverutil.Healthz())
 	go func() {
 		glog.Infof("Hosting metrics on %v", *metricsAddr)
 		if err := http.ListenAndServe(*metricsAddr, metricMux); err != nil {

diff --git a/cmd/serverutil/healthz.go b/cmd/serverutil/healthz.go
@@ -0,0 +1,40 @@
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package serverutil
+
+import "net/http"
+
+// Healthz is a liveness handler that always responds with HTTP 200.
+func Healthz() http.HandlerFunc { return healthz }
+
+func healthz(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte("ok"))
+}
+
+// RootHeaalthHandler handles liveness checks at "/".
+// All other requests are passed through to `otherHandler`.
+func RootHealthHandler(otherHandler http.Handler) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// Capture requests for the root "/" page first.
+		// This is the default load balancer health check.
+		// https://cloud.google.com/kubernetes-engine/docs/concepts/ingress#health_checks
+		if r.URL.Path == "/" {
+			healthz(w, r)
+			return
+		}
+		otherHandler.ServeHTTP(w, r)
+	}
+}
diff --git a/cmd/serverutil/readyz.go b/cmd/serverutil/readyz.go
@@ -0,0 +1,31 @@
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package serverutil
+
+import (
+	"database/sql"
+	"net/http"
+)
+
+// Readyz is a readiness probe.
+func Readyz(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if db == nil || db.PingContext(r.Context()) != nil {
+			http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}
diff --git a/cmd/serverutil/serverutil.go b/cmd/serverutil/serverutil.go
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Package serverutil provides helper functions to main.go files.
 package serverutil
 
 import (

diff --git a/deploy/kubernetes/base/monitor-deployment.yaml b/deploy/kubernetes/base/monitor-deployment.yaml
@@ -21,7 +21,7 @@ spec:
       - command:
         - /keytransparency-monitor
         - --addr=0.0.0.0:8099
-        - --kt-url=server:8080
+        - --kt-url=server:443
         - --insecure
         - --directoryid=default
         - --tls-key=/run/secrets/server.key
@@ -31,6 +31,16 @@ spec:
         - --alsologtostderr
         - --v=3
         image: gcr.io/key-transparency/keytransparency-monitor:latest
+        livenessProbe:
+         httpGet:
+           path: /healthz
+           port: 8099
+           scheme: HTTPS
+        readinessProbe:
+         httpGet:
+           path: /readyz
+           port: 8099
+           scheme: HTTPS
         name: monitor
         ports:
         - containerPort: 8099

diff --git a/deploy/kubernetes/base/sequencer-deployment.yaml b/deploy/kubernetes/base/sequencer-deployment.yaml
@@ -32,7 +32,11 @@ spec:
         image: gcr.io/key-transparency/keytransparency-sequencer:latest
         livenessProbe:
          httpGet:
-           path: /metrics
+           path: /healthz
+           port: 8081
+        readinessProbe:
+         httpGet:
+           path: /readyz
            port: 8081
         name: sequencer
         ports:

diff --git a/deploy/kubernetes/base/server-deployment.yaml b/deploy/kubernetes/base/server-deployment.yaml
@@ -32,7 +32,11 @@ spec:
         image: gcr.io/key-transparency/keytransparency-server:latest
         livenessProbe:
          httpGet:
-           path: /metrics
+           path: /healthz
+           port: 8081
+        readinessProbe:
+         httpGet:
+           path: /readyz
            port: 8081
         name: server
         ports:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -133,7 +133,7 @@ services:
     labels:
       kompose.service.type: LoadBalancer
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8081/metrics"]
+      test: ["CMD", "curl", "-f", "http://localhost:8081/readyz"]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -165,7 +165,7 @@ services:
       - "8080"
       - "8081"
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8081/metrics"]
+      test: ["CMD", "curl", "-f", "http://localhost:8081/readyz"]
       interval: 30s
       timeout: 10s
       retries: 5

diff --git a/scripts/docker-compose_test.sh b/scripts/docker-compose_test.sh
@@ -18,7 +18,7 @@ timeout ${TIMEOUT} bash -c -- 'until [ "`docker inspect -f {{.State.Status}} $(d
 timeout ${TIMEOUT} bash -c -- 'until [ "`docker inspect -f {{.State.Status}} $(docker-compose ps -q server)`" == "running" ]; do sleep 0.1; done;'
 timeout ${TIMEOUT} bash -c -- 'until [ "`docker inspect -f {{.State.Status}} $(docker-compose ps -q monitor)`" == "running" ]; do sleep 0.1; done;'
 
-wget -T 60 --spider --retry-connrefused --waitretry=1 http://localhost:8081/metrics
+wget -T 60 --spider --retry-connrefused --waitretry=1 http://localhost:8081/readyz
 wget -T 60 -O /dev/null --no-check-certificate  \
 	--retry-connrefused --waitretry=1 \
 	--retry-on-http-error=405,404,503 \

diff --git a/scripts/kubernetes_test.sh b/scripts/kubernetes_test.sh
@@ -42,7 +42,7 @@ timeout ${TIMEOUT} kubectl rollout status deployment/monitor
 timeout ${TIMEOUT} kubectl rollout status deployment/sequencer
 timeout ${TIMEOUT} kubectl rollout status deployment/server
 
-wget -T 60 --spider --retry-connrefused --waitretry=1 http://localhost:8081/metrics
+wget -T 60 --spider --retry-connrefused --waitretry=1 http://localhost:8081/readyz
 wget -T 60 -O /dev/null --no-check-certificate  \
 	--retry-connrefused --waitretry=1 \
 	--retry-on-http-error=405,404,503 \