watch-termination: create NonGracefulTermination event on next launch

jmcmeek · Jun 24, 2020 · e63f062 · e63f062
1 parent bf741a8
commit e63f062
Showing 1 changed file with 104 additions and 0 deletions.
diff --git a/cmd/watch-termination/main.go b/cmd/watch-termination/main.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"io"
@@ -14,6 +15,12 @@ import (
 	"time"
 
 	"gopkg.in/natefinch/lumberjack.v2"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog"
 )
 
@@ -24,6 +31,7 @@ func main() {
 func run() int {
 	terminationLog := flag.String("termination-log-file", "", "Write logs after SIGTERM to this file (in addition to stderr)")
 	terminationLock := flag.String("termination-touch-file", "", "Touch this file on SIGTERM and delete on termination")
+	kubeconfigPath := flag.String("kubeconfig", "", "Optional kubeconfig used to create events")
 
 	klog.InitFlags(nil)
 	flag.Set("v", "9")
@@ -57,9 +65,44 @@ func run() int {
 		klog.SetOutputBySeverity("INFO", stderr)
 	}
 
+	var client kubernetes.Interface
+	if len(*kubeconfigPath) > 0 {
+		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: *kubeconfigPath}, nil)
+		if cfg, err := loader.ClientConfig(); err != nil {
+			klog.Errorf("failed to load kubeconfig %q: %v", *kubeconfigPath, err)
+			return 1
+		} else {
+			client = kubernetes.NewForConfigOrDie(cfg)
+		}
+	}
+
 	// touch file early. If the file is not removed on termination, we are not
 	// terminating cleanly via SIGTERM.
 	if len(*terminationLock) > 0 {
+		ref, err := eventReference()
+		if err != nil {
+			klog.Errorf("failed to get event target: %v", err)
+			return 1
+		}
+
+		if st, err := os.Stat(*terminationLock); err == nil {
+			klog.Warningf("Previous pod did not terminate gracefully: %s", st.ModTime().String())
+			if client != nil {
+				go wait.PollUntil(5*time.Second, func() (bool, error) {
+					if err := eventf(client.CoreV1().Events(ref.Namespace), *ref, corev1.EventTypeWarning, "NonGracefulTermination", "Previous pod did not terminate gracefully: %s", st.ModTime().String()); err != nil {
+						return false, nil
+					}
+
+					select {
+					case <-termCh:
+					default:
+						klog.Infof("Deleting old termination lock file %q", *terminationLock)
+						os.Remove(*terminationLock)
+					}
+					return true, nil
+				}, termCh)
+			}
+		}
 		klog.Infof("Touching termination lock file %q", *terminationLock)
 		if err := touch(*terminationLock); err != nil {
 			klog.Infof("error touching %s: %v", *terminationLock, err)
@@ -171,3 +214,64 @@ func touch(fn string) error {
 	currentTime := time.Now().Local()
 	return os.Chtimes(fn, currentTime, currentTime)
 }
+
+func eventf(client corev1client.EventInterface, ref corev1.ObjectReference, eventType, reason, messageFmt string, args ...interface{}) error {
+	t := metav1.Time{Time: time.Now()}
+	host, _ := os.Hostname() // expicitly ignore error. Empty host is fine
+
+	e := &corev1.Event{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%v.%x", ref.Name, t.UnixNano()),
+			Namespace: ref.Namespace,
+		},
+		InvolvedObject: ref,
+		Reason:         reason,
+		Message:        fmt.Sprintf(messageFmt, args...),
+		FirstTimestamp: t,
+		LastTimestamp:  t,
+		Count:          1,
+		Type:           eventType,
+		Source:         corev1.EventSource{Component: "apiserver", Host: host},
+	}
+
+	_, err := client.Create(context.TODO(), e, metav1.CreateOptions{})
+
+	if err == nil {
+		klog.V(2).Infof("Event(%#v): type: '%v' reason: '%v' %v", e.InvolvedObject, e.Type, e.Reason, e.Message)
+	}
+
+	return err
+}
+
+func eventReference() (*corev1.ObjectReference, error) {
+	ns := os.Getenv("POD_NAMESPACE")
+	pod := os.Getenv("POD_NAME")
+	if len(ns) == 0 && len(pod) > 0 {
+		serviceAccountNamespaceFile := "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
+		if _, err := os.Stat(serviceAccountNamespaceFile); err == nil {
+			bs, err := ioutil.ReadFile(serviceAccountNamespaceFile)
+			if err != nil {
+				return nil, err
+			}
+			ns = string(bs)
+		}
+	}
+	if len(ns) == 0 {
+		pod = ""
+		ns = "kube-system"
+	}
+	if len(pod) == 0 {
+		return &corev1.ObjectReference{
+			Kind:       "Namespace",
+			Name:       ns,
+			APIVersion: "v1",
+		}, nil
+	}
+
+	return &corev1.ObjectReference{
+		Kind:       "Pod",
+		Namespace:  ns,
+		Name:       pod,
+		APIVersion: "v1",
+	}, nil
+}