Show pod status more clearly (#1967)

During operations with `linkerd stat` sometimes it's not clear the actual pod status. This commit introduces a method, to the controller's public API, getting the real pod status, based on [`kubectl` logic](https://github.com/kubernetes/kubernetes/blob/33a3e325f754d179b25558dee116fca1c67d353a/pkg/printers/internalversion/printers.go#L558-L640) to expose the `STATUS` column for pods . Also, it changes the stat command on the` cli` package adding a column when the resource type is a Pod. Fixes #1967 Signed-off-by: Jonatha Juares Beber <jonathanbeber@gmail.com>
linkerd · Jun 25, 2019 · de17e65 · de17e65
1 parent 8988a57
commit de17e65
Show file tree

Hide file tree

Showing 9 changed files with 1,133 additions and 282 deletions.
diff --git a/cli/cmd/stat.go b/cli/cmd/stat.go
@@ -217,6 +217,7 @@ type rowStats struct {
 
 type row struct {
 	meshed string
+	status string
 	*rowStats
 }
 
@@ -268,6 +269,7 @@ func writeStatsToBuffer(rows []*pb.StatTable_PodGroup_Row, w *tabwriter.Writer,
 		}
 		statTables[resourceKey][key] = &row{
 			meshed: meshedCount,
+			status: r.Status,
 		}
 
 		if r.Stats != nil {
@@ -333,8 +335,14 @@ func printSingleStatTable(stats map[string]*row, resourceTypeLabel, resourceType
 		headers = append(headers,
 			namespaceHeader+strings.Repeat(" ", maxNamespaceLength-len(namespaceHeader)))
 	}
+
+	headers = append(headers, nameHeader+strings.Repeat(" ", maxNameLength-len(nameHeader)))
+
+	if resourceType == k8s.Pod {
+		headers = append(headers, "STATUS")
+	}
+
 	headers = append(headers, []string{
-		nameHeader + strings.Repeat(" ", maxNameLength-len(nameHeader)),
 		"MESHED",
 		"SUCCESS",
 		"RPS",
@@ -359,17 +367,23 @@ func printSingleStatTable(stats map[string]*row, resourceTypeLabel, resourceType
 	for _, key := range sortedKeys {
 		namespace, name := namespaceName(resourceTypeLabel, key)
 		values := make([]interface{}, 0)
-		templateString := "%s\t%s\t%.2f%%\t%.1frps\t%dms\t%dms\t%dms\t%d\t\n"
-		templateStringEmpty := "%s\t%s\t-\t-\t-\t-\t-\t-\t\n"
-
-		if showTCPBytes(options, resourceType) {
-			templateString = "%s\t%s\t%.2f%%\t%.1frps\t%dms\t%dms\t%dms\t%d\t%.1fB/s\t%.1fB/s\t\n"
-			templateStringEmpty = "%s\t%s\t-\t-\t-\t-\t-\t-\t-\t-\t\n"
+		templateString := "%s\t%s\t%.2f%%\t%.1frps\t%dms\t%dms\t%dms\t"
+		templateStringEmpty := "%s\t%s\t-\t-\t-\t-\t-\t-\t"
+		if resourceType == k8s.Pod {
+			templateString = "%s\t" + templateString
+			templateStringEmpty = "%s\t" + templateStringEmpty
 		}
 
 		if !showTCPConns(resourceType) {
 			// always show TCP Connections as - for Authorities
-			templateString = "%s\t%s\t%.2f%%\t%.1frps\t%dms\t%dms\t%dms\t-\t\n"
+			templateString = templateString + "-\t"
+		} else {
+			templateString = templateString + "%d\t"
+		}
+
+		if showTCPBytes(options, resourceType) {
+			templateString = templateString + "%.1fB/s\t%.1fB/s\t"
+			templateStringEmpty = templateStringEmpty + "-\t-\t"
 		}
 
 		if options.allNamespaces {
@@ -379,12 +393,20 @@ func printSingleStatTable(stats map[string]*row, resourceTypeLabel, resourceType
 			templateStringEmpty = "%s\t" + templateStringEmpty
 		}
 
+		templateString = templateString + "\n"
+		templateStringEmpty = templateStringEmpty + "\n"
+
 		padding := 0
 		if maxNameLength > len(name) {
 			padding = maxNameLength - len(name)
 		}
+
+		values = append(values, name+strings.Repeat(" ", padding))
+		if resourceType == k8s.Pod {
+			values = append(values, stats[key].status)
+		}
+
 		values = append(values, []interface{}{
-			name + strings.Repeat(" ", padding),
 			stats[key].meshed,
 		}...)
 

diff --git a/cli/cmd/stat_test.go b/cli/cmd/stat_test.go
@@ -26,7 +26,21 @@ func TestStat(t *testing.T) {
 			options: options,
 			resNs:   []string{"emojivoto1"},
 			file:    "stat_one_output.golden",
-		}, t)
+		}, k8s.Namespace, t)
+	})
+
+	t.Run("Returns pod stats", func(t *testing.T) {
+		testStatCall(paramsExp{
+			counts: &public.PodCounts{
+				Status:      "Running",
+				MeshedPods:  1,
+				RunningPods: 1,
+				FailedPods:  0,
+			},
+			options: options,
+			resNs:   []string{"emojivoto1"},
+			file:    "stat_one_pod_output.golden",
+		}, k8s.Pod, t)
 	})
 
 	options.outputFormat = jsonOutput
@@ -40,7 +54,7 @@ func TestStat(t *testing.T) {
 			options: options,
 			resNs:   []string{"emojivoto1"},
 			file:    "stat_one_output_json.golden",
-		}, t)
+		}, k8s.Namespace, t)
 	})
 
 	options = newStatOptions()
@@ -55,7 +69,7 @@ func TestStat(t *testing.T) {
 			options: options,
 			resNs:   []string{"emojivoto1", "emojivoto2"},
 			file:    "stat_all_output.golden",
-		}, t)
+		}, k8s.Namespace, t)
 	})
 
 	options.outputFormat = jsonOutput
@@ -69,7 +83,7 @@ func TestStat(t *testing.T) {
 			options: options,
 			resNs:   []string{"emojivoto1", "emojivoto2"},
 			file:    "stat_all_output_json.golden",
-		}, t)
+		}, k8s.Namespace, t)
 	})
 
 	options = newStatOptions()
@@ -84,7 +98,7 @@ func TestStat(t *testing.T) {
 			options: options,
 			resNs:   []string{"emojivoto1"},
 			file:    "stat_one_tcp_output.golden",
-		}, t)
+		}, k8s.Namespace, t)
 	})
 
 	t.Run("Returns an error for named resource queries with the --all-namespaces flag", func(t *testing.T) {
@@ -150,9 +164,9 @@ func TestStat(t *testing.T) {
 	})
 }
 
-func testStatCall(exp paramsExp, t *testing.T) {
+func testStatCall(exp paramsExp, resourceType string, t *testing.T) {
 	mockClient := &public.MockAPIClient{}
-	response := public.GenStatSummaryResponse("emoji", k8s.Namespace, exp.resNs, exp.counts, true, true)
+	response := public.GenStatSummaryResponse("emoji", resourceType, exp.resNs, exp.counts, true, true)
 
 	mockClient.StatSummaryResponseToReturn = &response
 

diff --git a/cli/cmd/testdata/stat_one_pod_output.golden b/cli/cmd/testdata/stat_one_pod_output.golden
@@ -0,0 +1,2 @@
+NAME     STATUS   MESHED   SUCCESS      RPS   LATENCY_P50   LATENCY_P95   LATENCY_P99   TCP_CONN
+emoji   Running      1/1   100.00%   2.0rps         123ms         123ms         123ms        123
diff --git a/controller/api/public/stat_summary.go b/controller/api/public/stat_summary.go
@@ -2,6 +2,7 @@ package public
 
 import (
 	"context"
+	"fmt"
 	"reflect"
 
 	proto "github.com/golang/protobuf/proto"
@@ -43,6 +44,7 @@ const (
 )
 
 type podStats struct {
+	status string
 	inMesh uint64
 	total  uint64
 	failed uint64
@@ -216,6 +218,7 @@ func (s *grpcServer) k8sResourceQuery(ctx context.Context, req *pb.StatSummaryRe
 		}
 
 		podStat := objInfo.podStats
+		row.Status = podStat.status
 		row.MeshedPodCount = podStat.inMesh
 		row.RunningPodCount = podStat.total
 		row.FailedPodCount = podStat.failed
@@ -428,6 +431,10 @@ func (s *grpcServer) getPodStats(obj runtime.Object) (*podStats, error) {
 	podErrors := make(map[string]*pb.PodErrors)
 	meshCount := &podStats{}
 
+	if pod, ok := obj.(*corev1.Pod); ok {
+		meshCount.status = getPodStatus(pod)
+	}
+
 	for _, pod := range pods {
 		if pod.Status.Phase == corev1.PodFailed {
 			meshCount.failed++
@@ -470,7 +477,7 @@ func checkContainerErrors(containerStatuses []corev1.ContainerStatus) []*pb.PodE
 				errors = append(errors, toPodError(st.Name, st.Image, st.State.Waiting.Reason, st.State.Waiting.Message))
 			}
 
-			if st.State.Terminated != nil {
+			if st.State.Terminated != nil && (st.State.Terminated.ExitCode != 0 || st.State.Terminated.Signal != 0) {
 				errors = append(errors, toPodError(st.Name, st.Image, st.State.Terminated.Reason, st.State.Terminated.Message))
 			}
 
@@ -485,3 +492,65 @@ func checkContainerErrors(containerStatuses []corev1.ContainerStatus) []*pb.PodE
 	}
 	return errors
 }
+
+func getPodStatus(pod *corev1.Pod) string {
+	reason := string(pod.Status.Phase)
+	if pod.Status.Reason != "" {
+		reason = pod.Status.Reason
+	}
+
+	initializing := false
+	for i := range pod.Status.InitContainerStatuses {
+		container := pod.Status.InitContainerStatuses[i]
+		switch {
+		case container.State.Terminated != nil && container.State.Terminated.ExitCode == 0 && container.State.Terminated.Signal == 0:
+			continue
+		case container.State.Terminated != nil:
+			// initialization is failed
+			if len(container.State.Terminated.Reason) == 0 {
+				if container.State.Terminated.Signal != 0 {
+					reason = fmt.Sprintf("Init:Signal:%d", container.State.Terminated.Signal)
+				} else {
+					reason = fmt.Sprintf("Init:ExitCode:%d", container.State.Terminated.ExitCode)
+				}
+			} else {
+				reason = "Init:" + container.State.Terminated.Reason
+			}
+			initializing = true
+		case container.State.Waiting != nil && len(container.State.Waiting.Reason) > 0 && container.State.Waiting.Reason != "PodInitializing":
+			reason = "Init:" + container.State.Waiting.Reason
+			initializing = true
+		default:
+			reason = fmt.Sprintf("Init:%d/%d", i, len(pod.Spec.InitContainers))
+			initializing = true
+		}
+		break
+	}
+	if !initializing {
+		hasRunning := false
+		for i := len(pod.Status.ContainerStatuses) - 1; i >= 0; i-- {
+			container := pod.Status.ContainerStatuses[i]
+
+			if container.State.Waiting != nil && container.State.Waiting.Reason != "" {
+				reason = container.State.Waiting.Reason
+			} else if container.State.Terminated != nil && container.State.Terminated.Reason != "" {
+				reason = container.State.Terminated.Reason
+			} else if container.State.Terminated != nil && container.State.Terminated.Reason == "" {
+				if container.State.Terminated.Signal != 0 {
+					reason = fmt.Sprintf("Signal:%d", container.State.Terminated.Signal)
+				} else {
+					reason = fmt.Sprintf("ExitCode:%d", container.State.Terminated.ExitCode)
+				}
+			} else if container.Ready && container.State.Running != nil {
+				hasRunning = true
+			}
+		}
+
+		// change pod status back to "Running" if there is at least one container still reporting as "Running" status
+		if reason == "Completed" && hasRunning {
+			reason = "Running"
+		}
+	}
+
+	return reason
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		NAME STATUS MESHED SUCCESS RPS LATENCY_P50 LATENCY_P95 LATENCY_P99 TCP_CONN
		emoji Running 1/1 100.00% 2.0rps 123ms 123ms 123ms 123