Skip to content

Commit

Permalink
Add trafficsplit metrics to CLI (linkerd#3176)
Browse files Browse the repository at this point in the history 
This PR adds `trafficsplit` as a supported resource for the `linkerd stat` command. Users can type `linkerd stat ts` to see the apex and leaf services of their trafficsplits, as well as metrics for those leaf services.
  • Loading branch information
Carol A. Scott authored Aug 14, 2019
1 parent 9826cbd commit 0043770
Show file tree
Hide file tree
Showing 11 changed files with 722 additions and 228 deletions.
144 changes: 125 additions & 19 deletions cli/cmd/stat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ func newCmdStat() *cobra.Command {
* po mypod1 mypod2
* rc/my-replication-controller
* sts/my-statefulset
* ts/my-split
* authority
* au/my-authority
* all
Expand All @@ -80,6 +81,7 @@ func newCmdStat() *cobra.Command {
* pods
* replicationcontrollers
* statefulsets
* trafficsplits
* authorities (not supported in --from)
* services (only supported if a --from is also specified, or as a --to)
* all (all resource types, not supported in --from or --to)
Expand Down Expand Up @@ -113,6 +115,15 @@ If no resource name is specified, displays stats about all resources of the spec
# Get all services in all namespaces that receive calls from hello1 deployment in the test namespace.
linkerd stat services --from deploy/hello1 --from-namespace test --all-namespaces
# Get all trafficsplits and their leaf services.
linkerd stat ts
# Get the hello-split trafficsplit and its leaf services.
linkerd stat ts/hello-split
# Get all trafficsplits and their leaf services, and metrics for any traffic coming to the leaf services from the hello1 deployment.
linkerd stat ts --from deploy/hello1
# Get all namespaces that receive traffic from the default namespace.
linkerd stat namespaces --from ns/default
Expand Down Expand Up @@ -219,16 +230,30 @@ type row struct {
meshed string
status string
*rowStats
*tsStats
}

type tsStats struct {
apex string
leaf string
weight string
}

var (
nameHeader = "NAME"
namespaceHeader = "NAMESPACE"
apexHeader = "APEX"
leafHeader = "LEAF"
weightHeader = "WEIGHT"
)

func writeStatsToBuffer(rows []*pb.StatTable_PodGroup_Row, w *tabwriter.Writer, options *statOptions) {
maxNameLength := len(nameHeader)
maxNamespaceLength := len(namespaceHeader)
maxApexLength := len(apexHeader)
maxLeafLength := len(leafHeader)
maxWeightLength := len(weightHeader)

statTables := make(map[string]map[string]*row)

prefixTypes := make(map[string]bool)
Expand All @@ -249,6 +274,9 @@ func writeStatsToBuffer(rows []*pb.StatTable_PodGroup_Row, w *tabwriter.Writer,

namespace := r.Resource.Namespace
key := fmt.Sprintf("%s/%s", namespace, name)
if r.Resource.Type == k8s.TrafficSplit {
key = fmt.Sprintf("%s/%s/%s", namespace, name, r.TsStats.Leaf)
}
resourceKey := r.Resource.Type

if _, ok := statTables[resourceKey]; !ok {
Expand Down Expand Up @@ -284,6 +312,29 @@ func writeStatsToBuffer(rows []*pb.StatTable_PodGroup_Row, w *tabwriter.Writer,
tcpWriteBytes: getByteRate(r.GetTcpStats().GetWriteBytesTotal(), r.TimeWindow),
}
}
if r.TsStats != nil {
leaf := r.TsStats.Leaf
apex := r.TsStats.Apex
weight := r.TsStats.Weight

if len(leaf) > maxLeafLength {
maxLeafLength = len(leaf)
}

if len(apex) > maxApexLength {
maxApexLength = len(apex)
}

if len(weight) > maxWeightLength {
maxWeightLength = len(weight)
}

statTables[resourceKey][key].tsStats = &tsStats{
apex: apex,
leaf: leaf,
weight: weight,
}
}
}

switch options.outputFormat {
Expand All @@ -292,13 +343,13 @@ func writeStatsToBuffer(rows []*pb.StatTable_PodGroup_Row, w *tabwriter.Writer,
fmt.Fprintln(os.Stderr, "No traffic found.")
os.Exit(0)
}
printStatTables(statTables, w, maxNameLength, maxNamespaceLength, options)
printStatTables(statTables, w, maxNameLength, maxNamespaceLength, maxLeafLength, maxApexLength, maxWeightLength, options)
case jsonOutput:
printStatJSON(statTables, w)
}
}

func printStatTables(statTables map[string]map[string]*row, w *tabwriter.Writer, maxNameLength int, maxNamespaceLength int, options *statOptions) {
func printStatTables(statTables map[string]map[string]*row, w *tabwriter.Writer, maxNameLength, maxNamespaceLength, maxLeafLength, maxApexLength, maxWeightLength int, options *statOptions) {
usePrefix := false
if len(statTables) > 1 {
usePrefix = true
Expand All @@ -315,7 +366,7 @@ func printStatTables(statTables map[string]map[string]*row, w *tabwriter.Writer,
if !usePrefix {
resourceTypeLabel = ""
}
printSingleStatTable(stats, resourceTypeLabel, resourceType, w, maxNameLength, maxNamespaceLength, options)
printSingleStatTable(stats, resourceTypeLabel, resourceType, w, maxNameLength, maxNamespaceLength, maxLeafLength, maxApexLength, maxWeightLength, options)
}
}
}
Expand All @@ -326,32 +377,50 @@ func showTCPBytes(options *statOptions, resourceType string) bool {
}

func showTCPConns(resourceType string) bool {
return resourceType != k8s.Authority
return resourceType != k8s.Authority && resourceType != k8s.TrafficSplit
}

func printSingleStatTable(stats map[string]*row, resourceTypeLabel, resourceType string, w *tabwriter.Writer, maxNameLength int, maxNamespaceLength int, options *statOptions) {
func printSingleStatTable(stats map[string]*row, resourceTypeLabel, resourceType string, w *tabwriter.Writer, maxNameLength, maxNamespaceLength, maxLeafLength, maxApexLength, maxWeightLength int, options *statOptions) {
headers := make([]string, 0)
nameTemplate := fmt.Sprintf("%%-%ds", maxNameLength)
namespaceTemplate := fmt.Sprintf("%%-%ds", maxNamespaceLength)
apexTemplate := fmt.Sprintf("%%-%ds", maxApexLength)
leafTemplate := fmt.Sprintf("%%-%ds", maxLeafLength)
weightTemplate := fmt.Sprintf("%%-%ds", maxWeightLength)

if options.allNamespaces {
headers = append(headers,
namespaceHeader+strings.Repeat(" ", maxNamespaceLength-len(namespaceHeader)))
fmt.Sprintf(namespaceTemplate, namespaceHeader))
}

headers = append(headers, nameHeader+strings.Repeat(" ", maxNameLength-len(nameHeader)))
headers = append(headers,
fmt.Sprintf(nameTemplate, nameHeader))

if resourceType == k8s.Pod {
headers = append(headers, "STATUS")
}

if resourceType == k8s.TrafficSplit {
headers = append(headers,
fmt.Sprintf(apexTemplate, apexHeader),
fmt.Sprintf(leafTemplate, leafHeader),
fmt.Sprintf(weightTemplate, weightHeader))
} else {
headers = append(headers, "MESHED")
}

headers = append(headers, []string{
"MESHED",
"SUCCESS",
"RPS",
"LATENCY_P50",
"LATENCY_P95",
"LATENCY_P99",
"TCP_CONN",
}...)

if resourceType != k8s.TrafficSplit {
headers = append(headers, "TCP_CONN")
}

if showTCPBytes(options, resourceType) {
headers = append(headers, []string{
"READ_BYTES/SEC",
Expand All @@ -374,9 +443,16 @@ func printSingleStatTable(stats map[string]*row, resourceTypeLabel, resourceType
templateStringEmpty = "%s\t" + templateStringEmpty
}

if resourceType == k8s.TrafficSplit {
templateString = "%s\t%s\t%s\t%s\t%.2f%%\t%.1frps\t%dms\t%dms\t%dms\t"
templateStringEmpty = "%s\t%s\t%s\t%s\t-\t-\t-\t-\t-\t"
}

if !showTCPConns(resourceType) {
// always show TCP Connections as - for Authorities
templateString = templateString + "-\t"
if resourceType == k8s.Authority {
// always show TCP Connections as - for Authorities
templateString = templateString + "-\t"
}
} else {
templateString = templateString + "%d\t"
}
Expand All @@ -401,14 +477,34 @@ func printSingleStatTable(stats map[string]*row, resourceTypeLabel, resourceType
padding = maxNameLength - len(name)
}

apexPadding := 0
leafPadding := 0

if stats[key].tsStats != nil {
if maxApexLength > len(stats[key].tsStats.apex) {
apexPadding = maxApexLength - len(stats[key].tsStats.apex)
}
if maxLeafLength > len(stats[key].tsStats.leaf) {
leafPadding = maxLeafLength - len(stats[key].tsStats.leaf)
}
}

values = append(values, name+strings.Repeat(" ", padding))
if resourceType == k8s.Pod {
values = append(values, stats[key].status)
}

values = append(values, []interface{}{
stats[key].meshed,
}...)
if resourceType == k8s.TrafficSplit {
values = append(values,
stats[key].tsStats.apex+strings.Repeat(" ", apexPadding),
stats[key].tsStats.leaf+strings.Repeat(" ", leafPadding),
stats[key].tsStats.weight,
)
} else {
values = append(values, []interface{}{
stats[key].meshed,
}...)
}

if stats[key].rowStats != nil {
values = append(values, []interface{}{
Expand Down Expand Up @@ -450,15 +546,18 @@ type jsonStats struct {
Namespace string `json:"namespace"`
Kind string `json:"kind"`
Name string `json:"name"`
Meshed string `json:"meshed"`
Meshed string `json:"meshed,omitempty"`
Success *float64 `json:"success"`
Rps *float64 `json:"rps"`
LatencyMSp50 *uint64 `json:"latency_ms_p50"`
LatencyMSp95 *uint64 `json:"latency_ms_p95"`
LatencyMSp99 *uint64 `json:"latency_ms_p99"`
TCPConnections *uint64 `json:"tcp_open_connections"`
TCPReadBytes *float64 `json:"tcp_read_bytes_rate"`
TCPWriteBytes *float64 `json:"tcp_write_bytes_rate"`
TCPConnections *uint64 `json:"tcp_open_connections,omitempty"`
TCPReadBytes *float64 `json:"tcp_read_bytes_rate,omitempty"`
TCPWriteBytes *float64 `json:"tcp_write_bytes_rate,omitempty"`
Apex string `json:"apex,omitempty"`
Leaf string `json:"leaf,omitempty"`
Weight string `json:"weight,omitempty"`
}

func printStatJSON(statTables map[string]map[string]*row, w *tabwriter.Writer) {
Expand All @@ -473,7 +572,9 @@ func printStatJSON(statTables map[string]map[string]*row, w *tabwriter.Writer) {
Namespace: namespace,
Kind: resourceType,
Name: name,
Meshed: stats[key].meshed,
}
if resourceType != k8s.TrafficSplit {
entry.Meshed = stats[key].meshed
}
if stats[key].rowStats != nil {
entry.Success = &stats[key].successRate
Expand All @@ -489,6 +590,11 @@ func printStatJSON(statTables map[string]map[string]*row, w *tabwriter.Writer) {
}
}

if stats[key].tsStats != nil {
entry.Apex = stats[key].apex
entry.Leaf = stats[key].leaf
entry.Weight = stats[key].weight
}
entries = append(entries, entry)
}
}
Expand Down
22 changes: 22 additions & 0 deletions cli/cmd/stat_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,14 @@ func TestStat(t *testing.T) {
}, k8s.Pod, t)
})

t.Run("Returns trafficsplit stats", func(t *testing.T) {
testStatCall(paramsExp{
options: options,
resNs: []string{"default"},
file: "stat_one_ts_output.golden",
}, k8s.TrafficSplit, t)
})

options.outputFormat = jsonOutput
t.Run("Returns namespace stats (json)", func(t *testing.T) {
testStatCall(paramsExp{
Expand All @@ -57,6 +65,14 @@ func TestStat(t *testing.T) {
}, k8s.Namespace, t)
})

t.Run("Returns trafficsplit stats (json)", func(t *testing.T) {
testStatCall(paramsExp{
options: options,
resNs: []string{"default"},
file: "stat_one_ts_output_json.golden",
}, k8s.TrafficSplit, t)
})

options = newStatOptions()
options.allNamespaces = true
t.Run("Returns all namespace stats", func(t *testing.T) {
Expand Down Expand Up @@ -167,10 +183,16 @@ func TestStat(t *testing.T) {
func testStatCall(exp paramsExp, resourceType string, t *testing.T) {
mockClient := &public.MockAPIClient{}
response := public.GenStatSummaryResponse("emoji", resourceType, exp.resNs, exp.counts, true, true)
if resourceType == k8s.TrafficSplit {
response = public.GenStatTsResponse("foo-split", resourceType, exp.resNs, true, true)
}

mockClient.StatSummaryResponseToReturn = &response

args := []string{"ns"}
if resourceType == k8s.TrafficSplit {
args = []string{"trafficsplit"}
}
reqs, err := buildStatSummaryRequests(args, exp.options)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
Expand Down
3 changes: 3 additions & 0 deletions cli/cmd/testdata/stat_one_ts_output.golden
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
NAME APEX LEAF WEIGHT SUCCESS RPS LATENCY_P50 LATENCY_P95 LATENCY_P99
foo-split apex_name service-1 900m 100.00% 2.0rps 123ms 123ms 123ms
foo-split apex_name service-2 100m 100.00% 2.0rps 123ms 123ms 123ms
28 changes: 28 additions & 0 deletions cli/cmd/testdata/stat_one_ts_output_json.golden
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
[
{
"namespace": "default",
"kind": "trafficsplit",
"name": "foo-split",
"success": 1,
"rps": 2.05,
"latency_ms_p50": 123,
"latency_ms_p95": 123,
"latency_ms_p99": 123,
"apex": "apex_name",
"leaf": "service-1",
"weight": "900m"
},
{
"namespace": "default",
"kind": "trafficsplit",
"name": "foo-split",
"success": 1,
"rps": 2.05,
"latency_ms_p50": 123,
"latency_ms_p95": 123,
"latency_ms_p99": 123,
"apex": "apex_name",
"leaf": "service-2",
"weight": "100m"
}
]
13 changes: 13 additions & 0 deletions controller/api/public/prometheus.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,19 @@ func generateLabelStringWithExclusion(l model.LabelSet, labelName string) string
return fmt.Sprintf("{%s}", strings.Join(lstrs, ", "))
}

// insert a regex-match check into a LabelSet for labels that match the provided
// string. this is modeled on generateLabelStringWithExclusion().
func generateLabelStringWithRegex(l model.LabelSet, labelName string, stringToMatch string) string {
lstrs := make([]string, 0, len(l))
for l, v := range l {
lstrs = append(lstrs, fmt.Sprintf("%s=%q", l, v))
}
lstrs = append(lstrs, fmt.Sprintf(`%s=~"^%s.+"`, labelName, stringToMatch))

sort.Strings(lstrs)
return fmt.Sprintf("{%s}", strings.Join(lstrs, ", "))
}

// determine if we should add "namespace=<namespace>" to a named query
func shouldAddNamespaceLabel(resource *pb.Resource) bool {
return resource.Type != k8s.Namespace && resource.Namespace != ""
Expand Down
Loading

0 comments on commit 0043770

Please sign in to comment.