tools/cpudist: Add extension summary support (#4131)

Sometimes, I want to known total on-CPU or off-CPU time and count (same as context switch times) at a fixed interval (for example: 1s). Like #3384, This patch try to add an option -e to show extension summary (average/total/count). $ ./cpudist.py -p $(pgrep -nx mysqld) -e 1 usecs : count distribution 0 -> 1 : 4123 |************** | 2 -> 3 : 11690 |****************************************| 4 -> 7 : 1668 |***** | 8 -> 15 : 859 |** | 16 -> 31 : 618 |** | 32 -> 63 : 290 | | 64 -> 127 : 247 | | 128 -> 255 : 198 | | 256 -> 511 : 161 | | 512 -> 1023 : 370 |* | 1024 -> 2047 : 98 | | 2048 -> 4095 : 6 | | 4096 -> 8191 : 16 | | avg = 33 usecs, total: 682091 usecs, count: 20383
iovisor · Jul 29, 2022 · 4a60443 · 4a60443
1 parent 300fc6a
commit 4a60443
Showing 3 changed files with 66 additions and 18 deletions.
diff --git a/man/man8/cpudist.8 b/man/man8/cpudist.8
@@ -2,7 +2,7 @@
 .SH NAME
 cpudist \- On- and off-CPU task time as a histogram.
 .SH SYNOPSIS
-.B cpudist [\-h] [-O] [\-T] [\-m] [\-P] [\-L] [\-p PID] [\-I] [interval] [count]
+.B cpudist [\-h] [-O] [\-T] [\-m] [\-P] [\-L] [\-p PID] [\-I] [\-e] [interval] [count]
 .SH DESCRIPTION
 This measures the time a task spends on the CPU before being descheduled, and
 shows the times as a histogram. Tasks that spend a very short time on the CPU
@@ -50,6 +50,9 @@ Only show this PID (filtered in kernel for efficiency).
 \-I
 Include CPU idle time (by default these are excluded).
 .TP
+\-e
+Show extension summary (average/total/count).
+.TP
 interval
 Output interval, in seconds.
 .TP
@@ -63,7 +66,7 @@ Summarize task on-CPU time as a histogram:
 .TP
 Summarize task off-CPU time as a histogram:
 #
-.B cpudist -O
+.B cpudist \-O
 .TP
 Print 1 second summaries, 10 times:
 #
@@ -75,11 +78,15 @@ Print 1 second summaries, using milliseconds as units for the histogram, and inc
 .TP
 Trace PID 185 only, 1 second summaries:
 #
-.B cpudist -p 185 1
+.B cpudist \-p 185 1
 .TP
 Include CPU idle time:
 #
-.B cpudist -I
+.B cpudist \-I
+.TP
+Also show extension summary:
+#
+.B cpudist \-e
 .SH FIELDS
 .TP
 usecs
@@ -111,6 +118,6 @@ Linux
 .SH STABILITY
 Unstable - in development.
 .SH AUTHOR
-Sasha Goldshtein
+Sasha Goldshtein, Rocky Xing
 .SH SEE ALSO
 pidstat(1), runqlat(8)
diff --git a/tools/cpudist.py b/tools/cpudist.py
@@ -3,7 +3,7 @@
 #
 # cpudist   Summarize on- and off-CPU time per task as a histogram.
 #
-# USAGE: cpudist [-h] [-O] [-T] [-m] [-P] [-L] [-p PID] [-I] [interval] [count]
+# USAGE: cpudist [-h] [-O] [-T] [-m] [-P] [-L] [-p PID] [-I] [-e] [interval] [count]
 #
 # This measures the time a task spends on or off the CPU, and shows this time
 # as a histogram, optionally per-process.
@@ -14,6 +14,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 #
 # 27-Mar-2022   Rocky Xing      Changed to exclude CPU idle time by default.
+# 25-Jul-2022   Rocky Xing      Added extension summary support.
 
 from __future__ import print_function
 from bcc import BPF
@@ -28,9 +29,10 @@
     cpudist -P           # show each PID separately
     cpudist -p 185       # trace PID 185 only
     cpudist -I           # include CPU idle time
+    cpudist -e           # show extension summary (average/total/count)
 """
 parser = argparse.ArgumentParser(
-    description="Summarize on-CPU time per task as a histogram.",
+    description="Summarize on- and off-CPU time per task as a histogram.",
     formatter_class=argparse.RawDescriptionHelpFormatter,
     epilog=examples)
 parser.add_argument("-O", "--offcpu", action="store_true",
@@ -47,6 +49,8 @@
     help="trace this PID only")
 parser.add_argument("-I", "--include-idle", action="store_true",
     help="include CPU idle time")
+parser.add_argument("-e", "--extension", action="store_true",
+    help="show extension summary (average/total/count)")
 parser.add_argument("interval", nargs="?", default=99999999,
     help="output interval, in seconds")
 parser.add_argument("count", nargs="?", default=99999999,
@@ -57,7 +61,8 @@
 countdown = int(args.count)
 debug = 0
 
-bpf_text = """#include <uapi/linux/ptrace.h>
+bpf_text = """
+#include <uapi/linux/ptrace.h>
 #include <linux/sched.h>
 """
 
@@ -75,6 +80,10 @@
     u64 slot;
 } pid_key_t;
 
+typedef struct ext_val {
+    u64 total;
+    u64 count;
+} ext_val_t;
 
 BPF_HASH(start, entry_key_t, u64, MAX_PID);
 STORAGE
@@ -157,22 +166,40 @@
 else:
     bpf_text = bpf_text.replace('FACTOR', 'delta /= 1000;')
     label = "usecs"
+
+storage_str = ""
+store_str = ""
+
 if args.pids or args.tids:
     section = "pid"
     pid = "tgid"
     if args.tids:
         pid = "pid"
         section = "tid"
-    bpf_text = bpf_text.replace('STORAGE',
-        'BPF_HISTOGRAM(dist, pid_key_t, MAX_PID);')
-    bpf_text = bpf_text.replace('STORE',
-        'pid_key_t key = {.id = ' + pid + ', .slot = bpf_log2l(delta)}; ' +
-        'dist.increment(key);')
+    storage_str += "BPF_HISTOGRAM(dist, pid_key_t, MAX_PID);"
+    store_str += """
+    pid_key_t key = {.id = ' + pid + ', .slot = bpf_log2l(delta)};
+    dist.increment(key);
+    """
 else:
     section = ""
-    bpf_text = bpf_text.replace('STORAGE', 'BPF_HISTOGRAM(dist);')
-    bpf_text = bpf_text.replace('STORE',
-        'dist.atomic_increment(bpf_log2l(delta));')
+    storage_str += "BPF_HISTOGRAM(dist);"
+    store_str += "dist.atomic_increment(bpf_log2l(delta));"
+
+if args.extension:
+    storage_str += "BPF_ARRAY(extension, ext_val_t, 1);"
+    store_str += """
+    u32 index = 0;
+    ext_val_t *ext_val = extension.lookup(&index);
+    if (ext_val) {
+        lock_xadd(&ext_val->total, delta);
+        lock_xadd(&ext_val->count, 1);
+    }
+    """
+
+bpf_text = bpf_text.replace("STORAGE", storage_str)
+bpf_text = bpf_text.replace("STORE", store_str)
+
 if debug or args.ebpf:
     print(bpf_text)
     if args.ebpf:
@@ -189,6 +216,8 @@
 
 exiting = 0 if args.interval else 1
 dist = b.get_table("dist")
+if args.extension:
+    extension = b.get_table("extension")
 while (1):
     try:
         sleep(int(args.interval))
@@ -207,6 +236,15 @@ def pid_to_comm(pid):
             return str(pid)
 
     dist.print_log2_hist(label, section, section_print_fn=pid_to_comm)
+
+    if args.extension:
+        total = extension[0].total
+        count = extension[0].count
+        if count > 0:
+            print("\navg = %ld %s, total: %ld %s, count: %ld\n" %
+                (total / count, label, total, label, count))
+        extension.clear()
+
     dist.clear()
 
     countdown -= 1

diff --git a/tools/cpudist_example.txt b/tools/cpudist_example.txt
@@ -282,9 +282,10 @@ USAGE message:
 
 # ./cpudist.py -h
 
-usage: cpudist.py [-h] [-O] [-T] [-m] [-P] [-L] [-p PID] [-I] [interval] [count]
+usage: cpudist.py [-h] [-O] [-T] [-m] [-P] [-L] [-p PID] [-I] [-e]
+                  [interval] [count]
 
-Summarize on-CPU time per task as a histogram.
+Summarize on- and off-CPU time per task as a histogram.
 
 positional arguments:
   interval            output interval, in seconds
@@ -299,6 +300,7 @@ optional arguments:
   -L, --tids          print a histogram per thread ID
   -p PID, --pid PID   trace this PID only
   -I, --include-idle  include CPU idle time
+  -e, --extension     show extension summary (average/total/count)
 
 examples:
     cpudist              # summarize on-CPU time as a histogram
@@ -308,4 +310,5 @@ examples:
     cpudist -P           # show each PID separately
     cpudist -p 185       # trace PID 185 only
     cpudist -I           # include CPU idle time
+    cpudist -e           # show extension summary (average/total/count)