intial commit: added scripts for analyzing scheduling results

casys-kaist · Jul 18, 2023 · 7759991 · 7759991
1 parent 149365a
commit 7759991
Show file tree

Hide file tree

Showing 2 changed files with 294 additions and 0 deletions.
diff --git a/scripts/analyzeInterScheduleResults.py b/scripts/analyzeInterScheduleResults.py
@@ -0,0 +1,179 @@
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+import numpy as np
+import pandas as pd
+import csv
+import os
+from operator import itemgetter
+from basicAnalysis import med
+
+
+metrics=('avg_latency', 'tail_latency', 'total_throughput', 'slo')
+
+def parse_args():
+    parser = ArgumentParser(description=__doc__,
+            formatter_class=ArgumentDefaultsHelpFormatter)
+    parser.add_argument('log_root_dir',
+            help='directory which holds log files')
+    parser.add_argument('result_file',
+            help='file which is going to hold analyzed results')
+    parser.add_argument('iternum',
+            help='number of iterations')
+
+
+    args = parser.parse_args()
+    return args
+
+
+def readandfillTable(metric_file, benchwise_info, benchmarks, benchwise_client_info):
+        with open (metric_file) as fp:
+                lines = fp.readlines()
+                numofbench=len(benchmarks)
+                cnt=0
+                foundSign=False
+                foundTotalThroughput=False
+                foundServerLatency=False
+                foundClientLatency=False
+                foundServerSLO=False
+                for line in lines:
+                        if not foundSign:
+                                if "SERVER" in line and "Total" in line and "APP" in line:
+                                        foundSign=True
+                                        foundTotalThroughput=True
+                                        cnt=0
+                                elif "SERVER" in line and "Latency" in line and "APP" in line:
+                                        foundSign=True
+                                        foundServerLatency=True
+                                        cnt=0
+                                elif "SERVER" in line and "SLO" in line and "APP" in line:
+                                        foundSign=True
+                                        foundServerSLO=True
+                                        cnt=0
+                                continue
+
+                        if foundTotalThroughput:
+                                if cnt == 0:
+                                        cnt = cnt + 1
+                                        continue
+                                key=line.split(",")[0]
+                                value=float(line.split(",")[1])
+                                benchwise_info[key]["total_throughput"].append(value)
+                                cnt = cnt +1
+                                if cnt == numofbench + 1:
+                                        foundSign=False
+                                        foundTotalThroughput=False
+                                        cnt=0
+                                continue
+                        elif foundServerLatency:
+                                if cnt == 0:
+                                        cnt = cnt + 1
+                                        continue
+                                key=line.split(",")[0]
+                                value1=float(line.split(",")[1])
+                                value2=float(line.split(",")[2])
+                                benchwise_info[key]["avg_latency"].append(value1)
+                                benchwise_info[key]["tail_latency"].append(value2)
+                                cnt = cnt +1
+                                if cnt == numofbench + 1:
+                                        foundSign=False
+                                        foundServerLatency=False
+                                        cnt=0
+                                continue
+                        elif foundServerSLO:
+                                if cnt == 0:
+                                        cnt = cnt + 1
+                                        continue
+                                key=line.split(",")[0]
+                                value1=float(line.split(",")[1])
+                                benchwise_info[key]["slo"].append(value1)
+                                cnt = cnt +1
+                                if cnt == numofbench + 1:
+                                        foundSign=False
+                                        foundServerSLO=False
+                                        cnt=0
+                                continue
+                        elif foundClientLatency:
+                                if cnt == 0:
+                                        cnt = cnt + 1
+                                        continue
+                                key=line.split(",")[0]
+                                value1=float(line.split(",")[1])
+                                value2=float(line.split(",")[2])
+                                benchwise_client_info[key]["avg_latency"].append(value1)
+                                benchwise_client_info[key]["tail_latency"].append(value2)
+                                cnt = cnt +1
+                                if cnt == numofbench + 1:
+                                        foundSign=False
+                                        foundClientLatency=False
+                                        cnt=0
+                                continue
+
+def writeRecords(result_file, benchwise_info, schedulers, benchmarks, benchwise_client_info):
+        with open(result_file,"w") as fp:
+            for metric in metrics:
+                    for schedule in schedulers:
+                        fp.write("[Server-"+metric+"-"+schedule+"],")
+                        fp.write("max,average,min")
+                        fp.write("\n")
+                        for bench in benchmarks:
+                            fp.write(bench+",")
+                            key=schedule+"-"+bench
+                            print(metric)
+                            fp.write(str(max(benchwise_info[key][metric])) +",")
+                            fp.write(str(med(benchwise_info[key][metric])) +",")
+                            fp.write(str(min(benchwise_info[key][metric])) +",")
+                            fp.write("\n")
+def fillSchedulersandBenchmarks(metric_file, schedulers, benchmarks):
+    with open (metric_file) as fp:
+                lines = fp.readlines()
+                found=False
+                cnt=1
+                for line in lines:
+                    if "SERVER" in line and "APP" in line and "Latency" in line:
+                        found=True
+                        continue
+                    if not found:
+                        continue
+                    if cnt != 0:
+                        cnt = cnt -1
+                        continue
+                    words=line.split(",")
+                    if len(words) == 1 or len(words) == 0:
+                        found=False
+                        cnt = cnt +1
+                        continue
+                    schedule=words[0].split("-")[0]
+                    benchmark=words[0].split("-")[1]
+                    if benchmark == 'ssd' or benchmark == 'mobilenetv1':
+                        benchmark = 'ssd-mobilenetv1'
+                    if schedule not in schedulers:
+                        schedulers.append(schedule)
+                    if benchmark not in benchmarks:
+                        benchmarks.append(benchmark)
+
+
+def main():
+    args = parse_args()
+    #setup schedule-benchmark wise data
+    schedulers=[]
+    benchmarks=[]
+    fillSchedulersandBenchmarks(args.log_root_dir+"/"+"1-schd_metrics.csv",schedulers, benchmarks) 
+    print (benchmarks)
+    print(schedulers)
+    benchwise_info={}
+    benchwise_client_info={}
+    for schedule in schedulers:
+        for bench in benchmarks:
+            key=schedule+"-"+bench
+            benchwise_info[key]={}
+            benchwise_client_info[key]={}
+            for metric in metrics:
+                benchwise_info[key][metric]=[]
+                benchwise_client_info[key][metric]=[]
+    for i in range(1,int(args.iternum)+1):
+            metric_file_name=str(i)+"-schd_metrics.csv"
+            readandfillTable(args.log_root_dir+"/"+metric_file_name, benchwise_info,benchmarks ,benchwise_client_info)
+    writeRecords(args.result_file, benchwise_info,schedulers, benchmarks, benchwise_client_info)
+
+if __name__ == '__main__':
+    main()
+
diff --git a/scripts/analyzeServerClient.py b/scripts/analyzeServerClient.py
@@ -0,0 +1,115 @@
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+import numpy as np
+import pandas as pd
+import csv
+import os
+#from itertools import izip_longest
+from operator import itemgetter
+from basicAnalysis import average, tail, getThroughput_MakeSpan, diffTimestamp_s
+from basicFileAnalysis import analyzeServerLatency,analyzeServerBreakdownAVG,analyzeServerBreakdownTAIL,analyzeSLO,analyzeThroughput2
+
+def parse_args():
+    parser = ArgumentParser(description=__doc__,
+            formatter_class=ArgumentDefaultsHelpFormatter)
+    parser.add_argument('log_root_dir',
+            help='directory which holds log files')
+    parser.add_argument('result_file',
+            help='file which is going to hold analyzed results')
+    args = parser.parse_args()
+    return args
+
+def readBenchmarksandStages(server_log_file):
+    benchmarks=[]
+    stages=[]
+    with open (server_log_file) as fp:
+        lines = fp.readlines()
+        cnt=1
+        for line in lines:
+            if cnt != 0:
+                # read and store stages
+                line=line[:-1]
+                words = line.split(',')
+                for x in range(2, len(words)):
+                    if "ID" not in words[x]:
+                        stages.append(words[x])
+                cnt = cnt -1
+                continue
+            # store benchmarks
+            bench = line.split(',')[1]
+            if bench not in benchmarks:
+                benchmarks.append(bench)
+    return benchmarks,stages
+
+
+def gatherClientLatency(client_file, client_benchwise_info):
+        with open (client_file) as fp:
+                print("processing " + client_file)
+                dirnames = client_file.split("/")
+                filename = dirnames[-1]
+                benchmark = filename.split("-")[0]
+                if benchmark == 'ssd': # this happens due to '-' seperated naming
+                    benchmark='ssd-mobilenetv1'
+                lines = fp.readlines()
+                if len(lines) == 1: # something is wrong....
+                        item=(0,0)
+                        return item
+                for line in lines:
+                        words=line.split(' ')
+#                       print line
+                        if words[0] != "Respond":
+                                continue
+                        if float(words[2]) > 10000000 : # we have a serious problem of huge numbers, skip it if so
+                                continue
+                        #perf.append(float(words[2]))
+                        client_benchwise_info[benchmark].append(float(words[2]))
+
+def writeClientLatency(result_file, schedule, client_benchwise_info):
+    with open(result_file,"a") as fp:
+        fp.write("====CLIENT Latency====\n")
+        fp.write("schedule-name,AVG latency, TAIL latency\n")
+        for bench in client_benchwise_info:
+            fp.write(schedule+"-"+bench+",")
+            fp.write(str(average(client_benchwise_info[bench]))+",")
+            fp.write(str(tail(client_benchwise_info[bench]))+"\n")
+
+def parsetoFile( result_file, log_root_dir):
+    for subdir in os.listdir(log_root_dir):
+        client_benchwise_info={}
+        model_benchmarks,model_stages=readBenchmarksandStages(log_root_dir+"/"+subdir+"/server-model.csv")
+        app_benchmarks,app_stages=readBenchmarksandStages(log_root_dir+"/"+subdir+"/server-app.csv")
+
+        for bench in app_benchmarks:
+            client_benchwise_info[bench] = []
+            client_data=[]
+        print(model_stages)
+        print(model_benchmarks)
+        print(app_stages)
+        print(app_benchmarks)
+        schedule=subdir
+        for f in os.listdir(log_root_dir+"/"+subdir):
+            name=f.split(".")[0]
+            bench=name.split("-")[0]
+            metric=name.split("-")[-1]
+            item=(schedule,bench)
+            if metric == "app":
+                analyzeServerLatency(log_root_dir+"/"+subdir+"/"+f, result_file,schedule, app_benchmarks, app_stages, metric)
+                analyzeServerBreakdownAVG(log_root_dir+"/"+subdir+"/"+f, result_file,schedule, app_benchmarks, app_stages, metric)
+                analyzeServerBreakdownTAIL(log_root_dir+"/"+subdir+"/"+f, result_file,schedule, app_benchmarks,app_stages, metric)
+                analyzeThroughput2(log_root_dir+"/"+subdir+"/"+f, result_file, schedule, app_benchmarks, metric)
+                analyzeSLO(log_root_dir+"/"+subdir+"/"+f, result_file, schedule, app_benchmarks,app_stages ,metric)
+            elif metric == "model":
+                analyzeServerLatency(log_root_dir+"/"+subdir+"/"+f, result_file,schedule, model_benchmarks, model_stages, metric)
+                analyzeServerBreakdownAVG(log_root_dir+"/"+subdir+"/"+f, result_file,schedule, model_benchmarks, model_stages, metric)
+                analyzeServerBreakdownTAIL(log_root_dir+"/"+subdir+"/"+f, result_file,schedule, model_benchmarks,model_stages, metric)
+                analyzeThroughput2(log_root_dir+"/"+subdir+"/"+f, result_file, schedule, model_benchmarks, metric)
+            elif metric == "client":
+                gatherClientLatency(log_root_dir+"/"+subdir+"/"+f, client_benchwise_info)
+        writeClientLatency(result_file,schedule,client_benchwise_info)
+
+def main():
+    args = parse_args()
+    parsetoFile(args.result_file, args.log_root_dir)
+
+if __name__ == '__main__':
+    main()
+