Enable JSON reports for qps drivers

fabware · Apr 14, 2016 · 969ffaf · 969ffaf
1 parent f2ba7fe
commit 969ffaf
Show file tree

Hide file tree

Showing 5 changed files with 38 additions and 3 deletions.
diff --git a/test/cpp/qps/qps_json_driver.cc b/test/cpp/qps/qps_json_driver.cc
@@ -102,12 +102,16 @@ static void QpsDriver() {
   for (int i = 0; i < scenarios.scenarios_size(); i++) {
     const Scenario &scenario = scenarios.scenarios(i);
     std::cerr << "RUNNING SCENARIO: " << scenario.name() << "\n";
-    const auto result =
+    auto result =
         RunScenario(scenario.client_config(), scenario.num_clients(),
                     scenario.server_config(), scenario.num_servers(),
                     scenario.warmup_seconds(), scenario.benchmark_seconds(),
                     scenario.spawn_local_worker_count());
 
+    // Amend the result with scenario config. Eventually we should adjust
+    // RunScenario contract so we don't need to touch the result here.
+    result->mutable_scenario()->CopyFrom(scenario);
+
     GetReporter()->ReportQPS(*result);
     GetReporter()->ReportQPSPerCore(*result);
     GetReporter()->ReportLatency(*result);

diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc
@@ -33,6 +33,11 @@
 
 #include "test/cpp/qps/report.h"
 
+#include <fstream>
+
+#include <google/protobuf/util/json_util.h>
+#include <google/protobuf/util/type_resolver_util.h>
+
 #include <grpc/support/log.h>
 #include "test/cpp/qps/driver.h"
 #include "test/cpp/qps/stats.h"
@@ -120,7 +125,21 @@ void GprLogReporter::ReportTimes(const ScenarioResult& result) {
 }
 
 void JsonReporter::ReportQPS(const ScenarioResult& result) {
-
+  std::unique_ptr<google::protobuf::util::TypeResolver> type_resolver(
+      google::protobuf::util::NewTypeResolverForDescriptorPool(
+          "type.googleapis.com",
+          google::protobuf::DescriptorPool::generated_pool()));
+  grpc::string binary;
+  grpc::string json_string;
+  result.SerializeToString(&binary);
+  auto status = BinaryToJsonString(type_resolver.get(),
+                                   "type.googleapis.com/grpc.testing.ScenarioResult",
+                                   binary, &json_string);
+  GPR_ASSERT(status.ok());
+
+  std::ofstream output_file(report_file_);
+  output_file << json_string;
+  output_file.close();
 }
 
 void JsonReporter::ReportQPSPerCore(const ScenarioResult& result) {

diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h
@@ -107,13 +107,17 @@ class GprLogReporter : public Reporter {
 /** Dumps the report to a JSON file. */
 class JsonReporter : public Reporter {
  public:
-  JsonReporter(const string& name) : Reporter(name) {}
+  JsonReporter(const string& name, const string& report_file) :
+    Reporter(name),
+    report_file_(report_file) {}
 
  private:
   void ReportQPS(const ScenarioResult& result) GRPC_OVERRIDE;
   void ReportQPSPerCore(const ScenarioResult& result) GRPC_OVERRIDE;
   void ReportLatency(const ScenarioResult& result) GRPC_OVERRIDE;
   void ReportTimes(const ScenarioResult& result) GRPC_OVERRIDE;
+
+  const string report_file_;
 };
 
 }  // namespace testing

diff --git a/test/cpp/util/benchmark_config.cc b/test/cpp/util/benchmark_config.cc
@@ -37,6 +37,9 @@
 DEFINE_bool(enable_log_reporter, true,
             "Enable reporting of benchmark results through GprLog");
 
+DEFINE_string(scenario_result_file, "",
+              "Write JSON benchmark report to the file specified.");
+
 DEFINE_string(hashed_id, "", "Hash of the user id");
 
 DEFINE_string(test_name, "", "Name of the test being executed");
@@ -68,6 +71,10 @@ static std::shared_ptr<Reporter> InitBenchmarkReporters() {
     composite_reporter->add(
         std::unique_ptr<Reporter>(new GprLogReporter("LogReporter")));
   }
+  if (FLAGS_scenario_result_file != "") {
+    composite_reporter->add(std::unique_ptr<Reporter>(
+        new JsonReporter("JsonReporter", FLAGS_scenario_result_file)));
+  }
 
   return std::shared_ptr<Reporter>(composite_reporter);
 }

diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py
@@ -98,6 +98,7 @@ def create_scenario_jobspec(scenario_json, workers, remote_host=None):
   # setting QPS_WORKERS env variable here makes sure it works with SSH too.
   cmd = 'QPS_WORKERS="%s" bins/opt/qps_json_driver ' % ','.join(workers)
   cmd += '--scenarios_json=%s' % pipes.quote(json.dumps({'scenarios': [scenario_json]}))
+  cmd += ' --scenario_result_file=scenario_result.json'
   if remote_host:
     user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host)
     cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && %s"' % (user_at_host, cmd)