Merge pull request grpc#4035 from sreecha/stress_tests_metrics

Add more config options to stress tests and export metrics
azureplus · Nov 19, 2015 · e779084 · e779084
2 parents 6d80632 + bc3127d
commit e779084
Show file tree

Hide file tree

Showing 14 changed files with 836 additions and 41 deletions.
diff --git a/Makefile b/Makefile
diff --git a/build.yaml b/build.yaml
@@ -1887,6 +1887,20 @@ targets:
   - mac
   - linux
   - posix
+- name: metrics_client
+  build: test
+  run: false
+  language: c++
+  headers:
+  - test/cpp/util/metrics_server.h
+  src:
+  - test/proto/metrics.proto
+  - test/cpp/interop/metrics_client.cc
+  deps:
+  - grpc++
+  - grpc
+  - gpr
+  - grpc++_test_config
 - name: mock_test
   build: test
   language: c++
@@ -2127,13 +2141,16 @@ targets:
   - test/cpp/interop/client_helper.h
   - test/cpp/interop/interop_client.h
   - test/cpp/interop/stress_interop_client.h
+  - test/cpp/util/metrics_server.h
   src:
   - test/proto/empty.proto
   - test/proto/messages.proto
+  - test/proto/metrics.proto
   - test/proto/test.proto
   - test/cpp/interop/interop_client.cc
   - test/cpp/interop/stress_interop_client.cc
   - test/cpp/interop/stress_test.cc
+  - test/cpp/util/metrics_server.cc
   deps:
   - grpc++_test_util
   - grpc_test_util

diff --git a/test/cpp/interop/metrics_client.cc b/test/cpp/interop/metrics_client.cc
@@ -0,0 +1,102 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *is % allowed in string
+ */
+
+#include <memory>
+#include <string>
+
+#include <gflags/gflags.h>
+#include <grpc++/grpc++.h>
+
+#include "test/cpp/util/metrics_server.h"
+#include "test/cpp/util/test_config.h"
+#include "test/proto/metrics.grpc.pb.h"
+#include "test/proto/metrics.pb.h"
+
+DEFINE_string(metrics_server_address, "",
+              "The metrics server addresses in the fomrat <hostname>:<port>");
+
+using grpc::testing::EmptyMessage;
+using grpc::testing::GaugeResponse;
+using grpc::testing::MetricsService;
+using grpc::testing::MetricsServiceImpl;
+
+void PrintMetrics(grpc::string& server_address) {
+  gpr_log(GPR_INFO, "creating a channel to %s", server_address.c_str());
+  std::shared_ptr<grpc::Channel> channel(
+      grpc::CreateChannel(server_address, grpc::InsecureChannelCredentials()));
+
+  std::unique_ptr<MetricsService::Stub> stub(MetricsService::NewStub(channel));
+
+  grpc::ClientContext context;
+  EmptyMessage message;
+
+  std::unique_ptr<grpc::ClientReader<GaugeResponse>> reader(
+      stub->GetAllGauges(&context, message));
+
+  GaugeResponse gauge_response;
+  long overall_qps = 0;
+  int idx = 0;
+  while (reader->Read(&gauge_response)) {
+    if (gauge_response.value_case() == GaugeResponse::kLongValue) {
+      gpr_log(GPR_INFO, "Gauge: %d (%s: %ld)", ++idx,
+              gauge_response.name().c_str(), gauge_response.long_value());
+      overall_qps += gauge_response.long_value();
+    } else {
+      gpr_log(GPR_INFO, "Gauge %s is not a long value", gauge_response.name().c_str());
+    }
+  }
+
+  gpr_log(GPR_INFO, "OVERALL: %ld", overall_qps);
+
+  const grpc::Status status = reader->Finish();
+  if (!status.ok()) {
+    gpr_log(GPR_ERROR, "Error in getting metrics from the client");
+  }
+}
+
+int main(int argc, char** argv) {
+  grpc::testing::InitTest(&argc, &argv, true);
+
+  // Make sure server_addresses flag is not empty
+  if (FLAGS_metrics_server_address.empty()) {
+    gpr_log(
+        GPR_ERROR,
+        "Cannot connect to the Metrics server. Please pass the address of the"
+        "metrics server to connect to via the 'metrics_server_address' flag");
+    return 1;
+  }
+
+  PrintMetrics(FLAGS_metrics_server_address);
+
+  return 0;
+}
diff --git a/test/cpp/interop/stress_interop_client.cc b/test/cpp/interop/stress_interop_client.cc
@@ -40,6 +40,7 @@
 #include <grpc++/create_channel.h>
 
 #include "test/cpp/interop/interop_client.h"
+#include "test/cpp/util/metrics_server.h"
 
 namespace grpc {
 namespace testing {
@@ -81,21 +82,19 @@ TestCaseType WeightedRandomTestSelector::GetNextTest() const {
 
 StressTestInteropClient::StressTestInteropClient(
     int test_id, const grpc::string& server_address,
+    std::shared_ptr<Channel> channel,
     const WeightedRandomTestSelector& test_selector, long test_duration_secs,
-    long sleep_duration_ms)
+    long sleep_duration_ms, long metrics_collection_interval_secs)
     : test_id_(test_id),
       server_address_(server_address),
+      channel_(channel),
+      interop_client_(new InteropClient(channel, false)),
       test_selector_(test_selector),
       test_duration_secs_(test_duration_secs),
-      sleep_duration_ms_(sleep_duration_ms) {
-  // TODO(sreek): This will change once we add support for other tests
-  // that won't work with InsecureChannelCredentials()
-  std::shared_ptr<Channel> channel(
-      CreateChannel(server_address, InsecureChannelCredentials()));
-  interop_client_.reset(new InteropClient(channel, false));
-}
+      sleep_duration_ms_(sleep_duration_ms),
+      metrics_collection_interval_secs_(metrics_collection_interval_secs) {}
 
-void StressTestInteropClient::MainLoop() {
+void StressTestInteropClient::MainLoop(std::shared_ptr<Gauge> qps_gauge) {
   gpr_log(GPR_INFO, "Running test %d. ServerAddr: %s", test_id_,
           server_address_.c_str());
 
@@ -104,21 +103,38 @@ void StressTestInteropClient::MainLoop() {
                    gpr_time_from_seconds(test_duration_secs_, GPR_TIMESPAN));
 
   gpr_timespec current_time = gpr_now(GPR_CLOCK_REALTIME);
+  gpr_timespec next_stat_collection_time = current_time;
+  gpr_timespec collection_interval =
+      gpr_time_from_seconds(metrics_collection_interval_secs_, GPR_TIMESPAN);
+  long num_calls_per_interval = 0;
+
   while (test_duration_secs_ < 0 ||
-         gpr_time_cmp(current_time, test_end_time) < 0) {
+         gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), test_end_time) < 0) {
     // Select the test case to execute based on the weights and execute it
     TestCaseType test_case = test_selector_.GetNextTest();
     gpr_log(GPR_INFO, "%d - Executing the test case %d", test_id_, test_case);
     RunTest(test_case);
 
+    num_calls_per_interval++;
+
+    // See if its time to collect stats yet
+    current_time = gpr_now(GPR_CLOCK_REALTIME);
+    if (gpr_time_cmp(next_stat_collection_time, current_time) < 0) {
+      qps_gauge->Set(num_calls_per_interval /
+                     metrics_collection_interval_secs_);
+
+      num_calls_per_interval = 0;
+      next_stat_collection_time =
+          gpr_time_add(current_time, collection_interval);
+    }
+
     // Sleep between successive calls if needed
     if (sleep_duration_ms_ > 0) {
-      gpr_timespec sleep_time = gpr_time_add(
-          current_time, gpr_time_from_millis(sleep_duration_ms_, GPR_TIMESPAN));
+      gpr_timespec sleep_time =
+          gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
+                       gpr_time_from_millis(sleep_duration_ms_, GPR_TIMESPAN));
       gpr_sleep_until(sleep_time);
     }
-
-    current_time = gpr_now(GPR_CLOCK_REALTIME);
   }
 }
 

diff --git a/test/cpp/interop/stress_interop_client.h b/test/cpp/interop/stress_interop_client.h
@@ -41,6 +41,7 @@
 #include <grpc++/create_channel.h>
 
 #include "test/cpp/interop/interop_client.h"
+#include "test/cpp/util/metrics_server.h"
 
 namespace grpc {
 namespace testing {
@@ -84,20 +85,26 @@ class WeightedRandomTestSelector {
 class StressTestInteropClient {
  public:
   StressTestInteropClient(int test_id, const grpc::string& server_address,
+                          std::shared_ptr<Channel> channel,
                           const WeightedRandomTestSelector& test_selector,
-                          long test_duration_secs, long sleep_duration_ms);
+                          long test_duration_secs, long sleep_duration_ms,
+                          long metrics_collection_interval_secs);
 
-  void MainLoop();  // The main function. Use this as the thread entry point.
+  // The main function. Use this as the thread entry point.
+  // qps_gauge is the Gauge to record the requests per second metric
+  void MainLoop(std::shared_ptr<Gauge> qps_gauge);
 
  private:
   void RunTest(TestCaseType test_case);
 
   int test_id_;
-  std::unique_ptr<InteropClient> interop_client_;
   const grpc::string& server_address_;
+  std::shared_ptr<Channel> channel_;
+  std::unique_ptr<InteropClient> interop_client_;
   const WeightedRandomTestSelector& test_selector_;
   long test_duration_secs_;
   long sleep_duration_ms_;
+  long metrics_collection_interval_secs_;
 };
 
 }  // namespace testing