Added C++ benchmark. (protocolbuffers#1525)

goldenbull · Sep 23, 2016 · a289d43 · a289d43
1 parent d9ff3ef
commit a289d43
Show file tree

Hide file tree

Showing 9 changed files with 266 additions and 1 deletion.
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "third_party/benchmark"]
+	path = third_party/benchmark
+	url = https://github.com/google/benchmark.git
diff --git a/.travis.yml b/.travis.yml
@@ -1,4 +1,5 @@
 sudo: required
+dist: trusty
 # Note: travis currently does not support listing more than one language so
 # this cheats and claims to only be cpp.  If they add multiple language
 # support, this should probably get updated to install steps and/or

diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am
@@ -16,7 +16,9 @@ benchmarks_protoc_outputs_proto2 =                             \
   benchmark_messages_proto2.pb.cc                              \
   benchmark_messages_proto2.pb.h
 
-bin_PROGRAMS = generate-datasets
+AM_CXXFLAGS = $(NO_OPT_CXXFLAGS) $(PROTOBUF_OPT_FLAG) -Wall -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare
+
+bin_PROGRAMS = generate-datasets cpp-benchmark
 
 generate_datasets_LDADD = $(top_srcdir)/src/libprotobuf.la
 generate_datasets_SOURCES = generate_datasets.cc
@@ -30,6 +32,13 @@ nodist_generate_datasets_SOURCES =                             \
 # See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually
 generate_datasets-generate_datasets.$(OBJEXT): benchmarks.pb.h
 
+cpp_benchmark_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a
+cpp_benchmark_SOURCES = cpp_benchmark.cc
+cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) -I$(top_srcdir)/third_party/benchmark/include
+nodist_cpp_benchmark_SOURCES =                             \
+  $(benchmarks_protoc_outputs)                                 \
+  $(benchmarks_protoc_outputs_proto2)
+
 $(benchmarks_protoc_outputs): protoc_middleman
 $(benchmarks_protoc_outputs_proto2): protoc_middleman2
 

diff --git a/benchmarks/benchmark_messages_proto2.proto b/benchmarks/benchmark_messages_proto2.proto
@@ -8,6 +8,8 @@ option java_package = "com.google.protobuf.benchmarks";
 // This is the default, but we specify it here explicitly.
 option optimize_for = SPEED;
 
+option cc_enable_arenas = true;
+
 message GoogleMessage1 {
   required string field1 = 1;
   optional string field9 = 9;

diff --git a/benchmarks/benchmark_messages_proto3.proto b/benchmarks/benchmark_messages_proto3.proto
@@ -8,6 +8,8 @@ option java_package = "com.google.protobuf.benchmarks";
 // This is the default, but we specify it here explicitly.
 option optimize_for = SPEED;
 
+option cc_enable_arenas = true;
+
 message GoogleMessage1 {
   string field1 = 1;
   string field9 = 9;

diff --git a/benchmarks/cpp_benchmark.cc b/benchmarks/cpp_benchmark.cc
@@ -0,0 +1,242 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <glob.h>
+#include <iostream>
+#include <fstream>
+#include "benchmark/benchmark_api.h"
+#include "benchmarks.pb.h"
+#include "benchmark_messages_proto2.pb.h"
+#include "benchmark_messages_proto3.pb.h"
+
+#define PREFIX "dataset."
+#define SUFFIX ".pb"
+
+using benchmarks::BenchmarkDataset;
+using google::protobuf::Arena;
+using google::protobuf::Descriptor;
+using google::protobuf::DescriptorPool;
+using google::protobuf::Message;
+using google::protobuf::MessageFactory;
+
+class Fixture : public benchmark::Fixture {
+ public:
+  Fixture(const BenchmarkDataset& dataset, const std::string& suffix) {
+    for (int i = 0; i < dataset.payload_size(); i++) {
+      payloads_.push_back(dataset.payload(i));
+    }
+
+    const Descriptor* d =
+        DescriptorPool::generated_pool()->FindMessageTypeByName(
+            dataset.message_name());
+
+    if (!d) {
+      std::cerr << "Couldn't find message named '" << dataset.message_name()
+                << "\n";
+    }
+
+    prototype_ = MessageFactory::generated_factory()->GetPrototype(d);
+    SetName((dataset.name() + suffix).c_str());
+  }
+
+ protected:
+  std::vector<std::string> payloads_;
+  const Message* prototype_;
+};
+
+class WrappingCounter {
+ public:
+  WrappingCounter(size_t limit) : value_(0), limit_(limit) {}
+
+  size_t Next() {
+    size_t ret = value_;
+    if (++value_ == limit_) {
+      value_ = 0;
+    }
+    return ret;
+  }
+
+ private:
+  size_t value_;
+  size_t limit_;
+};
+
+template <class T>
+class ParseNewFixture : public Fixture {
+ public:
+  ParseNewFixture(const BenchmarkDataset& dataset)
+      : Fixture(dataset, "_parse_new") {}
+
+  virtual void BenchmarkCase(benchmark::State& state) {
+    WrappingCounter i(payloads_.size());
+    size_t total = 0;
+
+    while (state.KeepRunning()) {
+      T m;
+      const std::string& payload = payloads_[i.Next()];
+      total += payload.size();
+      m.ParseFromString(payload);
+    }
+
+    state.SetBytesProcessed(total);
+  }
+};
+
+template <class T>
+class ParseNewArenaFixture : public Fixture {
+ public:
+  ParseNewArenaFixture(const BenchmarkDataset& dataset)
+      : Fixture(dataset, "_parse_newarena") {}
+
+  virtual void BenchmarkCase(benchmark::State& state) {
+    WrappingCounter i(payloads_.size());
+    size_t total = 0;
+
+    while (state.KeepRunning()) {
+      Arena arena;
+      Message* m = Arena::CreateMessage<T>(&arena);
+      const std::string& payload = payloads_[i.Next()];
+      total += payload.size();
+      m->ParseFromString(payload);
+    }
+
+    state.SetBytesProcessed(total);
+  }
+};
+
+template <class T>
+class ParseReuseFixture : public Fixture {
+ public:
+  ParseReuseFixture(const BenchmarkDataset& dataset)
+      : Fixture(dataset, "_parse_reuse") {}
+
+  virtual void BenchmarkCase(benchmark::State& state) {
+    T m;
+    WrappingCounter i(payloads_.size());
+    size_t total = 0;
+
+    while (state.KeepRunning()) {
+      const std::string& payload = payloads_[i.Next()];
+      total += payload.size();
+      m.ParseFromString(payload);
+    }
+
+    state.SetBytesProcessed(total);
+  }
+};
+
+template <class T>
+class SerializeFixture : public Fixture {
+ public:
+  SerializeFixture(const BenchmarkDataset& dataset)
+      : Fixture(dataset, "_serialize") {
+    for (size_t i = 0; i < payloads_.size(); i++) {
+      message_.push_back(new T);
+      message_.back()->ParseFromString(payloads_[i]);
+    }
+  }
+
+  ~SerializeFixture() {
+    for (size_t i = 0; i < message_.size(); i++) {
+      delete message_[i];
+    }
+  }
+
+  virtual void BenchmarkCase(benchmark::State& state) {
+    size_t total = 0;
+    std::string str;
+    WrappingCounter i(payloads_.size());
+
+    while (state.KeepRunning()) {
+      str.clear();
+      message_[i.Next()]->SerializeToString(&str);
+      total += str.size();
+    }
+
+    state.SetBytesProcessed(total);
+  }
+
+ private:
+  std::vector<T*> message_;
+};
+
+std::string ReadFile(const std::string& name) {
+  std::ifstream file(name.c_str());
+  GOOGLE_CHECK(file.is_open()) << "Couldn't find file '" << name <<
+                                  "', please make sure you are running "
+                                  "this command from the benchmarks/ "
+                                  "directory.\n";
+  return std::string((std::istreambuf_iterator<char>(file)),
+                     std::istreambuf_iterator<char>());
+}
+
+template <class T>
+void RegisterBenchmarksForType(const BenchmarkDataset& dataset) {
+  ::benchmark::internal::RegisterBenchmarkInternal(
+      new ParseNewFixture<T>(dataset));
+  ::benchmark::internal::RegisterBenchmarkInternal(
+      new ParseReuseFixture<T>(dataset));
+  ::benchmark::internal::RegisterBenchmarkInternal(
+      new ParseNewArenaFixture<T>(dataset));
+  ::benchmark::internal::RegisterBenchmarkInternal(
+      new SerializeFixture<T>(dataset));
+}
+
+void RegisterBenchmarks(const std::string& dataset_bytes) {
+  BenchmarkDataset dataset;
+  GOOGLE_CHECK(dataset.ParseFromString(dataset_bytes));
+
+  if (dataset.message_name() == "benchmarks.proto3.GoogleMessage1") {
+    RegisterBenchmarksForType<benchmarks::proto3::GoogleMessage1>(dataset);
+  } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage1") {
+    RegisterBenchmarksForType<benchmarks::proto2::GoogleMessage1>(dataset);
+  } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage2") {
+    RegisterBenchmarksForType<benchmarks::proto2::GoogleMessage2>(dataset);
+  } else {
+    std::cerr << "Unknown message type: " << dataset.message_name();
+    exit(1);
+  }
+}
+
+int main(int argc, char *argv[]) {
+  glob_t glob_result;
+  if (glob("dataset.*.pb", 0, NULL, &glob_result) != 0) {
+    fprintf(stderr, "No dataset files found.\n");
+    return 1;
+  }
+
+  for (size_t i = 0; i < glob_result.gl_pathc; i++) {
+    fprintf(stderr, "Found input dataset: %s\n", glob_result.gl_pathv[i]);
+    RegisterBenchmarks(ReadFile(glob_result.gl_pathv[i]));
+  }
+
+  ::benchmark::Initialize(&argc, argv);
+  ::benchmark::RunSpecifiedBenchmarks();
+}
diff --git a/jenkins/docker/Dockerfile b/jenkins/docker/Dockerfile
@@ -71,6 +71,8 @@ RUN apt-get update && apt-get install -y \
   python3.4-dev \
   # -- For Ruby --
   ruby \
+  # -- For C++ benchmarks --
+  cmake \
   && apt-get clean
 
 ##################

diff --git a/tests.sh b/tests.sh
@@ -38,6 +38,9 @@ build_cpp() {
   cd conformance && make test_cpp && cd ..
 
   # Verify benchmarking code can build successfully.
+  git submodule init
+  git submodule update
+  cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
   cd benchmarks && make && ./generate-datasets && cd ..
 }
 

diff --git a/third_party/benchmark b/third_party/benchmark