Add the benchmark test for vineyard clients. (#1966)

Fixes #1946 Signed-off-by: Ye Cao <caoye.cao@alibaba-inc.com>
v6d-io · Aug 9, 2024 · 3a2364b · 3a2364b
1 parent 87c8c02
commit 3a2364b
Show file tree

Hide file tree

Showing 7 changed files with 908 additions and 2 deletions.
diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
@@ -1,3 +1,4 @@
 if(BUILD_VINEYARD_MALLOC)
     add_subdirectory(alloc_test)
+    add_subdirectory(blob_test)
 endif()
diff --git a/benchmark/alloc_test/CMakeLists.txt b/benchmark/alloc_test/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(BENCH_ALLOCATOR_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/bench_allocator.cpp)
+set(BENCH_ALLOCATOR_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/bench_allocator.cc)
 
 macro(add_benchmark target)
     if(BUILD_VINEYARD_BENCHMARKS_ALL)

diff --git a/benchmark/alloc_test/README.md b/benchmark/alloc_test/README.md
@@ -9,7 +9,7 @@ Referred from <https://github.com/daanx/mimalloc-bench/blob/master/bench/alloc-t
 Configure with the following arguments when building vineyard:
 
 ```bash
-cmake .. -DBUILD_VINEYARD_MALLOC=ON -DBUILD_VINEYARD_BENCHMARK=ON
+cmake .. -DBUILD_VINEYARD_MALLOC=ON -DBUILD_VINEYARD_BENCHMARKS=ON
 ```
 
 Then make the following targets:

diff --git a/benchmark/alloc_test/bench_allocator.cpp → benchmark/alloc_test/bench_allocator.cc b/benchmark/alloc_test/bench_allocator.cpp → benchmark/alloc_test/bench_allocator.cc
@@ -35,6 +35,7 @@
 
 #include <time.h>
 #include <chrono>
+#include <cstring>
 #include <fstream>
 #include <iostream>
 #include <memory>

diff --git a/benchmark/blob_test/CMakeLists.txt b/benchmark/blob_test/CMakeLists.txt
@@ -0,0 +1,15 @@
+set(BENCH_BLOB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/blob_test.cc)
+
+if(BUILD_VINEYARD_BENCHMARKS_ALL)
+    set(blob_benchmark_options "")
+else()
+    set(blob_benchmark_options "EXCLUDE_FROM_ALL")
+endif()
+
+add_executable(blob_benchmark ${blob_benchmark_options} ${BENCH_BLOB_SRCS})
+
+target_include_directories(blob_benchmark PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+
+target_link_libraries(blob_benchmark PRIVATE vineyard_client)
+
+add_dependencies(vineyard_benchmarks blob_benchmark)
diff --git a/benchmark/blob_test/README.md b/benchmark/blob_test/README.md
@@ -0,0 +1,62 @@
+# Blob benchmark test
+
+In the blob benchmark test, we will focus on the performance of the basic blob
+operations: 
+
+- `PutBlob`: Put data into local vineyard. It contains two steps: first, create a blob in vineyard, then copy the data into the blob.
+- `GetBlob`: Get data from local vineyard.
+- `PutBlobs`: Put multiple blobs into local vineyard.
+- `GetBlobs`: Get multiple blobs from local vineyard.
+- `PutRemoteBlob`: Put data into remote vineyard. Unlike `PutBlob`, the data is prepared beforehand and then copied into a Vineyard blob. Therefore, this operation avoids manual memory copying.
+- `GetRemoteBlob`: Get data from remote vineyard.
+- `PutRemoteBlobs`: Put multiple blobs into remote vineyard.
+- `GetRemoteBlobs`: Get multiple blobs from remote vineyard.
+
+Also, the performance is measured in terms of **throughput** and **latency**.
+
+## Build the benchmark
+
+Configure with the following arguments when building vineyard:
+
+```bash
+cmake .. -DBUILD_VINEYARD_BENCHMARKS=ON
+```
+
+Then make the following targets:
+
+```bash
+make vineyard_benchmarks -j
+```
+
+The artifacts will be placed under the `${CMAKE_BINARY_DIR}/bin/` directory:
+
+```bash
+./bin/blob_benchmark
+```
+
+## Run the benchmark with customized parameters
+
+**Important** Before running the benchmark, you need to start the vineyard server first. You could refer to the [launching vineyard server guide](https://v6d.io/notes/getting-started.html#launching-vineyard-server) for more information.
+
+After that, you could get the help information by running the following command:
+
+```bash
+./bin/blob_benchmark --help
+Usage: ./bin/blob_benchmark [OPTIONS]
+Options:
+  -h, --help                     Show this help message and exit
+  -i, --ipc_socket=IPC_SOCKET    Specify the IPC socket path (required)
+  -r, --rpc_endpoint=RPC_ENDPOINT  Specify the RPC endpoint (required)
+  -d, --rdma_endpoint=RDMA_ENDPOINT  Specify the RDMA endpoint (required)
+  -c, --clients_num=NUM          Number of clients (required)
+  -s, --data_size=SIZE           Data size (e.g., 1KB, 1MB) (required)
+  -n, --requests_num=NUM         Number of requests (required)
+  -t, --num_threads=NUM          Number of threads (required)
+  -o, --operation=TYPE           Operation type (put_blob, get_blob, put_blobs, get_blobs, put_remote_blob, get_remote_blob, put_remote_blobs, get_remote_blobs) (required)
+```
+
+For example, you could run the following command to test the performance of `PutBlob`:
+
+```bash
+./bin/blob_benchmark -i /tmp/vineyard.sock -r "127.0.0.1:9600" -d "" -c 50 -s 8MB -n 1000 -t 10 -o "put_blob"
+```