Merge branch 'master' of github.com:ySteinhart1/ORTOA into restructure-2

dsg-uwaterloo · Dec 29, 2023 · d1b1075 · d1b1075
2 parents af8a9bc + fd91634
commit d1b1075
Show file tree

Hide file tree

Showing 17 changed files with 2,536 additions and 0 deletions.
diff --git a/ortoa-lbl/.gitignore b/ortoa-lbl/.gitignore
@@ -0,0 +1,11 @@
+
+db/
+gen-cpp/
+
+client
+server
+
+*.o
+*.so
+
+*.data
diff --git a/ortoa-lbl/BS_thread_pool.hpp b/ortoa-lbl/BS_thread_pool.hpp
diff --git a/ortoa-lbl/Dockerfile b/ortoa-lbl/Dockerfile
@@ -0,0 +1,42 @@
+FROM ubuntu:latest
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+thrift-compiler \
+librocksdb-dev \
+libsodium-dev \
+libboost-all-dev \
+automake \
+libthrift-dev \
+g++ \
+git \ 
+libevent-dev \
+openjdk-11-jdk \
+vim \
+make \
+curl \
+python2 \
+maven
+
+# RUN sudo apt-get install ant
+# RUN wget http://www.apache.org/dyn/closer.cgi?path=/thrift/0.15.0/thrift-0.15.0.tar.gz
+# RUN tar -xvf thrift-0.15.0.tar.gz
+# RUN cd thrift-0.15.0
+# RUN ./bootstrap.sh \
+# ./configure \
+# sudo make \
+# sudo make install
+
+
+RUN ln -s /usr/bin/clang-9 /usr/bin/clang
+RUN ln -s /usr/bin/clang++-9 /usr/bin/clang++
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-arm64
+RUN git clone https://github.com/ySteinhart1/OpScure.git
+# RUN curl -O --location https://github.com/brianfrankcooper/YCSB/releases/download/0.5.0/ycsb-0.5.0.tar.gz
+# RUN tar xfvz ycsb-0.5.0.tar.gz
+RUN export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/share/boost-build/
+RUN export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/OpScure
+
+WORKDIR "/OpScure"
+RUN mkdir db
+# RUN make
diff --git a/ortoa-lbl/KV_RPC.thrift b/ortoa-lbl/KV_RPC.thrift
@@ -0,0 +1,13 @@
+
+struct Entry {
+  1: string keyName,
+  2: binary encryptedLabelsA,
+  3: binary encryptedLabelsB,
+  4: binary encryptedLabelsC,
+  5: binary encryptedLabelsD
+}
+
+service KV_RPC {
+  oneway void create(1:Entry entry),
+  binary access(1:Entry entry),
+}
diff --git a/ortoa-lbl/Makefile b/ortoa-lbl/Makefile
@@ -0,0 +1,69 @@
+
+ALL = server client benchmark encryption_benchmark proxy clients concurrent_benchmark
+
+all: $(ALL) constants.h
+
+CPPFLAGS = --std=c++17 -O3 -Wall -pthread -g
+
+
+client: gen-cpp/KV_RPC.o gen-cpp/KV_RPC_types.o client.o clientHelper.o
+	g++ $(CPPFLAGS) client.cpp clientHelper.o gen-cpp/KV_RPC.o gen-cpp/KV_RPC_types.o -lboost_filesystem -lboost_serialization -lthrift -lsodium -pthread -fPIC -o client
+
+clients: clients.cpp gen-cpp/Send_Op.o gen-cpp/Operation_types.o
+	g++ $(CPPFLAGS) $^ -lboost_filesystem -lboost_serialization -lthrift -lsodium -pthread -fPIC -o $@
+
+proxy: gen-cpp/KV_RPC.o gen-cpp/KV_RPC_types.o gen-cpp/Send_Op.o gen-cpp/Operation_types.o proxy.o clientHelper.o
+	g++ $(CPPFLAGS) $^ -lboost_filesystem -lboost_serialization -lthrift -lsodium -pthread -fPIC -o $@
+
+benchmark: benchmark.cpp clientHelper.o gen-cpp/KV_RPC.o gen-cpp/KV_RPC_types.o
+	g++ $(CPPFLAGS) benchmark.cpp clientHelper.o gen-cpp/KV_RPC.o gen-cpp/KV_RPC_types.o -lboost_filesystem -lboost_serialization -lthrift -lsodium -pthread -fPIC -o benchmark
+
+concurrent_benchmark: gen-cpp/KV_RPC.o gen-cpp/KV_RPC_types.o gen-cpp/Operation_types.o gen-cpp/Send_Op.o concurrent_benchmark.cpp clientHelper.o 
+	g++ $(CPPFLAGS) $^ -lboost_filesystem -lboost_serialization -lthrift -lsodium -pthread -fPIC -o $@
+
+
+encryption_benchmark: estimate_encryption.cpp clientHelper.o gen-cpp/KV_RPC.o gen-cpp/KV_RPC_types.o
+	g++ $(CPPFLAGS) estimate_encryption.cpp clientHelper.o gen-cpp/KV_RPC.o gen-cpp/KV_RPC_types.o -lboost_filesystem -lboost_serialization -lthrift -lsodium -pthread -fPIC -o encryption_benchmark
+
+clientHelper.o: clientHelper.h clientHelper.cpp
+	g++ $(CPPFLAGS) clientHelper.cpp -c -fPIC
+
+server: server.cpp gen-cpp/KV_RPC.o gen-cpp/KV_RPC_types.o
+	g++ $(CPPFLAGS) $^ -lthrift -lsodium -lrocksdb -pthread -fPIC -o $@
+
+gen-cpp/KV_RPC.o: gen-cpp/KV_RPC.h gen-cpp/KV_RPC.cpp
+	g++ $(CPPFLAGS) gen-cpp/KV_RPC.cpp -c -fPIC -o gen-cpp/KV_RPC.o
+
+gen-cpp/KV_RPC_types.o: gen-cpp/KV_RPC_types.h gen-cpp/KV_RPC_types.cpp
+	g++ $(CPPFLAGS) gen-cpp/KV_RPC_types.cpp -c -fPIC -o gen-cpp/KV_RPC_types.o
+
+
+gen-cpp/KV_RPC.h:
+	thrift -r --gen cpp KV_RPC.thrift
+
+gen-cpp/KV_RPC.cpp:
+	thrift -r --gen cpp KV_RPC.thrift
+
+gen-cpp/KV_RPC_types.h:
+	thrift -r --gen cpp KV_RPC.thrift
+
+gen-cpp/KV_RPC_types.cpp:
+	thrift -r --gen cpp KV_RPC.thrift
+
+gen-cpp/Operation_types.cpp:
+	thrift -r --gen cpp Operation.thrift
+
+gen-cpp/Operation_types.h:
+	thrift -r --gen cpp Operation.thrift
+
+gen-cpp/Send_Op.cpp:
+	thrift -r --gen cpp Operation.thrift
+
+gen-cpp/Send_Op.h:
+	thrift -r --gen cpp Operation.thrift
+
+clean:
+	rm $(ALL) OpScure.data *.o db/* gen-cpp/*
+
+cleandb:
+	rm db/* OpScure.data
diff --git a/ortoa-lbl/Operation.thrift b/ortoa-lbl/Operation.thrift
@@ -0,0 +1,15 @@
+enum OpType {
+  GET, 
+  PUT, 
+  EOD
+}
+
+struct Operation {
+  1: OpType op,
+  2: string key,
+  3: string value
+}
+
+service RPC {
+  binary access(1:Operation operation),
+}
diff --git a/ortoa-lbl/README.md b/ortoa-lbl/README.md
@@ -0,0 +1,46 @@
+# OpScure
+A system to obscure the operation used in KV Store queries.
+
+## Dependencies
+- Apache Thrift
+- RocksDB
+- LibSodium
+- Boost
+
+Install all:
+```
+sudo apt install thrift-compiler libthrift-dev librocksdb-dev libsodium-dev libboost-all-dev
+```
+
+# Configuring system
+You can configure many system parameters by modifying ```constants.h```. This file controls ports, db size, value size, and IP addresses. Note that you will have to recompile each executable after modifying the constants file.
+
+
+## Building
+```
+$ mkdir db
+$ make
+```
+## Server
+```
+./server
+```
+will start the server. The server will run until a value fetch fails or until you quit it.
+
+# clients.cpp
+
+```
+$ make clients
+$ ./clients
+```
+By default, this will run with 64 threads, each making 100 requests to the proxy with equal probability of a get and put request. These values can all be changed by editing the clients.cpp file.
+
+## CLI Client
+```
+$ ./client
+> PUT hello world
+> GET hello
+world
+> 
+```
+Ctrl+C cleans up the program, saves what needs to be persisted into `OpScure.data`, and exits. EXIT will also quit the program.
diff --git a/ortoa-lbl/benchmark.cpp b/ortoa-lbl/benchmark.cpp
@@ -0,0 +1,117 @@
+#include <set>
+#include <string>
+#include <iostream>
+#include <unordered_map>
+#include <numeric>
+#include <chrono>
+
+#include <thrift/protocol/TBinaryProtocol.h>
+#include <thrift/transport/TSocket.h>
+#include <thrift/transport/TTransportUtils.h>
+
+#include "gen-cpp/KV_RPC.h"
+#include "clientHelper.h"
+#include "constants.h"
+
+
+using namespace std;
+using namespace apache::thrift;
+using namespace apache::thrift::protocol;
+using namespace apache::thrift::transport;
+using namespace std::chrono;
+
+
+enum OperationType {
+  GET,
+  PUT
+};
+
+struct Operation {
+  OperationType type;
+  std::string key;
+  std::string value;
+};
+
+void signal_callback_handler(int signum) {
+   OpScureCleanup(DATA_FILE);
+   delete pool;
+   exit(signum);
+}
+
+Operation parseOperation() {
+  Operation op;
+  std::string tmp;
+  std::cin >> tmp;
+  op.type = (tmp == "PUT") ? PUT : GET;
+  std::cin >> op.key;
+  if(op.type == PUT)
+    std::cin >> op.value;
+  return op;
+}
+
+int main() {
+  OpScureSetup(DATA_FILE);
+  pool = new BS::thread_pool(HW_THREADS);
+  srand( (unsigned)time( NULL ) );
+
+
+  signal(SIGINT, signal_callback_handler);
+
+  std::shared_ptr<TTransport> socket(new TSocket(SERVER_IP, SERVER_PORT));
+  std::shared_ptr<TTransport> transport(new TBufferedTransport(socket));
+  std::shared_ptr<TProtocol> protocol(new TBinaryProtocol(transport));
+  KV_RPCClient client(protocol);
+
+  Operation op;
+
+  try {
+    transport->open();
+    float diff;
+    std::vector<float> put_times, get_times, put_times_encrypt, put_times_access, get_times_encrypt, get_times_access;
+    std::string key;
+    std::string value;
+    for(int i = 0; i < 100; i++) {
+        if(i % 5 == 0){
+          std::cout << i << std::endl;
+        }
+
+        key = to_string(rand()% KEY_MAX);
+        std::string labels;
+        auto start = high_resolution_clock::now();
+        Entry getEntry = constructGetEntry(key);
+        auto encrypt_done = high_resolution_clock::now();
+        client.access(labels, getEntry);
+        std::string val = readValueFromLabels(key, labels);
+        auto stop = high_resolution_clock::now();
+
+
+        get_times.push_back(duration_cast<microseconds>(stop - start).count());
+        get_times_access.push_back(duration_cast<microseconds>(stop - encrypt_done).count());
+        get_times_encrypt.push_back(duration_cast<microseconds>(encrypt_done - start).count());
+
+        start = high_resolution_clock::now();
+        std::string v = std::string(value);
+        Entry putEntry = constructPutEntry(key, v);
+        encrypt_done = high_resolution_clock::now();
+        valueSizes[key] = VALUE_SIZE;
+        client.access(labels, putEntry);
+        stop = high_resolution_clock::now();
+        put_times.push_back(duration_cast<microseconds>(stop - start).count());
+        put_times_access.push_back(duration_cast<microseconds>(stop - encrypt_done).count());
+        put_times_encrypt.push_back(duration_cast<microseconds>(encrypt_done - start).count());
+        // std::cout << get_times.back() << put_times.back()  << std::endl;
+      //std::cerr << (op.type ? "PUT" : "GET") << " " << op.key << " " << op.value << std::endl;
+    }    
+
+    transport->close();
+    std::cout << "get_times avg: " << 1.0 * std::accumulate(get_times.begin(), get_times.end(), 0.0) / get_times.size() << std::endl;
+    std::cout << "get_times_encrypt avg: " << 1.0 * std::accumulate(get_times_encrypt.begin(), get_times_encrypt.end(), 0.0) / get_times_encrypt.size() << std::endl;
+    std::cout << "get_times_access avg: " << 1.0 * std::accumulate(get_times_access.begin(), get_times_access.end(), 0.0) / get_times_access.size() << std::endl;
+    std::cout << "put_times avg: " << 1.0 * std::accumulate(put_times.begin(), put_times.end(), 0.0) / put_times.size() << std::endl;
+    std::cout << "put_times_encrypt avg: " << 1.0 * std::accumulate(put_times_encrypt.begin(), put_times_encrypt.end(), 0.0) / put_times_encrypt.size() << std::endl;
+    std::cout << "put_times_access avg: " << 1.0 * std::accumulate(put_times_access.begin(), put_times_access.end(), 0.0) / put_times_access.size() << std::endl;
+
+  } catch (TException& tx) {
+    cout << "ERROR: " << tx.what() << endl;
+  }
+}