[executor-benchmark] adjustments

1. create-db pipelines execution and commission 2. up to 1M accounts involved in p2p test 3. shorten commit queue length, 50->3, to save some memory and make it more realistic (less SMT nodes cached in mem); for db-create, also shorten the execution queue 50->3, there's no point buffering too much and it makes the progress bar I'll introduce later too not accurate.
666lcz · Dec 23, 2021 · 09b9119 · 09b9119
1 parent 42df9cd
commit 09b9119
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 5 deletions.
diff --git a/execution/executor-benchmark/src/db_generator.rs b/execution/executor-benchmark/src/db_generator.rs
@@ -3,6 +3,7 @@
 
 use crate::{
     transaction_executor::TransactionExecutor, transaction_generator::TransactionGenerator,
+    TransactionCommitter,
 };
 use diem_config::{config::RocksdbConfig, utils::get_genesis_txn};
 use diem_jellyfish_merkle::metrics::{
@@ -59,8 +60,10 @@ pub fn run(
     maybe_bootstrap::<DiemVM>(&db_rw, get_genesis_txn(&config).unwrap(), waypoint).unwrap();
 
     let executor = Arc::new(BlockExecutor::new(db_rw));
+    let executor_2 = executor.clone();
     let genesis_block_id = executor.committed_block_id();
-    let (block_sender, block_receiver) = mpsc::sync_channel(50 /* bound */);
+    let (block_sender, block_receiver) = mpsc::sync_channel(3 /* bound */);
+    let (commit_sender, commit_receiver) = mpsc::sync_channel(3 /* bound */);
 
     // Set a progressing bar
     let bar = Arc::new(ProgressBar::new(num_accounts as u64 * 2));
@@ -69,7 +72,7 @@ pub fn run(
     );
     let exe_thread_bar = Arc::clone(&bar);
 
-    // Spawn two threads to run transaction generator and executor separately.
+    // Spawn threads to run transaction generator, executor and committer separately.
     let gen_thread = std::thread::Builder::new()
         .name("txn_generator".to_string())
         .spawn(move || {
@@ -86,7 +89,7 @@ pub fn run(
                 executor,
                 genesis_block_id,
                 0, /* start_verison */
-                None,
+                Some(commit_sender),
             );
             while let Ok(transactions) = block_receiver.recv() {
                 let version_bump = transactions.len() as u64;
@@ -95,12 +98,20 @@ pub fn run(
             }
         })
         .expect("Failed to spawn transaction executor thread.");
+    let commit_thread = std::thread::Builder::new()
+        .name("txn_committer".to_string())
+        .spawn(move || {
+            let mut committer = TransactionCommitter::new(executor_2, 0, commit_receiver);
+            committer.run();
+        })
+        .expect("Failed to spawn transaction committer thread.");
 
     // Wait for generator to finish.
     let mut generator = gen_thread.join().unwrap();
     generator.drop_sender();
     // Wait until all transactions are committed.
     exe_thread.join().unwrap();
+    commit_thread.join().unwrap();
     // Do a sanity check on the sequence number to make sure all transactions are committed.
     generator.verify_sequence_number(db.as_ref());
 

diff --git a/execution/executor-benchmark/src/lib.rs b/execution/executor-benchmark/src/lib.rs
@@ -75,7 +75,7 @@ pub fn run_benchmark(
     let executor_2 = executor_1.clone();
 
     let (block_sender, block_receiver) = mpsc::sync_channel(50 /* bound */);
-    let (commit_sender, commit_receiver) = mpsc::sync_channel(50 /* bound */);
+    let (commit_sender, commit_receiver) = mpsc::sync_channel(3 /* bound */);
 
     let mut generator =
         TransactionGenerator::new_with_metafile(genesis_key, block_sender, source_dir);

diff --git a/execution/executor-benchmark/src/transaction_generator.rs b/execution/executor-benchmark/src/transaction_generator.rs
@@ -34,7 +34,7 @@ use std::{
 use storage_interface::DbReader;
 
 const META_FILENAME: &str = "metadata.toml";
-const MAX_ACCOUNTS_INVOLVED_IN_P2P: usize = 100_000;
+const MAX_ACCOUNTS_INVOLVED_IN_P2P: usize = 1_000_000;
 
 #[derive(Serialize, Deserialize)]
 #[serde(tag = "type", content = "args")]