[executor-benchmark] add progress bars

666lcz · Dec 23, 2021 · 6f5de36 · 6f5de36
1 parent 09b9119
commit 6f5de36
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 11 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/execution/executor-benchmark/Cargo.toml b/execution/executor-benchmark/Cargo.toml
@@ -12,6 +12,7 @@ edition = "2018"
 [dependencies]
 bcs = "0.1.2"
 criterion = "0.3.4"
+chrono = "0.4.19"
 indicatif = "0.15.0"
 itertools = { version = "0.10.0", default-features = false }
 jemallocator = { version = "0.3.2", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] }

diff --git a/execution/executor-benchmark/src/db_generator.rs b/execution/executor-benchmark/src/db_generator.rs
@@ -19,7 +19,6 @@ use executor::{
     db_bootstrapper::{generate_waypoint, maybe_bootstrap},
 };
 use executor_types::BlockExecutorTrait;
-use indicatif::{ProgressBar, ProgressStyle};
 use std::{
     fs,
     path::Path,
@@ -66,12 +65,6 @@ pub fn run(
     let (commit_sender, commit_receiver) = mpsc::sync_channel(3 /* bound */);
 
     // Set a progressing bar
-    let bar = Arc::new(ProgressBar::new(num_accounts as u64 * 2));
-    bar.set_style(
-        ProgressStyle::default_bar().template("[{elapsed}] {bar:100.cyan/blue} {percent}%"),
-    );
-    let exe_thread_bar = Arc::clone(&bar);
-
     // Spawn threads to run transaction generator, executor and committer separately.
     let gen_thread = std::thread::Builder::new()
         .name("txn_generator".to_string())
@@ -92,9 +85,7 @@ pub fn run(
                 Some(commit_sender),
             );
             while let Ok(transactions) = block_receiver.recv() {
-                let version_bump = transactions.len() as u64;
                 exe.execute_block(transactions);
-                exe_thread_bar.inc(version_bump);
             }
         })
         .expect("Failed to spawn transaction executor thread.");
@@ -115,8 +106,6 @@ pub fn run(
     // Do a sanity check on the sequence number to make sure all transactions are committed.
     generator.verify_sequence_number(db.as_ref());
 
-    bar.finish();
-
     let final_version = generator.version();
     // Write metadata
     generator.write_meta(&db_dir);

diff --git a/execution/executor-benchmark/src/transaction_generator.rs b/execution/executor-benchmark/src/transaction_generator.rs
@@ -1,6 +1,7 @@
 // Copyright (c) The Diem Core Contributors
 // SPDX-License-Identifier: Apache-2.0
 
+use chrono::Local;
 use diem_crypto::{
     ed25519::{Ed25519PrivateKey, Ed25519PublicKey},
     PrivateKey, SigningKey, Uniform,
@@ -21,6 +22,7 @@ use diem_types::{
         Version,
     },
 };
+use indicatif::{ProgressBar, ProgressStyle};
 use rand::{rngs::StdRng, SeedableRng};
 use serde::{Deserialize, Serialize};
 use std::{
@@ -36,6 +38,21 @@ use storage_interface::DbReader;
 const META_FILENAME: &str = "metadata.toml";
 const MAX_ACCOUNTS_INVOLVED_IN_P2P: usize = 1_000_000;
 
+fn get_progress_bar(num_accounts: usize) -> ProgressBar {
+    let bar = ProgressBar::new(num_accounts as u64);
+    bar.set_style(
+        ProgressStyle::default_bar()
+            .template("[{elapsed_precise}] {bar:100.cyan/blue} {percent}% ETA {eta_precise}"),
+    );
+    bar
+}
+
+macro_rules! now_fmt {
+    () => {
+        Local::now().format("%m-%d %H:%M:%S")
+    };
+}
+
 #[derive(Serialize, Deserialize)]
 #[serde(tag = "type", content = "args")]
 enum TestCase {
@@ -121,6 +138,8 @@ impl TransactionGenerator {
         let mut rng = StdRng::from_seed(seed);
 
         let mut accounts = Vec::with_capacity(num_accounts);
+        println!("[{}] Generating {} accounts.", now_fmt!(), num_accounts);
+        let bar = get_progress_bar(num_accounts);
         for _i in 0..num_accounts {
             let private_key = Ed25519PrivateKey::generate(&mut rng);
             let public_key = private_key.public_key();
@@ -132,7 +151,10 @@ impl TransactionGenerator {
                 sequence_number: 0,
             };
             accounts.push(account);
+            bar.inc(1);
         }
+        bar.finish();
+        println!("[{}] done.", now_fmt!());
 
         info!(
             num_accounts_generated = num_accounts,
@@ -201,6 +223,12 @@ impl TransactionGenerator {
         let tc_account = treasury_compliance_account_address();
         let mut txn_block = vec![];
 
+        println!(
+            "[{}] Generating {} account creation txns.",
+            now_fmt!(),
+            self.accounts_cache.len(),
+        );
+        let bar = get_progress_bar(self.accounts_cache.len());
         for (i, block) in self.accounts_cache.chunks(block_size).enumerate() {
             let mut transactions = Vec::with_capacity(block_size);
             for (j, account) in block.iter().enumerate() {
@@ -226,7 +254,10 @@ impl TransactionGenerator {
             } else {
                 txn_block.push(transactions);
             }
+            bar.inc(block_size as u64);
         }
+        bar.finish();
+        println!("[{}] done.", now_fmt!());
         txn_block
     }
 
@@ -239,6 +270,12 @@ impl TransactionGenerator {
         let testnet_dd_account = testnet_dd_account_address();
         let mut txn_block = vec![];
 
+        println!(
+            "[{}] Generating {} mint txns.",
+            now_fmt!(),
+            self.accounts_cache.len(),
+        );
+        let bar = get_progress_bar(self.accounts_cache.len());
         for (i, block) in self.accounts_cache.chunks(block_size).enumerate() {
             let mut transactions = Vec::with_capacity(block_size);
             for (j, account) in block.iter().enumerate() {
@@ -264,7 +301,10 @@ impl TransactionGenerator {
             } else {
                 txn_block.push(transactions);
             }
+            bar.inc(block.len() as u64)
         }
+        bar.finish();
+        println!("[{}] done.", now_fmt!());
         txn_block
     }
 
@@ -314,6 +354,12 @@ impl TransactionGenerator {
 
     /// Verifies the sequence numbers in storage match what we have locally.
     pub fn verify_sequence_number(&self, db: &dyn DbReader) {
+        println!(
+            "[{}] verify {} account sequence numbers.",
+            now_fmt!(),
+            self.accounts_cache.len(),
+        );
+        let bar = get_progress_bar(self.accounts_cache.len());
         for account in &self.accounts_cache {
             let address = account.address;
             let blob = db
@@ -322,7 +368,10 @@ impl TransactionGenerator {
                 .expect("Account must exist.");
             let account_resource = AccountResource::try_from(&blob).unwrap();
             assert_eq!(account_resource.sequence_number(), account.sequence_number);
+            bar.inc(1);
         }
+        bar.finish();
+        println!("[{}] done.", now_fmt!());
     }
 
     /// Drops the sender to notify the receiving end of the channel.