From 6f5de36a5f36056a509a74de68e28014dfd854f9 Mon Sep 17 00:00:00 2001 From: aldenhu Date: Fri, 17 Dec 2021 19:38:08 -0800 Subject: [PATCH] [executor-benchmark] add progress bars --- Cargo.lock | 1 + execution/executor-benchmark/Cargo.toml | 1 + .../executor-benchmark/src/db_generator.rs | 11 ----- .../src/transaction_generator.rs | 49 +++++++++++++++++++ 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd3c711def..e73dd2f5b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3094,6 +3094,7 @@ name = "executor-benchmark" version = "0.1.0" dependencies = [ "bcs", + "chrono", "criterion", "diem-config", "diem-crypto", diff --git a/execution/executor-benchmark/Cargo.toml b/execution/executor-benchmark/Cargo.toml index b4d4ce2af1..ec826fd59b 100644 --- a/execution/executor-benchmark/Cargo.toml +++ b/execution/executor-benchmark/Cargo.toml @@ -12,6 +12,7 @@ edition = "2018" [dependencies] bcs = "0.1.2" criterion = "0.3.4" +chrono = "0.4.19" indicatif = "0.15.0" itertools = { version = "0.10.0", default-features = false } jemallocator = { version = "0.3.2", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] } diff --git a/execution/executor-benchmark/src/db_generator.rs b/execution/executor-benchmark/src/db_generator.rs index 957cfbe423..4b08c883c7 100644 --- a/execution/executor-benchmark/src/db_generator.rs +++ b/execution/executor-benchmark/src/db_generator.rs @@ -19,7 +19,6 @@ use executor::{ db_bootstrapper::{generate_waypoint, maybe_bootstrap}, }; use executor_types::BlockExecutorTrait; -use indicatif::{ProgressBar, ProgressStyle}; use std::{ fs, path::Path, @@ -66,12 +65,6 @@ pub fn run( let (commit_sender, commit_receiver) = mpsc::sync_channel(3 /* bound */); // Set a progressing bar - let bar = Arc::new(ProgressBar::new(num_accounts as u64 * 2)); - bar.set_style( - ProgressStyle::default_bar().template("[{elapsed}] {bar:100.cyan/blue} {percent}%"), - ); - let exe_thread_bar = Arc::clone(&bar); - // Spawn threads to run transaction generator, executor and committer separately. let gen_thread = std::thread::Builder::new() .name("txn_generator".to_string()) @@ -92,9 +85,7 @@ pub fn run( Some(commit_sender), ); while let Ok(transactions) = block_receiver.recv() { - let version_bump = transactions.len() as u64; exe.execute_block(transactions); - exe_thread_bar.inc(version_bump); } }) .expect("Failed to spawn transaction executor thread."); @@ -115,8 +106,6 @@ pub fn run( // Do a sanity check on the sequence number to make sure all transactions are committed. generator.verify_sequence_number(db.as_ref()); - bar.finish(); - let final_version = generator.version(); // Write metadata generator.write_meta(&db_dir); diff --git a/execution/executor-benchmark/src/transaction_generator.rs b/execution/executor-benchmark/src/transaction_generator.rs index 1268366337..a6a8b9c89b 100644 --- a/execution/executor-benchmark/src/transaction_generator.rs +++ b/execution/executor-benchmark/src/transaction_generator.rs @@ -1,6 +1,7 @@ // Copyright (c) The Diem Core Contributors // SPDX-License-Identifier: Apache-2.0 +use chrono::Local; use diem_crypto::{ ed25519::{Ed25519PrivateKey, Ed25519PublicKey}, PrivateKey, SigningKey, Uniform, @@ -21,6 +22,7 @@ use diem_types::{ Version, }, }; +use indicatif::{ProgressBar, ProgressStyle}; use rand::{rngs::StdRng, SeedableRng}; use serde::{Deserialize, Serialize}; use std::{ @@ -36,6 +38,21 @@ use storage_interface::DbReader; const META_FILENAME: &str = "metadata.toml"; const MAX_ACCOUNTS_INVOLVED_IN_P2P: usize = 1_000_000; +fn get_progress_bar(num_accounts: usize) -> ProgressBar { + let bar = ProgressBar::new(num_accounts as u64); + bar.set_style( + ProgressStyle::default_bar() + .template("[{elapsed_precise}] {bar:100.cyan/blue} {percent}% ETA {eta_precise}"), + ); + bar +} + +macro_rules! now_fmt { + () => { + Local::now().format("%m-%d %H:%M:%S") + }; +} + #[derive(Serialize, Deserialize)] #[serde(tag = "type", content = "args")] enum TestCase { @@ -121,6 +138,8 @@ impl TransactionGenerator { let mut rng = StdRng::from_seed(seed); let mut accounts = Vec::with_capacity(num_accounts); + println!("[{}] Generating {} accounts.", now_fmt!(), num_accounts); + let bar = get_progress_bar(num_accounts); for _i in 0..num_accounts { let private_key = Ed25519PrivateKey::generate(&mut rng); let public_key = private_key.public_key(); @@ -132,7 +151,10 @@ impl TransactionGenerator { sequence_number: 0, }; accounts.push(account); + bar.inc(1); } + bar.finish(); + println!("[{}] done.", now_fmt!()); info!( num_accounts_generated = num_accounts, @@ -201,6 +223,12 @@ impl TransactionGenerator { let tc_account = treasury_compliance_account_address(); let mut txn_block = vec![]; + println!( + "[{}] Generating {} account creation txns.", + now_fmt!(), + self.accounts_cache.len(), + ); + let bar = get_progress_bar(self.accounts_cache.len()); for (i, block) in self.accounts_cache.chunks(block_size).enumerate() { let mut transactions = Vec::with_capacity(block_size); for (j, account) in block.iter().enumerate() { @@ -226,7 +254,10 @@ impl TransactionGenerator { } else { txn_block.push(transactions); } + bar.inc(block_size as u64); } + bar.finish(); + println!("[{}] done.", now_fmt!()); txn_block } @@ -239,6 +270,12 @@ impl TransactionGenerator { let testnet_dd_account = testnet_dd_account_address(); let mut txn_block = vec![]; + println!( + "[{}] Generating {} mint txns.", + now_fmt!(), + self.accounts_cache.len(), + ); + let bar = get_progress_bar(self.accounts_cache.len()); for (i, block) in self.accounts_cache.chunks(block_size).enumerate() { let mut transactions = Vec::with_capacity(block_size); for (j, account) in block.iter().enumerate() { @@ -264,7 +301,10 @@ impl TransactionGenerator { } else { txn_block.push(transactions); } + bar.inc(block.len() as u64) } + bar.finish(); + println!("[{}] done.", now_fmt!()); txn_block } @@ -314,6 +354,12 @@ impl TransactionGenerator { /// Verifies the sequence numbers in storage match what we have locally. pub fn verify_sequence_number(&self, db: &dyn DbReader) { + println!( + "[{}] verify {} account sequence numbers.", + now_fmt!(), + self.accounts_cache.len(), + ); + let bar = get_progress_bar(self.accounts_cache.len()); for account in &self.accounts_cache { let address = account.address; let blob = db @@ -322,7 +368,10 @@ impl TransactionGenerator { .expect("Account must exist."); let account_resource = AccountResource::try_from(&blob).unwrap(); assert_eq!(account_resource.sequence_number(), account.sequence_number); + bar.inc(1); } + bar.finish(); + println!("[{}] done.", now_fmt!()); } /// Drops the sender to notify the receiving end of the channel.