Merge branch 'master' into giles-common-version

paritytech · paritytech-processbot · Dec 5, 2022 · Oct 2, 2022 · Oct 5, 2022 · Oct 11, 2022
commit 2e9209622e712d93bd60b163097329eea1e75a14
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -21,7 +21,7 @@ version = "0.9.29"
 [dependencies]
 polkadot-cli = { path = "cli", features = [ "kusama-native", "westend-native", "rococo-native" ]  }
 color-eyre = { version = "0.6.1", default-features = false }
-parity-util-mem = { version = "0.11.0", default-features = false, features = ["jemalloc-global"] }
+parity-util-mem = { version = "0.12.0", default-features = false, features = ["jemalloc-global"] }
 
 [dev-dependencies]
 assert_cmd = "2.0.4"
@@ -89,8 +89,6 @@ members = [
 	"node/network/gossip-support",
 	"node/network/dispute-distribution",
 	"node/overseer",
-	"node/orchestra",
-	"node/orchestra/proc-macro",
 	"node/malus",
 	"node/primitives",
 	"node/service",
@@ -102,7 +100,6 @@ members = [
 	"node/gum",
 	"node/gum/proc-macro",
 	"node/metrics",
-	"node/metered-channel",
 	"node/test/client",
 	"node/test/performance-test",
 	"node/test/service",
@@ -134,9 +131,9 @@ maintenance = { status = "actively-developed" }
 #
 # This list is ordered alphabetically.
 [profile.dev.package]
-blake2b_simd = { opt-level = 3 }
 blake2 = { opt-level = 3 }
 blake2-rfc = { opt-level = 3 }
+blake2b_simd = { opt-level = 3 }
 chacha20poly1305 = { opt-level = 3 }
 cranelift-codegen = { opt-level = 3 }
 cranelift-wasm = { opt-level = 3 }
@@ -147,8 +144,8 @@ curve25519-dalek = { opt-level = 3 }
 ed25519-dalek = { opt-level = 3 }
 flate2 = { opt-level = 3 }
 futures-channel = { opt-level = 3 }
-hashbrown = { opt-level = 3 }
 hash-db = { opt-level = 3 }
+hashbrown = { opt-level = 3 }
 hmac = { opt-level = 3 }
 httparse = { opt-level = 3 }
 integer-sqrt = { opt-level = 3 }
@@ -160,8 +157,8 @@ libz-sys = { opt-level = 3 }
 mio = { opt-level = 3 }
 nalgebra = { opt-level = 3 }
 num-bigint = { opt-level = 3 }
-parking_lot_core = { opt-level = 3 }
 parking_lot = { opt-level = 3 }
+parking_lot_core = { opt-level = 3 }
 percent-encoding = { opt-level = 3 }
 primitive-types = { opt-level = 3 }
 reed-solomon-novelpoly = { opt-level = 3 }
@@ -171,6 +168,7 @@ sha2 = { opt-level = 3 }
 sha3 = { opt-level = 3 }
 smallvec = { opt-level = 3 }
 snow = { opt-level = 3 }
+substrate-bip39 = {opt-level = 3}
 twox-hash = { opt-level = 3 }
 uint = { opt-level = 3 }
 wasmi = { opt-level = 3 }

diff --git a/cli/Cargo.toml b/cli/Cargo.toml
@@ -56,7 +56,11 @@ cli = [
 	"polkadot-client",
 	"polkadot-node-core-pvf",
 ]
-runtime-benchmarks = ["service/runtime-benchmarks", "polkadot-node-metrics/runtime-benchmarks"]
+runtime-benchmarks = [
+	"service/runtime-benchmarks",
+	"polkadot-node-metrics/runtime-benchmarks",
+	"polkadot-performance-test?/runtime-benchmarks"
+]
 trie-memory-tracker = ["sp-trie/memory-tracker"]
 full-node = ["service/full-node"]
 try-runtime = ["service/try-runtime"]

diff --git a/core-primitives/Cargo.toml b/core-primitives/Cargo.toml
@@ -10,7 +10,7 @@ sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", d
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 scale-info = { version = "2.1.2", default-features = false, features = ["derive"] }
 parity-scale-codec = { version = "3.1.5", default-features = false, features = [ "derive" ] }
-parity-util-mem = { version = "0.11.0", default-features = false, optional = true }
+parity-util-mem = { version = "0.12.0", default-features = false, optional = true }
 
 [features]
 default = [ "std" ]

diff --git a/node/core/approval-voting/Cargo.toml b/node/core/approval-voting/Cargo.toml
@@ -10,10 +10,10 @@ futures-timer = "3.0.2"
 parity-scale-codec = { version = "3.1.5", default-features = false, features = ["bit-vec", "derive"] }
 gum = { package = "tracing-gum", path = "../../gum" }
 bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }
-lru = "0.7"
+lru = "0.8"
 merlin = "2.0"
 schnorrkel = "0.9.1"
-kvdb = "0.11.0"
+kvdb = "0.12.0"
 derive_more = "0.99.17"
 thiserror = "1.0.31"
 
@@ -40,5 +40,5 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 assert_matches = "1.4.0"
-kvdb-memorydb = "0.11.0"
+kvdb-memorydb = "0.12.0"
 test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }
diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs
@@ -1296,6 +1296,38 @@ pub(crate) mod tests {
 				}
 			);
 
+			// Caching of sesssions needs sessoion of first unfinalied block.
+			assert_matches!(
+				handle.recv().await,
+				AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(
+					s_tx,
+				)) => {
+					let _ = s_tx.send(Ok(header.number));
+				}
+			);
+
+			assert_matches!(
+				handle.recv().await,
+				AllMessages::ChainApi(ChainApiMessage::FinalizedBlockHash(
+					block_number,
+					s_tx,
+				)) => {
+					assert_eq!(block_number, header.number);
+					let _ = s_tx.send(Ok(Some(header.hash())));
+				}
+			);
+
+			assert_matches!(
+				handle.recv().await,
+				AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+					h,
+					RuntimeApiRequest::SessionIndexForChild(s_tx),
+				)) => {
+					assert_eq!(h, header.hash());
+					let _ = s_tx.send(Ok(session));
+				}
+			);
+
 			// determine_new_blocks exits early as the parent_hash is in the DB
 
 			assert_matches!(

diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs
@@ -70,6 +70,7 @@ use std::{
 	collections::{
 		btree_map::Entry as BTMEntry, hash_map::Entry as HMEntry, BTreeMap, HashMap, HashSet,
 	},
+	num::NonZeroUsize,
 	sync::Arc,
 	time::Duration,
 };
@@ -104,7 +105,11 @@ const APPROVAL_CHECKING_TIMEOUT: Duration = Duration::from_secs(120);
 /// Value rather arbitrarily: Should not be hit in practice, it exists to more easily diagnose dead
 /// lock issues for example.
 const WAIT_FOR_SIGS_TIMEOUT: Duration = Duration::from_millis(500);
-const APPROVAL_CACHE_SIZE: usize = 1024;
+const APPROVAL_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(1024) {
+	Some(cap) => cap,
+	None => panic!("Approval cache size must be non-zero."),
+};
+
 const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds.
 const APPROVAL_DELAY: Tick = 2;
 const LOG_TARGET: &str = "parachain::approval-voting";

diff --git a/node/core/approval-voting/src/tests.rs b/node/core/approval-voting/src/tests.rs
@@ -807,6 +807,37 @@ async fn import_block(
 		}
 	);
 
+	assert_matches!(
+		overseer_recv(overseer).await,
+		AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(
+			s_tx,
+		)) => {
+			let _ = s_tx.send(Ok(number));
+		}
+	);
+
+	assert_matches!(
+		overseer_recv(overseer).await,
+		AllMessages::ChainApi(ChainApiMessage::FinalizedBlockHash(
+			block_number,
+			s_tx,
+		)) => {
+			assert_eq!(block_number, number);
+			let _ = s_tx.send(Ok(Some(hashes[number as usize].0)));
+		}
+	);
+
+	assert_matches!(
+		overseer_recv(overseer).await,
+		AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+			h,
+			RuntimeApiRequest::SessionIndexForChild(s_tx),
+		)) => {
+			assert_eq!(h, hashes[number as usize].0);
+			let _ = s_tx.send(Ok(number.into()));
+		}
+	);
+
 	if !fork {
 		assert_matches!(
 			overseer_recv(overseer).await,

diff --git a/node/core/av-store/Cargo.toml b/node/core/av-store/Cargo.toml
@@ -7,7 +7,7 @@ edition.workspace = true
 [dependencies]
 futures = "0.3.21"
 futures-timer = "3.0.2"
-kvdb = "0.11.0"
+kvdb = "0.12.0"
 thiserror = "1.0.31"
 gum = { package = "tracing-gum", path = "../../gum" }
 bitvec = "1.0.0"
@@ -24,7 +24,7 @@ polkadot-node-primitives = { path = "../../primitives" }
 log = "0.4.17"
 env_logger = "0.9.0"
 assert_matches = "1.4.0"
-kvdb-memorydb = "0.11.0"
+kvdb-memorydb = "0.12.0"
 
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }

diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs
@@ -792,8 +792,9 @@ fn note_block_included(
 macro_rules! peek_num {
 	($iter:ident) => {
 		match $iter.peek() {
-			Some((k, _)) => decode_unfinalized_key(&k[..]).ok().map(|(b, _, _)| b),
-			None => None,
+			Some(Ok((k, _))) => Ok(decode_unfinalized_key(&k[..]).ok().map(|(b, _, _)| b)),
+			Some(Err(_)) => Err($iter.next().expect("peek returned Some(Err); qed").unwrap_err()),
+			None => Ok(None),
 		}
 	};
 }
@@ -819,10 +820,10 @@ async fn process_block_finalized<Context>(
 			let mut iter = subsystem
 				.db
 				.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
-				.take_while(|(k, _)| &k[..] < &end_prefix[..])
+				.take_while(|r| r.as_ref().map_or(true, |(k, _v)| &k[..] < &end_prefix[..]))
 				.peekable();
 
-			match peek_num!(iter) {
+			match peek_num!(iter)? {
 				None => break, // end of iterator.
 				Some(n) => n,
 			}
@@ -867,10 +868,10 @@ async fn process_block_finalized<Context>(
 		let iter = subsystem
 			.db
 			.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
-			.take_while(|(k, _)| &k[..] < &end_prefix[..])
+			.take_while(|r| r.as_ref().map_or(true, |(k, _v)| &k[..] < &end_prefix[..]))
 			.peekable();
 
-		let batch = load_all_at_finalized_height(iter, batch_num, batch_finalized_hash);
+		let batch = load_all_at_finalized_height(iter, batch_num, batch_finalized_hash)?;
 
 		// Now that we've iterated over the entire batch at this finalized height,
 		// update the meta.
@@ -890,22 +891,22 @@ async fn process_block_finalized<Context>(
 // loads all candidates at the finalized height and maps them to `true` if finalized
 // and `false` if unfinalized.
 fn load_all_at_finalized_height(
-	mut iter: std::iter::Peekable<impl Iterator<Item = (Box<[u8]>, Box<[u8]>)>>,
+	mut iter: std::iter::Peekable<impl Iterator<Item = io::Result<util::database::DBKeyValue>>>,
 	block_number: BlockNumber,
 	finalized_hash: Hash,
-) -> impl IntoIterator<Item = (CandidateHash, bool)> {
+) -> io::Result<impl IntoIterator<Item = (CandidateHash, bool)>> {
 	// maps candidate hashes to true if finalized, false otherwise.
 	let mut candidates = HashMap::new();
 
 	// Load all candidates that were included at this height.
 	loop {
-		match peek_num!(iter) {
+		match peek_num!(iter)? {
 			None => break,                         // end of iterator.
 			Some(n) if n != block_number => break, // end of batch.
 			_ => {},
 		}
 
-		let (k, _v) = iter.next().expect("`peek` used to check non-empty; qed");
+		let (k, _v) = iter.next().expect("`peek` used to check non-empty; qed")?;
 		let (_, block_hash, candidate_hash) =
 			decode_unfinalized_key(&k[..]).expect("`peek_num` checks validity of key; qed");
 
@@ -916,7 +917,7 @@ fn load_all_at_finalized_height(
 		}
 	}
 
-	candidates
+	Ok(candidates)
 }
 
 fn update_blocks_at_finalized_height(
@@ -1214,9 +1215,10 @@ fn prune_all(db: &Arc<dyn Database>, config: &Config, clock: &dyn Clock) -> Resu
 	let mut tx = DBTransaction::new();
 	let iter = db
 		.iter_with_prefix(config.col_meta, &range_start[..])
-		.take_while(|(k, _)| &k[..] < &range_end[..]);
+		.take_while(|r| r.as_ref().map_or(true, |(k, _v)| &k[..] < &range_end[..]));
 
-	for (k, _v) in iter {
+	for r in iter {
+		let (k, _v) = r?;
 		tx.delete(config.col_meta, &k[..]);
 
 		let (_, candidate_hash) = match decode_pruning_key(&k[..]) {

diff --git a/node/core/chain-selection/Cargo.toml b/node/core/chain-selection/Cargo.toml
@@ -13,7 +13,7 @@ polkadot-primitives = { path = "../../../primitives" }
 polkadot-node-primitives = { path = "../../primitives" }
 polkadot-node-subsystem = { path = "../../subsystem" }
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }
-kvdb = "0.11.0"
+kvdb = "0.12.0"
 thiserror = "1.0.31"
 parity-scale-codec = "3.1.5"
 
@@ -22,4 +22,4 @@ polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 parking_lot = "0.12.0"
 assert_matches = "1"
-kvdb-memorydb = "0.11.0"
+kvdb-memorydb = "0.12.0"
diff --git a/node/core/chain-selection/src/db_backend/v1.rs b/node/core/chain-selection/src/db_backend/v1.rs
@@ -235,16 +235,21 @@ impl Backend for DbBackend {
 			self.inner.iter_with_prefix(self.config.col_data, &STAGNANT_AT_PREFIX[..]);
 
 		let val = stagnant_at_iter
-			.filter_map(|(k, v)| {
-				match (decode_stagnant_at_key(&mut &k[..]), <Vec<_>>::decode(&mut &v[..]).ok()) {
-					(Some(at), Some(stagnant_at)) => Some((at, stagnant_at)),
-					_ => None,
-				}
+			.filter_map(|r| match r {
+				Ok((k, v)) =>
+					match (decode_stagnant_at_key(&mut &k[..]), <Vec<_>>::decode(&mut &v[..]).ok())
+					{
+						(Some(at), Some(stagnant_at)) => Some(Ok((at, stagnant_at))),
+						_ => None,
+					},
+				Err(e) => Some(Err(e)),
 			})
 			.enumerate()
-			.take_while(|(idx, (at, _))| *at <= up_to.into() && *idx < max_elements)
+			.take_while(|(idx, r)| {
+				r.as_ref().map_or(true, |(at, _)| *at <= up_to.into() && *idx < max_elements)
+			})
 			.map(|(_, v)| v)
-			.collect::<Vec<_>>();
+			.collect::<Result<Vec<_>, _>>()?;
 
 		Ok(val)
 	}
@@ -254,10 +259,13 @@ impl Backend for DbBackend {
 			self.inner.iter_with_prefix(self.config.col_data, &BLOCK_HEIGHT_PREFIX[..]);
 
 		let val = blocks_at_height_iter
-			.filter_map(|(k, _)| decode_block_height_key(&k[..]))
+			.filter_map(|r| match r {
+				Ok((k, _)) => decode_block_height_key(&k[..]).map(Ok),
+				Err(e) => Some(Err(e)),
+			})
 			.next();
 
-		Ok(val)
+		val.transpose().map_err(Error::from)
 	}
 
 	fn load_blocks_by_number(&self, number: BlockNumber) -> Result<Vec<Hash>, Error> {

diff --git a/node/core/dispute-coordinator/Cargo.toml b/node/core/dispute-coordinator/Cargo.toml
@@ -8,9 +8,9 @@ edition.workspace = true
 futures = "0.3.21"
 gum = { package = "tracing-gum", path = "../../gum" }
 parity-scale-codec = "3.1.5"
-kvdb = "0.11.0"
+kvdb = "0.12.0"
 thiserror = "1.0.31"
-lru = "0.7.7"
+lru = "0.8.0"
 fatality = "0.0.6"
 
 polkadot-primitives = { path = "../../../primitives" }
@@ -22,7 +22,7 @@ sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "maste
 
 
 [dev-dependencies]
-kvdb-memorydb = "0.11.0"
+kvdb-memorydb = "0.12.0"
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }