From d998b12e7f8edeccb8cf64c84f4c0474a6b2ea61 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 15 Jan 2021 13:19:35 +0000 Subject: [PATCH 01/86] Base features and traits. --- Cargo.lock | 11 + Cargo.toml | 1 + primitives/election-providers/Cargo.toml | 33 ++ primitives/election-providers/src/lib.rs | 239 +++++++++++++ primitives/election-providers/src/onchain.rs | 168 +++++++++ .../npos-elections/compact/src/assignment.rs | 54 +-- primitives/npos-elections/compact/src/lib.rs | 123 ++++--- .../fuzzer/src/phragmen_balancing.rs | 23 +- .../fuzzer/src/phragmms_balancing.rs | 23 +- .../npos-elections/fuzzer/src/reduce.rs | 9 +- primitives/npos-elections/src/helpers.rs | 23 +- primitives/npos-elections/src/lib.rs | 324 ++++++++++++------ primitives/npos-elections/src/mock.rs | 12 +- primitives/npos-elections/src/phragmen.rs | 46 ++- primitives/npos-elections/src/phragmms.rs | 7 +- primitives/npos-elections/src/tests.rs | 99 ++++-- 16 files changed, 908 insertions(+), 287 deletions(-) create mode 100644 primitives/election-providers/Cargo.toml create mode 100644 primitives/election-providers/src/lib.rs create mode 100644 primitives/election-providers/src/onchain.rs diff --git a/Cargo.lock b/Cargo.lock index c42127aead43c..05f4896e071ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8357,6 +8357,17 @@ dependencies = [ "syn", ] +[[package]] +name = "sp-election-providers" +version = "2.0.0" +dependencies = [ + "parity-scale-codec", + "sp-arithmetic", + "sp-npos-elections", + "sp-runtime", + "sp-std", +] + [[package]] name = "sp-externalities" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index 12e79490ef6b0..1754f896c8846 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -139,6 +139,7 @@ members = [ "primitives/database", "primitives/debug-derive", "primitives/externalities", + "primitives/election-providers", "primitives/finality-grandpa", "primitives/inherents", "primitives/io", diff --git a/primitives/election-providers/Cargo.toml b/primitives/election-providers/Cargo.toml new file mode 100644 index 0000000000000..65ca0e400958e --- /dev/null +++ b/primitives/election-providers/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "sp-election-providers" +version = "2.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Primitive election providers" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "1.3.4", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc6", default-features = false, path = "../std" } +sp-arithmetic = { version = "2.0.0-rc6", default-features = false, path = "../arithmetic" } +sp-npos-elections = { version = "2.0.0-rc6", default-features = false, path = "../npos-elections" } + +[dev-dependencies] +sp-npos-elections = { version = "2.0.0-rc6", path = "../npos-elections" } +sp-runtime = { version = "2.0.0-rc6", path = "../runtime" } + +[features] +default = ["std"] +runtime-benchmarks = [] +std = [ + "codec/std", + "sp-std/std", + "sp-npos-elections/std", + "sp-arithmetic/std", +] diff --git a/primitives/election-providers/src/lib.rs b/primitives/election-providers/src/lib.rs new file mode 100644 index 0000000000000..69261920be9a2 --- /dev/null +++ b/primitives/election-providers/src/lib.rs @@ -0,0 +1,239 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Primitive traits for providing election functionality. +//! +//! This crate provides two traits that could interact to enable extensible election functionality +//! within FRAME pallets. +//! +//! Something that will provide the functionality of election will implement [`ElectionProvider`], +//! whilst needing an associated [`ElectionProvider::DataProvider`], which needs to be fulfilled by +//! an entity implementing [`ElectionDataProvider`]. Most often, *the data provider is* the receiver +//! of the election, resulting in a diagram as below: +//! +//! ```ignore +//! ElectionDataProvider +//! <------------------------------------------+ +//! | | +//! v | +//! +-----+----+ +------+---+ +//! | | | | +//! pallet-do-election | | | | pallet-needs-election +//! | | | | +//! | | | | +//! +-----+----+ +------+---+ +//! | ^ +//! | | +//! +------------------------------------------+ +//! ElectionProvider +//! ``` +//! +//! > It could also be possible that a third party pallet (C), provides the data of election to an +//! > election provider (B), which then passes the election result to another pallet (A). +//! +//! ## Election Types +//! +//! Typically, two types of elections exist: +//! +//! 1. **Stateless**: Election data is provided, and the election result is immediately ready. +//! 2. **Stateful**: Election data is is queried ahead of time, and the election result might be +//! ready some number of blocks in the future. +//! +//! To accommodate both type of elections in one trait, the traits lean toward **stateful +//! election**, as it is more general than the stateless. This is why [`ElectionProvider::elect`] +//! has no parameters. All value and type parameter must be provided by the [`ElectionDataProvider`] +//! trait, even if the election happens immediately. +//! +//! ## Election Data +//! +//! The data associated with an election, essentially what the [`ElectionDataProvider`] must convey +//! is as follows: +//! +//! 1. A list of voters, with their stake. +//! 2. A list of targets (i.e. _candidates_). +//! 3. A number of desired targets to be elected (i.e. _winners_) +//! +//! In addition to that, the [`ElectionDataProvider`] must also hint [`ElectionProvider`] at when +//! the next election might happen ([`ElectionDataProvider::next_election_prediction`]). A stateless +//! election provider would probably ignore this. A stateful election provider can use this to +//! prepare the election result in advance. +//! +//! Nonetheless, an [`ElectionProvider`] shan't rely on this and should preferably provide some +//! means of fallback election as well, in case the `elect` was called immaturely early. +//! +//! ## Example +//! +//! ```rust +//! # use sp_election_providers::*; +//! # use sp_npos_elections::{Support, Assignment}; +//! +//! type AccountId = u64; +//! type Balance = u64; +//! type BlockNumber = u32; +//! +//! mod data_provider { +//! use super::*; +//! +//! pub trait Config: Sized { +//! type ElectionProvider: ElectionProvider< +//! AccountId, +//! BlockNumber, +//! DataProvider = Module, +//! >; +//! } +//! +//! pub struct Module(std::marker::PhantomData); +//! +//! impl ElectionDataProvider for Module { +//! fn desired_targets() -> u32 { +//! 1 +//! } +//! fn voters() -> Vec<(AccountId, VoteWeight, Vec)> { +//! Default::default() +//! } +//! fn targets() -> Vec { +//! vec![10, 20, 30] +//! } +//! fn next_election_prediction(now: BlockNumber) -> BlockNumber { +//! 0 +//! } +//! } +//! } +//! +//! +//! mod generic_election_provider { +//! use super::*; +//! +//! pub struct GenericElectionProvider(std::marker::PhantomData); +//! +//! pub trait Config { +//! type DataProvider: ElectionDataProvider; +//! } +//! +//! impl ElectionProvider for GenericElectionProvider { +//! type Error = (); +//! type DataProvider = T::DataProvider; +//! +//! fn elect() -> Result, Self::Error> { +//! Self::DataProvider::targets() +//! .first() +//! .map(|winner| vec![(*winner, Support::default())]) +//! .ok_or(()) +//! } +//! } +//! } +//! +//! mod runtime { +//! use super::generic_election_provider; +//! use super::data_provider; +//! use super::AccountId; +//! +//! struct Runtime; +//! impl generic_election_provider::Config for Runtime { +//! type DataProvider = data_provider::Module; +//! } +//! +//! impl data_provider::Config for Runtime { +//! type ElectionProvider = generic_election_provider::GenericElectionProvider; +//! } +//! +//! } +//! +//! # fn main() {} +//! ``` + +#![cfg_attr(not(feature = "std"), no_std)] + +pub mod onchain; +use sp_std::prelude::*; + +/// Re-export some type as they are used in the interface. +pub use sp_arithmetic::PerThing; +pub use sp_npos_elections::{Assignment, ExtendedBalance, PerThing128, Supports, VoteWeight}; + +/// Something that can provide the data to an [`ElectionProvider`]. +pub trait ElectionDataProvider { + /// All possible targets for the election, i.e. the candidates. + fn targets() -> Vec; + + /// All possible voters for the election. + /// + /// Note that if a notion of self-vote exists, it should be represented here. + fn voters() -> Vec<(AccountId, VoteWeight, Vec)>; + + /// The number of targets to elect. + fn desired_targets() -> u32; + + /// Provide a best effort prediction about when the next election is about to happen. + /// + /// In essence, the implementor should predict with this function when it will trigger the + /// [`ElectionProvider::elect`]. + /// + /// This is only useful for stateful election providers. + fn next_election_prediction(now: BlockNumber) -> BlockNumber; + + /// Utility function only to be used in benchmarking scenarios, to be implemented optionally, + /// else a noop. + #[cfg(any(feature = "runtime-benchmarks", test))] + fn put_snapshot( + _voters: Vec<(AccountId, VoteWeight, Vec)>, + _targets: Vec, + ) { + } +} + +impl ElectionDataProvider for () { + fn targets() -> Vec { + Default::default() + } + fn voters() -> Vec<(AccountId, VoteWeight, Vec)> { + Default::default() + } + fn desired_targets() -> u32 { + Default::default() + } + fn next_election_prediction(now: BlockNumber) -> BlockNumber { + now + } +} + +/// Something that can compute the result of an election and pass it back to the caller. +/// +/// This trait only provides an interface to _request_ an election, i.e. +/// [`ElectionProvider::elect`]. That data required for the election need to be passed to the +/// implemented of this trait through [`ElectionProvider::DataProvider`]. +pub trait ElectionProvider { + /// The error type that is returned by the provider. + type Error: sp_std::fmt::Debug; + + /// The data provider of the election. + type DataProvider: ElectionDataProvider; + + /// Elect a new set of winners. + /// + /// The result is returned in a target major format, namely as vector of supports. + fn elect() -> Result, Self::Error>; +} + +impl ElectionProvider for () { + type Error = &'static str; + type DataProvider = (); + + fn elect() -> Result, Self::Error> { + Err("<() as ElectionProvider> cannot do anything.") + } +} diff --git a/primitives/election-providers/src/onchain.rs b/primitives/election-providers/src/onchain.rs new file mode 100644 index 0000000000000..5813d385969fa --- /dev/null +++ b/primitives/election-providers/src/onchain.rs @@ -0,0 +1,168 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! An implementation of [`ElectionProvider`] that does an on-chain sequential phragmen. + +use sp_arithmetic::InnerOf; +use crate::{ElectionDataProvider, ElectionProvider}; +use sp_npos_elections::{ + ElectionResult, ExtendedBalance, IdentifierT, PerThing128, Supports, VoteWeight, +}; +use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData, prelude::*}; + +/// Errors of the on-chain election. +#[derive(Eq, PartialEq, Debug)] +pub enum Error { + /// An internal error in the NPoS elections crate. + NposElections(sp_npos_elections::Error), +} + +impl From for Error { + fn from(e: sp_npos_elections::Error) -> Self { + Error::NposElections(e) + } +} + +/// A simple on-chain implementation of the election provider trait. +/// +/// This will accept voting data on the fly and produce the results immediately. +/// +/// ### Warning +/// +/// This can be very expensive to run frequently on-chain. Use with care. +pub struct OnChainSequentialPhragmen(PhantomData); + +/// Configuration trait of [`OnChainSequentialPhragmen`]. +/// +/// Note that this is similar to a pallet traits, but [`OnChainSequentialPhragmen`] is not a pallet. +pub trait Config { + /// The account identifier type. + type AccountId: IdentifierT; + /// The block number type. + type BlockNumber; + /// The accuracy used to compute the election: + type Accuracy: PerThing128; + /// Something that provides the data for election. + type DataProvider: ElectionDataProvider; +} + +impl ElectionProvider for OnChainSequentialPhragmen +where + ExtendedBalance: From>, +{ + type Error = Error; + type DataProvider = T::DataProvider; + + fn elect() -> Result, Self::Error> { + let voters = Self::DataProvider::voters(); + let targets = Self::DataProvider::targets(); + let desired_targets = Self::DataProvider::desired_targets() as usize; + + let mut stake_map: BTreeMap = BTreeMap::new(); + + voters.iter().for_each(|(v, s, _)| { + stake_map.insert(v.clone(), *s); + }); + + let stake_of = Box::new(|w: &T::AccountId| -> VoteWeight { + stake_map.get(w).cloned().unwrap_or_default() + }); + + let ElectionResult { + winners, + assignments, + } = sp_npos_elections::seq_phragmen::<_, T::Accuracy>(desired_targets, targets, voters, None) + .map_err(Error::from)?; + + let staked = + sp_npos_elections::assignment_ratio_to_staked_normalized(assignments, &stake_of)?; + let winners = sp_npos_elections::to_without_backing(winners); + + sp_npos_elections::to_supports(&winners, &staked).map_err(Error::from) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use sp_npos_elections::Support; + use sp_runtime::Perbill; + + type AccountId = u64; + type BlockNumber = u32; + + struct Runtime; + impl Config for Runtime { + type AccountId = AccountId; + type BlockNumber = BlockNumber; + type Accuracy = Perbill; + type DataProvider = mock_data_provider::DataProvider; + } + + type OnChainPhragmen = OnChainSequentialPhragmen; + + mod mock_data_provider { + use super::*; + + pub struct DataProvider; + + impl ElectionDataProvider for DataProvider { + fn voters() -> Vec<(AccountId, VoteWeight, Vec)> { + vec![ + (1, 10, vec![10, 20]), + (2, 20, vec![30, 20]), + (3, 30, vec![10, 30]), + ] + } + + fn targets() -> Vec { + vec![10, 20, 30] + } + + fn desired_targets() -> u32 { + 2 + } + + fn next_election_prediction(_: BlockNumber) -> BlockNumber { + 0 + } + } + } + + #[test] + fn onchain_seq_phragmen_works() { + assert_eq!( + OnChainPhragmen::elect().unwrap(), + vec![ + ( + 10, + Support { + total: 25, + voters: vec![(1, 10), (3, 15)] + } + ), + ( + 30, + Support { + total: 35, + voters: vec![(2, 20), (3, 15)] + } + ) + ] + ); + } +} diff --git a/primitives/npos-elections/compact/src/assignment.rs b/primitives/npos-elections/compact/src/assignment.rs index 4f527aa40a748..12f5ca2b41735 100644 --- a/primitives/npos-elections/compact/src/assignment.rs +++ b/primitives/npos-elections/compact/src/assignment.rs @@ -21,7 +21,7 @@ use crate::field_name_for; use proc_macro2::TokenStream as TokenStream2; use quote::quote; -fn from_impl(count: usize) -> TokenStream2 { +pub(crate) fn from_impl(count: usize) -> TokenStream2 { let from_impl_single = { let name = field_name_for(1); quote!(1 => compact.#name.push( @@ -73,7 +73,7 @@ fn from_impl(count: usize) -> TokenStream2 { ) } -fn into_impl(count: usize, per_thing: syn::Type) -> TokenStream2 { +pub(crate) fn into_impl(count: usize, per_thing: syn::Type) -> TokenStream2 { let into_impl_single = { let name = field_name_for(1); quote!( @@ -153,53 +153,3 @@ fn into_impl(count: usize, per_thing: syn::Type) -> TokenStream2 { #into_impl_rest ) } - -pub(crate) fn assignment( - ident: syn::Ident, - voter_type: syn::Type, - target_type: syn::Type, - weight_type: syn::Type, - count: usize, -) -> TokenStream2 { - let from_impl = from_impl(count); - let into_impl = into_impl(count, weight_type.clone()); - - quote!( - use _npos::__OrInvalidIndex; - impl #ident { - pub fn from_assignment( - assignments: Vec<_npos::Assignment>, - index_of_voter: FV, - index_of_target: FT, - ) -> Result - where - A: _npos::IdentifierT, - for<'r> FV: Fn(&'r A) -> Option<#voter_type>, - for<'r> FT: Fn(&'r A) -> Option<#target_type>, - { - let mut compact: #ident = Default::default(); - - for _npos::Assignment { who, distribution } in assignments { - match distribution.len() { - 0 => continue, - #from_impl - _ => { - return Err(_npos::Error::CompactTargetOverflow); - } - } - }; - Ok(compact) - } - - pub fn into_assignment( - self, - voter_at: impl Fn(#voter_type) -> Option, - target_at: impl Fn(#target_type) -> Option, - ) -> Result>, _npos::Error> { - let mut assignments: Vec<_npos::Assignment> = Default::default(); - #into_impl - Ok(assignments) - } - } - ) -} diff --git a/primitives/npos-elections/compact/src/lib.rs b/primitives/npos-elections/compact/src/lib.rs index 32397652f9b93..c8008dba51d65 100644 --- a/primitives/npos-elections/compact/src/lib.rs +++ b/primitives/npos-elections/compact/src/lib.rs @@ -95,19 +95,11 @@ pub fn generate_solution_type(item: TokenStream) -> TokenStream { compact_encoding, ).unwrap_or_else(|e| e.to_compile_error()); - let assignment_impls = assignment::assignment( - ident.clone(), - voter_type.clone(), - target_type.clone(), - weight_type.clone(), - count, - ); - quote!( #imports #solution_struct - #assignment_impls - ).into() + ) + .into() } fn struct_def( @@ -125,29 +117,32 @@ fn struct_def( let singles = { let name = field_name_for(1); + // NOTE: we use the visibility of the struct for the fields as well.. could be made better. quote!( - #name: Vec<(#voter_type, #target_type)>, + #vis #name: Vec<(#voter_type, #target_type)>, ) }; let doubles = { let name = field_name_for(2); quote!( - #name: Vec<(#voter_type, (#target_type, #weight_type), #target_type)>, + #vis #name: Vec<(#voter_type, (#target_type, #weight_type), #target_type)>, ) }; - let rest = (3..=count).map(|c| { - let field_name = field_name_for(c); - let array_len = c - 1; - quote!( - #field_name: Vec<( - #voter_type, - [(#target_type, #weight_type); #array_len], - #target_type - )>, - ) - }).collect::(); + let rest = (3..=count) + .map(|c| { + let field_name = field_name_for(c); + let array_len = c - 1; + quote!( + #vis #field_name: Vec<( + #voter_type, + [(#target_type, #weight_type); #array_len], + #target_type + )>, + ) + }) + .collect::(); let len_impl = len_impl(count); let edge_count_impl = edge_count_impl(count); @@ -172,40 +167,39 @@ fn struct_def( quote!(#[derive(Default, PartialEq, Eq, Clone, Debug, _npos::codec::Encode, _npos::codec::Decode)]) }; + let from_impl = assignment::from_impl(count); + let into_impl = assignment::into_impl(count, weight_type.clone()); + Ok(quote! ( /// A struct to encode a election assignment in a compact way. #derives_and_maybe_compact_encoding #vis struct #ident { #singles #doubles #rest } - impl _npos::VotingLimit for #ident { + use _npos::__OrInvalidIndex; + impl _npos::CompactSolution for #ident { const LIMIT: usize = #count; - } + type Voter = #voter_type; + type Target = #target_type; + type Accuracy = #weight_type; - impl #ident { - /// Get the length of all the assignments that this type is encoding. This is basically - /// the same as the number of assignments, or the number of voters in total. - pub fn len(&self) -> usize { + fn voter_count(&self) -> usize { let mut all_len = 0usize; #len_impl all_len } - /// Get the total count of edges. - pub fn edge_count(&self) -> usize { + fn edge_count(&self) -> usize { let mut all_edges = 0usize; #edge_count_impl all_edges } - /// Get the number of unique targets in the whole struct. - /// - /// Once presented with a list of winners, this set and the set of winners must be - /// equal. - /// - /// The resulting indices are sorted. - pub fn unique_targets(&self) -> Vec<#target_type> { - let mut all_targets: Vec<#target_type> = Vec::with_capacity(self.average_edge_count()); - let mut maybe_insert_target = |t: #target_type| { + fn unique_targets(&self) -> Vec { + // NOTE: this implementation returns the targets sorted, but we don't use it yet per + // se, nor is the API enforcing it. + let mut all_targets: Vec = + Vec::with_capacity(self.average_edge_count()); + let mut maybe_insert_target = |t: Self::Target| { match all_targets.binary_search(&t) { Ok(_) => (), Err(pos) => all_targets.insert(pos, t) @@ -217,22 +211,44 @@ fn struct_def( all_targets } - /// Get the average edge count. - pub fn average_edge_count(&self) -> usize { - self.edge_count().checked_div(self.len()).unwrap_or(0) - } - - /// Remove a certain voter. - /// - /// This will only search until the first instance of `to_remove`, and return true. If - /// no instance is found (no-op), then it returns false. - /// - /// In other words, if this return true, exactly one element must have been removed from - /// `self.len()`. - pub fn remove_voter(&mut self, to_remove: #voter_type) -> bool { + fn remove_voter(&mut self, to_remove: Self::Voter) -> bool { #remove_voter_impl return false } + + fn from_assignment( + assignments: Vec<_npos::Assignment>, + index_of_voter: FV, + index_of_target: FT, + ) -> Result + where + A: _npos::IdentifierT, + for<'r> FV: Fn(&'r A) -> Option, + for<'r> FT: Fn(&'r A) -> Option, + { + let mut compact: #ident = Default::default(); + + for _npos::Assignment { who, distribution } in assignments { + match distribution.len() { + 0 => continue, + #from_impl + _ => { + return Err(_npos::Error::CompactTargetOverflow); + } + } + }; + Ok(compact) + } + + fn into_assignment( + self, + voter_at: impl Fn(Self::Voter) -> Option, + target_at: impl Fn(Self::Target) -> Option, + ) -> Result>, _npos::Error> { + let mut assignments: Vec<_npos::Assignment> = Default::default(); + #into_impl + Ok(assignments) + } } )) } @@ -347,7 +363,6 @@ fn imports() -> Result { } } } - struct SolutionDef { vis: syn::Visibility, ident: syn::Ident, diff --git a/primitives/npos-elections/fuzzer/src/phragmen_balancing.rs b/primitives/npos-elections/fuzzer/src/phragmen_balancing.rs index 024b721b222a7..2ba7e409568b3 100644 --- a/primitives/npos-elections/fuzzer/src/phragmen_balancing.rs +++ b/primitives/npos-elections/fuzzer/src/phragmen_balancing.rs @@ -22,8 +22,8 @@ mod common; use common::*; use honggfuzz::fuzz; use sp_npos_elections::{ - assignment_ratio_to_staked_normalized, build_support_map, to_without_backing, VoteWeight, - evaluate_support, is_score_better, seq_phragmen, + assignment_ratio_to_staked_normalized, is_score_better, seq_phragmen, to_supports, + to_without_backing, EvaluateSupport, VoteWeight, }; use sp_runtime::Perbill; use rand::{self, SeedableRng}; @@ -66,11 +66,16 @@ fn main() { }; let unbalanced_score = { - let staked = assignment_ratio_to_staked_normalized(unbalanced.assignments.clone(), &stake_of).unwrap(); + let staked = assignment_ratio_to_staked_normalized( + unbalanced.assignments.clone(), + &stake_of, + ) + .unwrap(); let winners = to_without_backing(unbalanced.winners.clone()); - let support = build_support_map(winners.as_ref(), staked.as_ref()).unwrap(); + let score = to_supports(winners.as_ref(), staked.as_ref()) + .unwrap() + .evaluate(); - let score = evaluate_support(&support); if score[0] == 0 { // such cases cannot be improved by balancing. return; @@ -87,11 +92,13 @@ fn main() { ).unwrap(); let balanced_score = { - let staked = assignment_ratio_to_staked_normalized(balanced.assignments.clone(), &stake_of).unwrap(); + let staked = assignment_ratio_to_staked_normalized( + balanced.assignments.clone(), + &stake_of, + ).unwrap(); let winners = to_without_backing(balanced.winners); - let support = build_support_map(winners.as_ref(), staked.as_ref()).unwrap(); + to_supports(winners.as_ref(), staked.as_ref()).unwrap().evaluate() - evaluate_support(&support) }; let enhance = is_score_better(balanced_score, unbalanced_score, Perbill::zero()); diff --git a/primitives/npos-elections/fuzzer/src/phragmms_balancing.rs b/primitives/npos-elections/fuzzer/src/phragmms_balancing.rs index 868aa67236f41..8ce7e7d415fa2 100644 --- a/primitives/npos-elections/fuzzer/src/phragmms_balancing.rs +++ b/primitives/npos-elections/fuzzer/src/phragmms_balancing.rs @@ -22,8 +22,8 @@ mod common; use common::*; use honggfuzz::fuzz; use sp_npos_elections::{ - assignment_ratio_to_staked_normalized, build_support_map, to_without_backing, VoteWeight, - evaluate_support, is_score_better, phragmms, + assignment_ratio_to_staked_normalized, is_score_better, phragmms, to_supports, + to_without_backing, EvaluateSupport, VoteWeight, }; use sp_runtime::Perbill; use rand::{self, SeedableRng}; @@ -66,11 +66,14 @@ fn main() { }; let unbalanced_score = { - let staked = assignment_ratio_to_staked_normalized(unbalanced.assignments.clone(), &stake_of).unwrap(); + let staked = assignment_ratio_to_staked_normalized( + unbalanced.assignments.clone(), + &stake_of, + ) + .unwrap(); let winners = to_without_backing(unbalanced.winners.clone()); - let support = build_support_map(winners.as_ref(), staked.as_ref()).unwrap(); + let score = to_supports(&winners, &staked).unwrap().evaluate(); - let score = evaluate_support(&support); if score[0] == 0 { // such cases cannot be improved by balancing. return; @@ -86,11 +89,13 @@ fn main() { ).unwrap(); let balanced_score = { - let staked = assignment_ratio_to_staked_normalized(balanced.assignments.clone(), &stake_of).unwrap(); + let staked = + assignment_ratio_to_staked_normalized(balanced.assignments.clone(), &stake_of) + .unwrap(); let winners = to_without_backing(balanced.winners); - let support = build_support_map(winners.as_ref(), staked.as_ref()).unwrap(); - - evaluate_support(&support) + to_supports(winners.as_ref(), staked.as_ref()) + .unwrap() + .evaluate() }; let enhance = is_score_better(balanced_score, unbalanced_score, Perbill::zero()); diff --git a/primitives/npos-elections/fuzzer/src/reduce.rs b/primitives/npos-elections/fuzzer/src/reduce.rs index 074c1546d49d8..4ee2468d9d140 100644 --- a/primitives/npos-elections/fuzzer/src/reduce.rs +++ b/primitives/npos-elections/fuzzer/src/reduce.rs @@ -34,8 +34,8 @@ use honggfuzz::fuzz; mod common; use common::to_range; -use sp_npos_elections::{StakedAssignment, ExtendedBalance, build_support_map, reduce}; -use rand::{self, Rng, SeedableRng, RngCore}; +use sp_npos_elections::{reduce, to_support_map, ExtendedBalance, StakedAssignment}; +use rand::{self, Rng, RngCore, SeedableRng}; type Balance = u128; type AccountId = u64; @@ -109,9 +109,8 @@ fn assert_assignments_equal( ass1: &Vec>, ass2: &Vec>, ) { - - let support_1 = build_support_map::(winners, ass1).unwrap(); - let support_2 = build_support_map::(winners, ass2).unwrap(); + let support_1 = to_support_map::(winners, ass1).unwrap(); + let support_2 = to_support_map::(winners, ass2).unwrap(); for (who, support) in support_1.iter() { assert_eq!(support.total, support_2.get(who).unwrap().total); diff --git a/primitives/npos-elections/src/helpers.rs b/primitives/npos-elections/src/helpers.rs index 6f4400b6748fd..3dbde0e03c386 100644 --- a/primitives/npos-elections/src/helpers.rs +++ b/primitives/npos-elections/src/helpers.rs @@ -18,21 +18,21 @@ //! Helper methods for npos-elections. use crate::{ - Assignment, ExtendedBalance, VoteWeight, IdentifierT, StakedAssignment, WithApprovalOf, Error, + Assignment, Error, ExtendedBalance, IdentifierT, PerThing128, StakedAssignment, VoteWeight, + WithApprovalOf, }; -use sp_arithmetic::{PerThing, InnerOf}; +use sp_arithmetic::{InnerOf, PerThing}; use sp_std::prelude::*; /// Converts a vector of ratio assignments into ones with absolute budget value. /// /// Note that this will NOT attempt at normalizing the result. -pub fn assignment_ratio_to_staked( +pub fn assignment_ratio_to_staked( ratios: Vec>, stake_of: FS, ) -> Vec> where for<'r> FS: Fn(&'r A) -> VoteWeight, - P: sp_std::ops::Mul, ExtendedBalance: From>, { ratios @@ -45,19 +45,22 @@ where } /// Same as [`assignment_ratio_to_staked`] and try and do normalization. -pub fn assignment_ratio_to_staked_normalized( +pub fn assignment_ratio_to_staked_normalized( ratio: Vec>, stake_of: FS, ) -> Result>, Error> where for<'r> FS: Fn(&'r A) -> VoteWeight, - P: sp_std::ops::Mul, ExtendedBalance: From>, { let mut staked = assignment_ratio_to_staked(ratio, &stake_of); - staked.iter_mut().map(|a| - a.try_normalize(stake_of(&a.who).into()).map_err(|err| Error::ArithmeticError(err)) - ).collect::>()?; + staked + .iter_mut() + .map(|a| { + a.try_normalize(stake_of(&a.who).into()) + .map_err(|err| Error::ArithmeticError(err)) + }) + .collect::>()?; Ok(staked) } @@ -74,7 +77,7 @@ where } /// Same as [`assignment_staked_to_ratio`] and try and do normalization. -pub fn assignment_staked_to_ratio_normalized( +pub fn assignment_staked_to_ratio_normalized( staked: Vec>, ) -> Result>, Error> where diff --git a/primitives/npos-elections/src/lib.rs b/primitives/npos-elections/src/lib.rs index 1e3c2707497c2..7966b66e383fc 100644 --- a/primitives/npos-elections/src/lib.rs +++ b/primitives/npos-elections/src/lib.rs @@ -21,8 +21,8 @@ //! - [`phragmms()`]: Implements a hybrid approach inspired by Phragmén which is executed faster but //! it can achieve a constant factor approximation of the maximin problem, similar to that of the //! MMS algorithm. -//! - [`balance`]: Implements the star balancing algorithm. This iterative process can push -//! a solution toward being more `balances`, which in turn can increase its score. +//! - [`balance`]: Implements the star balancing algorithm. This iterative process can push a +//! solution toward being more `balances`, which in turn can increase its score. //! //! ### Terminology //! @@ -57,7 +57,6 @@ //! //! // the combination of the two makes the election result. //! let election_result = ElectionResult { winners, assignments }; -//! //! ``` //! //! The `Assignment` field of the election result is voter-major, i.e. it is from the perspective of @@ -74,18 +73,24 @@ #![cfg_attr(not(feature = "std"), no_std)] -use sp_std::{ - prelude::*, collections::btree_map::BTreeMap, fmt::Debug, cmp::Ordering, rc::Rc, cell::RefCell, -}; use sp_arithmetic::{ - PerThing, Rational128, ThresholdOrd, InnerOf, Normalizable, - traits::{Zero, Bounded}, + traits::{Bounded, UniqueSaturatedInto, Zero}, + InnerOf, Normalizable, PerThing, Rational128, ThresholdOrd, +}; +use sp_std::{ + cell::RefCell, + cmp::Ordering, + collections::btree_map::BTreeMap, + convert::{TryFrom, TryInto}, + fmt::Debug, + ops::Mul, + prelude::*, + rc::Rc, }; +use codec::{Decode, Encode}; #[cfg(feature = "std")] -use serde::{Serialize, Deserialize}; -#[cfg(feature = "std")] -use codec::{Encode, Decode}; +use serde::{Deserialize, Serialize}; #[cfg(test)] mod mock; @@ -125,20 +130,105 @@ impl __OrInvalidIndex for Option { } } -// re-export the compact solution type. -pub use sp_npos_elections_compact::generate_solution_type; - -/// A trait to limit the number of votes per voter. The generated compact type will implement this. -pub trait VotingLimit { +/// A common interface for all compact solutions. +/// +/// See [`sp-npos-elections-compact`] for more info. +pub trait CompactSolution: Sized { + /// The maximum number of votes that are allowed. const LIMIT: usize; + + /// The voter type. + type Voter: UniqueSaturatedInto + TryInto + TryFrom + Debug + Copy + Clone; + + /// The target type + type Target: UniqueSaturatedInto + TryInto + TryFrom + Debug + Copy + Clone; + + /// The weight/accuracy type of each vote. + type Accuracy: PerThing128; + + /// Build self from a `Vec>`. + fn from_assignment( + assignments: Vec>, + voter_index: FV, + target_index: FT, + ) -> Result + where + A: IdentifierT, + for<'r> FV: Fn(&'r A) -> Option, + for<'r> FT: Fn(&'r A) -> Option; + + /// Convert self into a `Vec>` + fn into_assignment( + self, + voter_at: impl Fn(Self::Voter) -> Option, + target_at: impl Fn(Self::Target) -> Option, + ) -> Result>, Error>; + + /// Get the length of all the voters that this type is encoding. + /// + /// This is basically the same as the number of assignments. + fn voter_count(&self) -> usize; + + /// Get the total count of edges. + /// + /// This is effectively in the range of {[`Self::voter_count`], [`Self::voter_count`] * + /// [`Self::LIMIT`]}. + fn edge_count(&self) -> usize; + + /// Get the number of unique targets in the whole struct. + /// + /// Once presented with a list of winners, this set and the set of winners must be + /// equal. + fn unique_targets(&self) -> Vec; + + /// Get the average edge count. + fn average_edge_count(&self) -> usize { + self.edge_count() + .checked_div(self.voter_count()) + .unwrap_or(0) + } + + /// Remove a certain voter. + /// + /// This will only search until the first instance of `to_remove`, and return true. If + /// no instance is found (no-op), then it returns false. + /// + /// In other words, if this return true, exactly one element must have been removed from + /// `self.len()`. + fn remove_voter(&mut self, to_remove: Self::Voter) -> bool; + + /// Compute the score of this compact solution type. + fn score( + self, + winners: &[A], + stake_of: FS, + voter_at: impl Fn(Self::Voter) -> Option, + target_at: impl Fn(Self::Target) -> Option, + ) -> Result + where + for<'r> FS: Fn(&'r A) -> VoteWeight, + A: IdentifierT, + ExtendedBalance: From>, + { + let ratio = self.into_assignment(voter_at, target_at)?; + let staked = helpers::assignment_ratio_to_staked_normalized(ratio, stake_of)?; + let supports = to_supports(winners, &staked)?; + Ok(supports.evaluate()) + } } +// re-export the compact solution type. +pub use sp_npos_elections_compact::generate_solution_type; + /// an aggregator trait for a generic type of a voter/target identifier. This usually maps to /// substrate's account id. pub trait IdentifierT: Clone + Eq + Default + Ord + Debug + codec::Codec {} - impl IdentifierT for T {} +/// Aggregator trait for a PerThing that can be multiplied by u128 (ExtendedBalance). +pub trait PerThing128: PerThing + Mul {} +impl> PerThing128 for T {} + /// The errors that might occur in the this crate and compact. #[derive(Debug, Eq, PartialEq)] pub enum Error { @@ -151,6 +241,8 @@ pub enum Error { CompactInvalidIndex, /// An error occurred in some arithmetic operation. ArithmeticError(&'static str), + /// The data provided to create support map was invalid. + InvalidSupportEdge, } /// A type which is used in the API of this crate as a numeric weight of a vote, most often the @@ -160,7 +252,8 @@ pub type VoteWeight = u64; /// A type in which performing operations on vote weights are safe. pub type ExtendedBalance = u128; -/// The score of an assignment. This can be computed from the support map via [`evaluate_support`]. +/// The score of an assignment. This can be computed from the support map via +/// [`EvaluateSupport::evaluate`]. pub type ElectionScore = [ExtendedBalance; 3]; /// A winner, with their respective approval stake. @@ -331,10 +424,7 @@ pub struct Assignment { pub distribution: Vec<(AccountId, P)>, } -impl Assignment -where - ExtendedBalance: From>, -{ +impl Assignment { /// Convert from a ratio assignment into one with absolute values aka. [`StakedAssignment`]. /// /// It needs `stake` which is the total budget of the voter. If `fill` is set to true, it @@ -344,11 +434,9 @@ where /// /// If an edge ratio is [`Bounded::min_value()`], it is dropped. This edge can never mean /// anything useful. - pub fn into_staked(self, stake: ExtendedBalance) -> StakedAssignment - where - P: sp_std::ops::Mul, - { - let distribution = self.distribution + pub fn into_staked(self, stake: ExtendedBalance) -> StakedAssignment { + let distribution = self + .distribution .into_iter() .filter_map(|(target, p)| { // if this ratio is zero, then skip it. @@ -408,11 +496,8 @@ pub struct StakedAssignment { impl StakedAssignment { /// Converts self into the normal [`Assignment`] type. /// - /// If `fill` is set to true, it _tries_ to ensure that all the potential rounding errors are - /// compensated and the distribution's sum is exactly equal to 100%, by adding or subtracting - /// the remainder from the last distribution. - /// - /// NOTE: it is quite critical that this attempt always works. The data type returned here will + /// NOTE: This will always round down, and thus the results might be less than a full 100% `P`. + /// Use a normalization post-processing to fix this. The data type returned here will /// potentially get used to create a compact type; a compact type requires sum of ratios to be /// less than 100% upon un-compacting. /// @@ -479,8 +564,8 @@ impl StakedAssignment { /// /// This, at the current version, resembles the `Exposure` defined in the Staking pallet, yet they /// do not necessarily have to be the same. -#[derive(Default, Debug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Eq, PartialEq))] +#[derive(Default, Debug, Encode, Decode, Clone, Eq, PartialEq)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] pub struct Support { /// Total support. pub total: ExtendedBalance, @@ -488,51 +573,43 @@ pub struct Support { pub voters: Vec<(AccountId, ExtendedBalance)>, } -/// A linkage from a candidate and its [`Support`]. -pub type SupportMap = BTreeMap>; - -/// Build the support map from the given election result. It maps a flat structure like -/// -/// ```nocompile -/// assignments: vec![ -/// voter1, vec![(candidate1, w11), (candidate2, w12)], -/// voter2, vec![(candidate1, w21), (candidate2, w22)] -/// ] -/// ``` +/// A target-major representation of the the election outcome. /// -/// into a mapping of candidates and their respective support: +/// Essentially a flat variant of [`SupportMap`]. /// -/// ```nocompile -/// SupportMap { -/// candidate1: Support { -/// own:0, -/// total: w11 + w21, -/// others: vec![(candidate1, w11), (candidate2, w21)] -/// }, -/// candidate2: Support { -/// own:0, -/// total: w12 + w22, -/// others: vec![(candidate1, w12), (candidate2, w22)] -/// }, -/// } -/// ``` +/// The main advantage of this is that it is encodable. +pub type Supports = Vec<(A, Support)>; + +/// Linkage from a winner to their [`Support`]. /// -/// The second returned flag indicates the number of edges who didn't corresponded to an actual -/// winner from the given winner set. A value in this place larger than 0 indicates a potentially -/// faulty assignment. +/// This is more helpful than a normal [`Supports`] as it allows faster error checking. +pub type SupportMap = BTreeMap>; + +/// Helper trait to convert from a support map to a flat support vector. +pub trait FlattenSupportMap { + /// Flatten the support. + fn flatten(self) -> Supports; +} + +impl FlattenSupportMap for SupportMap { + fn flatten(self) -> Supports { + self.into_iter().collect::>() + } +} + +/// Build the support map from the winners and assignments. /// -/// `O(E)` where `E` is the total number of edges. -pub fn build_support_map( - winners: &[AccountId], - assignments: &[StakedAssignment], -) -> Result, AccountId> where - AccountId: IdentifierT, -{ +/// The list of winners is basically a redundancy for error checking only; It ensures that all the +/// targets pointed to by the [`Assignment`] are present in the `winners`. +pub fn to_support_map( + winners: &[A], + assignments: &[StakedAssignment], +) -> Result, Error> { // Initialize the support of each candidate. - let mut supports = >::new(); - winners - .iter() - .for_each(|e| { supports.insert(e.clone(), Default::default()); }); + let mut supports = >::new(); + winners.iter().for_each(|e| { + supports.insert(e.clone(), Default::default()); + }); // build support struct. for StakedAssignment { who, distribution } in assignments.iter() { @@ -541,37 +618,83 @@ pub fn build_support_map( support.total = support.total.saturating_add(*weight_extended); support.voters.push((who.clone(), *weight_extended)); } else { - return Err(c.clone()) + return Err(Error::InvalidSupportEdge) } } } Ok(supports) } -/// Evaluate a support map. The returned tuple contains: +/// Same as [`to_support_map`] except it calls `FlattenSupportMap` on top of the result to return a +/// flat vector. /// -/// - Minimum support. This value must be **maximized**. -/// - Sum of all supports. This value must be **maximized**. -/// - Sum of all supports squared. This value must be **minimized**. +/// Similar to [`to_support_map`], `winners` is used for error checking. +pub fn to_supports( + winners: &[A], + assignments: &[StakedAssignment], +) -> Result, Error> { + to_support_map(winners, assignments).map(FlattenSupportMap::flatten) +} + +/// Extension trait for evaluating a support map or vector. +pub trait EvaluateSupport { + /// Evaluate a support map. The returned tuple contains: + /// + /// - Minimum support. This value must be **maximized**. + /// - Sum of all supports. This value must be **maximized**. + /// - Sum of all supports squared. This value must be **minimized**. + fn evaluate(self) -> ElectionScore; +} + +/// A common wrapper trait for both (&A, &B) and &(A, B). /// -/// `O(E)` where `E` is the total number of edges. -pub fn evaluate_support( - support: &SupportMap, -) -> ElectionScore { - let mut min_support = ExtendedBalance::max_value(); - let mut sum: ExtendedBalance = Zero::zero(); - // NOTE: The third element might saturate but fine for now since this will run on-chain and need - // to be fast. - let mut sum_squared: ExtendedBalance = Zero::zero(); - for (_, support) in support.iter() { - sum = sum.saturating_add(support.total); - let squared = support.total.saturating_mul(support.total); - sum_squared = sum_squared.saturating_add(squared); - if support.total < min_support { - min_support = support.total; +/// This allows us to implemented something for both `Vec<_>` and `BTreeMap<_>`, such as +/// [`EvaluateSupport`]. +pub trait TupleRef { + fn extract(&self) -> (&K, &V); +} + +impl TupleRef for &(K, V) { + fn extract(&self) -> (&K, &V) { + (&self.0, &self.1) + } +} + +impl TupleRef for (K, V) { + fn extract(&self) -> (&K, &V) { + (&self.0, &self.1) + } +} + +impl TupleRef for (&K, &V) { + fn extract(&self) -> (&K, &V) { + (self.0, self.1) + } +} + +impl EvaluateSupport for C +where + C: IntoIterator, + I: TupleRef>, + A: IdentifierT, +{ + fn evaluate(self) -> ElectionScore { + let mut min_support = ExtendedBalance::max_value(); + let mut sum: ExtendedBalance = Zero::zero(); + // NOTE: The third element might saturate but fine for now since this will run on-chain and + // need to be fast. + let mut sum_squared: ExtendedBalance = Zero::zero(); + for item in self { + let (_, support) = item.extract(); + sum = sum.saturating_add(support.total); + let squared = support.total.saturating_mul(support.total); + sum_squared = sum_squared.saturating_add(squared); + if support.total < min_support { + min_support = support.total; + } } + [min_support, sum, sum_squared] } - [min_support, sum, sum_squared] } /// Compares two sets of election scores based on desirability and returns true if `this` is better @@ -582,14 +705,15 @@ pub fn evaluate_support( /// /// Note that the third component should be minimized. pub fn is_score_better(this: ElectionScore, that: ElectionScore, epsilon: P) -> bool - where ExtendedBalance: From> +where + ExtendedBalance: From>, { match this .iter() - .enumerate() - .map(|(i, e)| ( - e.ge(&that[i]), - e.tcmp(&that[i], epsilon.mul_ceil(that[i])), + .zip(that.iter()) + .map(|(thi, tha)| ( + thi.ge(&tha), + thi.tcmp(&tha, epsilon.mul_ceil(*tha)), )) .collect::>() .as_slice() diff --git a/primitives/npos-elections/src/mock.rs b/primitives/npos-elections/src/mock.rs index 410adcc3779e0..57b2204a72b48 100644 --- a/primitives/npos-elections/src/mock.rs +++ b/primitives/npos-elections/src/mock.rs @@ -19,10 +19,13 @@ #![cfg(test)] -use crate::{seq_phragmen, ElectionResult, Assignment, VoteWeight, ExtendedBalance}; -use sp_arithmetic::{PerThing, InnerOf, traits::{SaturatedConversion, Zero, One}}; -use sp_std::collections::btree_map::BTreeMap; +use crate::*; +use sp_arithmetic::{ + traits::{One, SaturatedConversion, Zero}, + InnerOf, PerThing, +}; use sp_runtime::assert_eq_error_rate; +use sp_std::collections::btree_map::BTreeMap; #[derive(Default, Debug)] pub(crate) struct _Candidate { @@ -313,14 +316,13 @@ pub fn check_assignments_sum(assignments: Vec( +pub(crate) fn run_and_compare( candidates: Vec, voters: Vec<(AccountId, Vec)>, stake_of: &Box VoteWeight>, to_elect: usize, ) where ExtendedBalance: From>, - Output: sp_std::ops::Mul, { // run fixed point code. let ElectionResult { winners, assignments } = seq_phragmen::<_, Output>( diff --git a/primitives/npos-elections/src/phragmen.rs b/primitives/npos-elections/src/phragmen.rs index 8f88c45ae6de8..ed45efe1b54f7 100644 --- a/primitives/npos-elections/src/phragmen.rs +++ b/primitives/npos-elections/src/phragmen.rs @@ -21,15 +21,15 @@ //! to the Maximin problem. use crate::{ - IdentifierT, VoteWeight, Voter, CandidatePtr, ExtendedBalance, setup_inputs, ElectionResult, + balancing, setup_inputs, CandidatePtr, ElectionResult, ExtendedBalance, IdentifierT, + PerThing128, VoteWeight, Voter, }; -use sp_std::prelude::*; use sp_arithmetic::{ - PerThing, InnerOf, Rational128, helpers_128bit::multiply_by_rational, - traits::{Zero, Bounded}, + traits::{Bounded, Zero}, + InnerOf, Rational128, }; -use crate::balancing; +use sp_std::prelude::*; /// The denominator used for loads. Since votes are collected as u64, the smallest ratio that we /// might collect is `1/approval_stake` where approval stake is the sum of votes. Hence, some number @@ -63,12 +63,15 @@ const DEN: ExtendedBalance = ExtendedBalance::max_value(); /// `expect` this to return `Ok`. /// /// This can only fail if the normalization fails. -pub fn seq_phragmen( +pub fn seq_phragmen( rounds: usize, initial_candidates: Vec, initial_voters: Vec<(AccountId, VoteWeight, Vec)>, balance: Option<(usize, ExtendedBalance)>, -) -> Result, &'static str> where ExtendedBalance: From> { +) -> Result, crate::Error> +where + ExtendedBalance: From>, +{ let (candidates, voters) = setup_inputs(initial_candidates, initial_voters); let (candidates, mut voters) = seq_phragmen_core::( @@ -93,13 +96,26 @@ pub fn seq_phragmen( // sort winners based on desirability. winners.sort_by_key(|c_ptr| c_ptr.borrow().round); - let mut assignments = voters.into_iter().filter_map(|v| v.into_assignment()).collect::>(); - let _ = assignments.iter_mut().map(|a| a.try_normalize()).collect::>()?; - let winners = winners.into_iter().map(|w_ptr| - (w_ptr.borrow().who.clone(), w_ptr.borrow().backed_stake) - ).collect(); + let mut assignments = voters + .into_iter() + .filter_map(|v| v.into_assignment()) + .collect::>(); + let _ = assignments + .iter_mut() + .map(|a| { + a.try_normalize() + .map_err(|e| crate::Error::ArithmeticError(e)) + }) + .collect::>()?; + let winners = winners + .into_iter() + .map(|w_ptr| (w_ptr.borrow().who.clone(), w_ptr.borrow().backed_stake)) + .collect(); - Ok(ElectionResult { winners, assignments }) + Ok(ElectionResult { + winners, + assignments, + }) } /// Core implementation of seq-phragmen. @@ -114,7 +130,7 @@ pub fn seq_phragmen_core( rounds: usize, candidates: Vec>, mut voters: Vec>, -) -> Result<(Vec>, Vec>), &'static str> { +) -> Result<(Vec>, Vec>), crate::Error> { // we have already checked that we have more candidates than minimum_candidate_count. let to_elect = rounds.min(candidates.len()); @@ -198,7 +214,7 @@ pub fn seq_phragmen_core( // edge of all candidates that eventually have a non-zero weight must be elected. debug_assert!(voter.edges.iter().all(|e| e.candidate.borrow().elected)); // inc budget to sum the budget. - voter.try_normalize_elected()?; + voter.try_normalize_elected().map_err(|e| crate::Error::ArithmeticError(e))?; } Ok((candidates, voters)) diff --git a/primitives/npos-elections/src/phragmms.rs b/primitives/npos-elections/src/phragmms.rs index b0f841e57f245..b37d3432f9d7e 100644 --- a/primitives/npos-elections/src/phragmms.rs +++ b/primitives/npos-elections/src/phragmms.rs @@ -23,7 +23,7 @@ use crate::{ IdentifierT, ElectionResult, ExtendedBalance, setup_inputs, VoteWeight, Voter, CandidatePtr, - balance, + balance, PerThing128, }; use sp_arithmetic::{PerThing, InnerOf, Rational128, traits::Bounded}; use sp_std::{prelude::*, rc::Rc}; @@ -41,13 +41,14 @@ use sp_std::{prelude::*, rc::Rc}; /// assignments, `assignment.distribution.map(|p| p.deconstruct()).sum()` fails to fit inside /// `UpperOf

`. A user of this crate may statically assert that this can never happen and safely /// `expect` this to return `Ok`. -pub fn phragmms( +pub fn phragmms( to_elect: usize, initial_candidates: Vec, initial_voters: Vec<(AccountId, VoteWeight, Vec)>, balancing_config: Option<(usize, ExtendedBalance)>, ) -> Result, &'static str> - where ExtendedBalance: From> +where + ExtendedBalance: From>, { let (candidates, mut voters) = setup_inputs(initial_candidates, initial_voters); diff --git a/primitives/npos-elections/src/tests.rs b/primitives/npos-elections/src/tests.rs index 1d26909911f33..7aac5aae1ddab 100644 --- a/primitives/npos-elections/src/tests.rs +++ b/primitives/npos-elections/src/tests.rs @@ -17,14 +17,13 @@ //! Tests for npos-elections. -use crate::mock::*; use crate::{ - seq_phragmen, balancing, build_support_map, is_score_better, helpers::*, - Support, StakedAssignment, Assignment, ElectionResult, ExtendedBalance, setup_inputs, - seq_phragmen_core, Voter, + balancing, helpers::*, is_score_better, mock::*, seq_phragmen, seq_phragmen_core, setup_inputs, + to_support_map, to_supports, Assignment, ElectionResult, ExtendedBalance, StakedAssignment, + Support, Voter, EvaluateSupport, }; +use sp_arithmetic::{PerU16, Perbill, Percent, Permill}; use substrate_test_utils::assert_eq_uvec; -use sp_arithmetic::{Perbill, Permill, Percent, PerU16}; #[test] fn float_phragmen_poc_works() { @@ -53,22 +52,44 @@ fn float_phragmen_poc_works() { assert_eq!( support_map.get(&2).unwrap(), - &_Support { own: 0.0, total: 25.0, others: vec![(10u64, 10.0), (30u64, 15.0)]} + &_Support { + own: 0.0, + total: 25.0, + others: vec![(10u64, 10.0), (30u64, 15.0)] + } ); assert_eq!( support_map.get(&3).unwrap(), - &_Support { own: 0.0, total: 35.0, others: vec![(20u64, 20.0), (30u64, 15.0)]} + &_Support { + own: 0.0, + total: 35.0, + others: vec![(20u64, 20.0), (30u64, 15.0)] + } ); - equalize_float(phragmen_result.assignments, &mut support_map, 0.0, 2, stake_of); + equalize_float( + phragmen_result.assignments, + &mut support_map, + 0.0, + 2, + stake_of, + ); assert_eq!( support_map.get(&2).unwrap(), - &_Support { own: 0.0, total: 30.0, others: vec![(10u64, 10.0), (30u64, 20.0)]} + &_Support { + own: 0.0, + total: 30.0, + others: vec![(10u64, 10.0), (30u64, 20.0)] + } ); assert_eq!( support_map.get(&3).unwrap(), - &_Support { own: 0.0, total: 30.0, others: vec![(20u64, 20.0), (30u64, 10.0)]} + &_Support { + own: 0.0, + total: 30.0, + others: vec![(20u64, 20.0), (30u64, 10.0)] + } ); } @@ -300,7 +321,7 @@ fn phragmen_poc_works() { let staked = assignment_ratio_to_staked(assignments, &stake_of); let winners = to_without_backing(winners); - let support_map = build_support_map::(&winners, &staked).unwrap(); + let support_map = to_support_map::(&winners, &staked).unwrap(); assert_eq_uvec!( staked, @@ -374,7 +395,7 @@ fn phragmen_poc_works_with_balancing() { let staked = assignment_ratio_to_staked(assignments, &stake_of); let winners = to_without_backing(winners); - let support_map = build_support_map::(&winners, &staked).unwrap(); + let support_map = to_support_map::(&winners, &staked).unwrap(); assert_eq_uvec!( staked, @@ -766,7 +787,7 @@ fn phragmen_self_votes_should_be_kept() { let staked_assignments = assignment_ratio_to_staked(result.assignments, &stake_of); let winners = to_without_backing(result.winners); - let supports = build_support_map::(&winners, &staked_assignments).unwrap(); + let supports = to_support_map::(&winners, &staked_assignments).unwrap(); assert_eq!(supports.get(&5u64), None); assert_eq!( @@ -839,6 +860,34 @@ fn duplicate_target_is_ignored_when_winner() { ); } +#[test] +fn support_map_and_vec_can_be_evaluated() { + let candidates = vec![1, 2, 3]; + let voters = vec![(10, vec![1, 2]), (20, vec![1, 3]), (30, vec![2, 3])]; + + let stake_of = create_stake_of(&[(10, 10), (20, 20), (30, 30)]); + let ElectionResult { + winners, + assignments, + } = seq_phragmen::<_, Perbill>( + 2, + candidates, + voters + .iter() + .map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())) + .collect::>(), + None, + ) + .unwrap(); + + let staked = assignment_ratio_to_staked(assignments, &stake_of); + let winners = to_without_backing(winners); + let support_map = to_support_map::(&winners, &staked).unwrap(); + let support_vec = to_supports(&winners, &staked).unwrap(); + + assert_eq!(support_map.evaluate(), support_vec.evaluate()); +} + mod assignment_convert_normalize { use super::*; #[test] @@ -1112,20 +1161,16 @@ mod score { } mod solution_type { - use codec::{Decode, Encode}; use super::AccountId; + use codec::{Decode, Encode}; // these need to come from the same dev-dependency `sp-npos-elections`, not from the crate. - use crate::{ - generate_solution_type, Assignment, - Error as PhragmenError, - }; - use sp_std::{convert::TryInto, fmt::Debug}; + use crate::{generate_solution_type, Assignment, CompactSolution, Error as PhragmenError}; use sp_arithmetic::Percent; + use sp_std::{convert::TryInto, fmt::Debug}; type TestAccuracy = Percent; generate_solution_type!(pub struct TestSolutionCompact::(16)); - #[allow(dead_code)] mod __private { // This is just to make sure that that the compact can be generated in a scope without any @@ -1136,7 +1181,6 @@ mod solution_type { #[compact] struct InnerTestSolutionCompact::(12) ); - } #[test] @@ -1190,7 +1234,7 @@ mod solution_type { compact, Decode::decode(&mut &encoded[..]).unwrap(), ); - assert_eq!(compact.len(), 4); + assert_eq!(compact.voter_count(), 4); assert_eq!(compact.edge_count(), 2 + 4); assert_eq!(compact.unique_targets(), vec![10, 11, 20, 40, 50, 51]); } @@ -1326,7 +1370,7 @@ mod solution_type { ).unwrap(); // basically number of assignments that it is encoding. - assert_eq!(compacted.len(), assignments.len()); + assert_eq!(compacted.voter_count(), assignments.len()); assert_eq!( compacted.edge_count(), assignments.iter().fold(0, |a, b| a + b.distribution.len()), @@ -1410,9 +1454,12 @@ mod solution_type { ..Default::default() }; - assert_eq!(compact.unique_targets(), vec![1, 2, 3, 4, 7, 8, 11, 12, 13, 66, 67]); + assert_eq!( + compact.unique_targets(), + vec![1, 2, 3, 4, 7, 8, 11, 12, 13, 66, 67] + ); assert_eq!(compact.edge_count(), 2 + (2 * 2) + 3 + 16); - assert_eq!(compact.len(), 6); + assert_eq!(compact.voter_count(), 6); // this one has some duplicates. let compact = TestSolutionCompact { @@ -1429,7 +1476,7 @@ mod solution_type { assert_eq!(compact.unique_targets(), vec![1, 3, 4, 7, 8, 11, 13]); assert_eq!(compact.edge_count(), 2 + (2 * 2) + 3); - assert_eq!(compact.len(), 5); + assert_eq!(compact.voter_count(), 5); } #[test] From 5aea9cc9681890eadc110028a49f9a2d65ab12aa Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 15 Jan 2021 15:17:56 +0000 Subject: [PATCH 02/86] pallet and unsigned phase --- Cargo.lock | 34 + Cargo.toml | 1 + bin/node/runtime/Cargo.toml | 5 + bin/node/runtime/src/constants.rs | 2 +- bin/node/runtime/src/lib.rs | 57 +- frame/babe/Cargo.toml | 1 + frame/babe/src/lib.rs | 12 +- frame/babe/src/mock.rs | 11 +- .../election-provider-multi-phase/Cargo.toml | 67 + .../src/benchmarking.rs | 283 ++++ .../src/helpers.rs | 199 +++ .../election-provider-multi-phase/src/lib.rs | 1495 +++++++++++++++++ .../election-provider-multi-phase/src/mock.rs | 386 +++++ .../src/unsigned.rs | 799 +++++++++ .../src/weights.rs | 147 ++ frame/grandpa/Cargo.toml | 1 + frame/grandpa/src/mock.rs | 9 + frame/offences/benchmarking/Cargo.toml | 2 + frame/offences/benchmarking/src/mock.rs | 10 +- frame/session/benchmarking/Cargo.toml | 2 + frame/session/benchmarking/src/mock.rs | 12 +- frame/session/src/lib.rs | 34 +- frame/staking/Cargo.toml | 5 + frame/staking/fuzzer/Cargo.toml | 1 + frame/staking/fuzzer/src/mock.rs | 18 +- frame/staking/src/lib.rs | 344 +++- frame/staking/src/mock.rs | 51 +- frame/staking/src/offchain_election.rs | 52 +- frame/staking/src/testing_utils.rs | 8 +- frame/staking/src/tests.rs | 98 +- frame/support/src/traits.rs | 65 +- 31 files changed, 4042 insertions(+), 169 deletions(-) create mode 100644 frame/election-provider-multi-phase/Cargo.toml create mode 100644 frame/election-provider-multi-phase/src/benchmarking.rs create mode 100644 frame/election-provider-multi-phase/src/helpers.rs create mode 100644 frame/election-provider-multi-phase/src/lib.rs create mode 100644 frame/election-provider-multi-phase/src/mock.rs create mode 100644 frame/election-provider-multi-phase/src/unsigned.rs create mode 100644 frame/election-provider-multi-phase/src/weights.rs diff --git a/Cargo.lock b/Cargo.lock index 05f4896e071ff..f02b4183a069f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3954,6 +3954,7 @@ dependencies = [ "pallet-contracts-primitives", "pallet-contracts-rpc-runtime-api", "pallet-democracy", + "pallet-election-provider-multi-phase", "pallet-elections-phragmen", "pallet-grandpa", "pallet-identity", @@ -3992,6 +3993,7 @@ dependencies = [ "sp-inherents", "sp-io", "sp-keyring", + "sp-npos-elections", "sp-offchain", "sp-runtime", "sp-session", @@ -4385,6 +4387,7 @@ dependencies = [ "sp-consensus-babe", "sp-consensus-vrf", "sp-core", + "sp-election-providers", "sp-inherents", "sp-io", "sp-runtime", @@ -4545,6 +4548,32 @@ dependencies = [ "substrate-test-utils", ] +[[package]] +name = "pallet-election-provider-multi-phase" +version = "2.0.0" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "hex-literal", + "pallet-balances", + "parity-scale-codec", + "parking_lot 0.11.1", + "paste 1.0.3", + "rand 0.7.3", + "serde", + "sp-arithmetic", + "sp-core", + "sp-election-providers", + "sp-io", + "sp-npos-elections", + "sp-runtime", + "sp-std", + "sp-tracing", + "static_assertions", + "substrate-test-utils", +] + [[package]] name = "pallet-elections" version = "2.0.1" @@ -4645,6 +4674,7 @@ dependencies = [ "serde", "sp-application-crypto", "sp-core", + "sp-election-providers", "sp-finality-grandpa", "sp-io", "sp-keyring", @@ -4834,6 +4864,7 @@ dependencies = [ "parity-scale-codec", "serde", "sp-core", + "sp-election-providers", "sp-io", "sp-runtime", "sp-staking", @@ -4955,6 +4986,7 @@ dependencies = [ "rand 0.7.3", "serde", "sp-core", + "sp-election-providers", "sp-io", "sp-runtime", "sp-session", @@ -4996,6 +5028,7 @@ dependencies = [ "serde", "sp-application-crypto", "sp-core", + "sp-election-providers", "sp-io", "sp-npos-elections", "sp-runtime", @@ -5022,6 +5055,7 @@ dependencies = [ "pallet-timestamp", "parity-scale-codec", "sp-core", + "sp-election-providers", "sp-io", "sp-npos-elections", "sp-runtime", diff --git a/Cargo.toml b/Cargo.toml index 1754f896c8846..e3f04cd996e41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ members = [ "frame/contracts/rpc/runtime-api", "frame/democracy", "frame/elections", + "frame/election-provider-multi-phase", "frame/example", "frame/example-offchain-worker", "frame/example-parallel", diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index d3cc0101e082b..e74600f9501f0 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -35,6 +35,7 @@ sp-keyring = { version = "2.0.0", optional = true, path = "../../../primitives/k sp-session = { version = "2.0.0", default-features = false, path = "../../../primitives/session" } sp-transaction-pool = { version = "2.0.0", default-features = false, path = "../../../primitives/transaction-pool" } sp-version = { version = "2.0.0", default-features = false, path = "../../../primitives/version" } +sp-npos-elections = { version = "2.0.0", default-features = false, path = "../../../primitives/npos-elections" } # frame dependencies frame-executive = { version = "2.0.0", default-features = false, path = "../../../frame/executive" } @@ -71,6 +72,7 @@ pallet-recovery = { version = "2.0.0", default-features = false, path = "../../. pallet-session = { version = "2.0.0", features = ["historical"], path = "../../../frame/session", default-features = false } pallet-session-benchmarking = { version = "2.0.0", path = "../../../frame/session/benchmarking", default-features = false, optional = true } pallet-staking = { version = "2.0.0", default-features = false, path = "../../../frame/staking" } +pallet-election-provider-multi-phase = { version = "2.0.0", default-features = false, path = "../../../frame/election-provider-multi-phase" } pallet-staking-reward-curve = { version = "2.0.0", default-features = false, path = "../../../frame/staking/reward-curve" } pallet-scheduler = { version = "2.0.0", default-features = false, path = "../../../frame/scheduler" } pallet-society = { version = "2.0.0", default-features = false, path = "../../../frame/society" } @@ -114,6 +116,7 @@ std = [ "pallet-im-online/std", "pallet-indices/std", "sp-inherents/std", + "sp-npos-elections/std", "pallet-lottery/std", "pallet-membership/std", "pallet-mmr/std", @@ -140,6 +143,7 @@ std = [ "frame-benchmarking/std", "frame-system-rpc-runtime-api/std", "frame-system/std", + "pallet-election-provider-multi-phase/std", "pallet-timestamp/std", "pallet-tips/std", "pallet-transaction-payment-rpc-runtime-api/std", @@ -156,6 +160,7 @@ runtime-benchmarks = [ "frame-benchmarking", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", + "pallet-election-provider-multi-phase/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "pallet-assets/runtime-benchmarks", "pallet-babe/runtime-benchmarks", diff --git a/bin/node/runtime/src/constants.rs b/bin/node/runtime/src/constants.rs index f447486c7ffc4..c549b1977d376 100644 --- a/bin/node/runtime/src/constants.rs +++ b/bin/node/runtime/src/constants.rs @@ -35,7 +35,7 @@ pub mod time { use node_primitives::{Moment, BlockNumber}; /// Since BABE is probabilistic this is the average expected block time that - /// we are targetting. Blocks will be produced at a minimum duration defined + /// we are targeting. Blocks will be produced at a minimum duration defined /// by `SLOT_DURATION`, but some slots will not be allocated to any /// authority and hence no block will be produced. We expect to have this /// block time on average following the defined slot duration and the value diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index e88484e472958..bc3526af195f7 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -28,7 +28,8 @@ use frame_support::{ construct_runtime, parameter_types, debug, RuntimeDebug, weights::{ Weight, IdentityFee, - constants::{BlockExecutionWeight, ExtrinsicBaseWeight, RocksDbWeight, WEIGHT_PER_SECOND}, DispatchClass, + constants::{BlockExecutionWeight, ExtrinsicBaseWeight, RocksDbWeight, WEIGHT_PER_SECOND}, + DispatchClass, }, traits::{ Currency, Imbalance, KeyOwnerProofSystem, OnUnbalanced, Randomness, LockIdentifier, @@ -50,14 +51,14 @@ pub use node_primitives::{AccountId, Signature}; use node_primitives::{AccountIndex, Balance, BlockNumber, Hash, Index, Moment}; use sp_api::impl_runtime_apis; use sp_runtime::{ - Permill, Perbill, Perquintill, Percent, ApplyExtrinsicResult, - impl_opaque_keys, generic, create_runtime_str, ModuleId, FixedPointNumber, + Permill, Perbill, Perquintill, Percent, ApplyExtrinsicResult, impl_opaque_keys, generic, + create_runtime_str, ModuleId, FixedPointNumber, }; use sp_runtime::curve::PiecewiseLinear; use sp_runtime::transaction_validity::{TransactionValidity, TransactionSource, TransactionPriority}; use sp_runtime::traits::{ - self, BlakeTwo256, Block as BlockT, StaticLookup, SaturatedConversion, - ConvertInto, OpaqueKeys, NumberFor, + self, BlakeTwo256, Block as BlockT, StaticLookup, SaturatedConversion, ConvertInto, OpaqueKeys, + NumberFor, }; use sp_version::RuntimeVersion; #[cfg(any(feature = "std", test))] @@ -145,7 +146,7 @@ impl OnUnbalanced for DealWithFees { } } -/// We assume that ~10% of the block weight is consumed by `on_initalize` handlers. +/// We assume that ~10% of the block weight is consumed by `on_initialize` handlers. /// This is used to limit the maximal weight of a single extrinsic. const AVERAGE_ON_INITIALIZE_RATIO: Perbill = Perbill::from_percent(10); /// We allow `Normal` extrinsics to fill up the block up to 75%, the rest can be used @@ -488,18 +489,56 @@ impl pallet_staking::Config for Runtime { type SessionInterface = Self; type RewardCurve = RewardCurve; type NextNewSession = Session; + type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type ElectionLookahead = ElectionLookahead; type Call = Call; type MaxIterations = MaxIterations; type MinSolutionScoreBump = MinSolutionScoreBump; - type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type UnsignedPriority = StakingUnsignedPriority; // The unsigned solution weight targeted by the OCW. We set it to the maximum possible value of // a single extrinsic. type OffchainSolutionWeightLimit = OffchainSolutionWeightLimit; + type ElectionProvider = ElectionProviderMultiPhase; type WeightInfo = pallet_staking::weights::SubstrateWeight; } +use pallet_election_provider_multi_phase::FallbackStrategy; +parameter_types! { + // phase durations + pub const SignedPhase: u32 = 100; + pub const UnsignedPhase: u32 = 100; + + // fallback: no need to do on-chain phragmen initially. + pub const Fallback: FallbackStrategy = FallbackStrategy::Nothing; + + pub SolutionImprovementThreshold: Perbill = Perbill::from_rational_approximation(1u32, 10_000); + + // miner configs + pub const TwoPhaseUnsignedPriority: TransactionPriority = StakingUnsignedPriority::get() - 1u64; + pub const MinerMaxIterations: u32 = 10; + pub MinerMaxWeight: Weight = RuntimeBlockWeights::get() + .get(DispatchClass::Normal) + .max_extrinsic.expect("Normal extrinsics have a weight limit configured; qed") + .saturating_sub(BlockExecutionWeight::get()); +} + +impl pallet_election_provider_multi_phase::Config for Runtime { + type Event = Event; + type Currency = Balances; + type SignedPhase = SignedPhase; + type UnsignedPhase = UnsignedPhase; + type SolutionImprovementThreshold = MinSolutionScoreBump; + type MinerMaxIterations = MinerMaxIterations; + type MinerMaxWeight = MinerMaxWeight; + type UnsignedPriority = TwoPhaseUnsignedPriority; + type DataProvider = Staking; + type OnChainAccuracy = Perbill; + type CompactSolution = pallet_staking::CompactAssignments; + type Fallback = Fallback; + type WeightInfo = pallet_election_provider_multi_phase::weights::SubstrateWeight; + type BenchmarkingConfig = (); +} + parameter_types! { pub const LaunchPeriod: BlockNumber = 28 * 24 * 60 * MINUTES; pub const VotingPeriod: BlockNumber = 28 * 24 * 60 * MINUTES; @@ -546,7 +585,7 @@ impl pallet_democracy::Config for Runtime { >; type BlacklistOrigin = EnsureRoot; // Any single technical committee member may veto a coming council proposal, however they can - // only do it once and it lasts only for the cooloff period. + // only do it once and it lasts only for the cool-off period. type VetoOrigin = pallet_collective::EnsureMember; type CooloffPeriod = CooloffPeriod; type PreimageByteDeposit = PreimageByteDeposit; @@ -1006,6 +1045,7 @@ construct_runtime!( Indices: pallet_indices::{Module, Call, Storage, Config, Event}, Balances: pallet_balances::{Module, Call, Storage, Config, Event}, TransactionPayment: pallet_transaction_payment::{Module, Storage}, + ElectionProviderMultiPhase: pallet_election_provider_multi_phase::{Module, Call, Storage, Event, ValidateUnsigned}, Staking: pallet_staking::{Module, Call, Config, Storage, Event, ValidateUnsigned}, Session: pallet_session::{Module, Call, Storage, Event, Config}, Democracy: pallet_democracy::{Module, Call, Storage, Config, Event}, @@ -1337,6 +1377,7 @@ impl_runtime_apis! { add_benchmark!(params, batches, pallet_treasury, Treasury); add_benchmark!(params, batches, pallet_utility, Utility); add_benchmark!(params, batches, pallet_vesting, Vesting); + add_benchmark!(params, batches, pallet_election_provider_multi_phase, ElectionProviderMultiPhase); if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) } Ok(batches) diff --git a/frame/babe/Cargo.toml b/frame/babe/Cargo.toml index 13ac2e4034c9f..a5e33bbf8f338 100644 --- a/frame/babe/Cargo.toml +++ b/frame/babe/Cargo.toml @@ -39,6 +39,7 @@ pallet-offences = { version = "2.0.0", path = "../offences" } pallet-staking = { version = "2.0.0", path = "../staking" } pallet-staking-reward-curve = { version = "2.0.0", path = "../staking/reward-curve" } sp-core = { version = "2.0.0", path = "../../primitives/core" } +sp-election-providers = { version = "2.0.0", path = "../../primitives/election-providers" } [features] default = ["std"] diff --git a/frame/babe/src/lib.rs b/frame/babe/src/lib.rs index d604bfd57d1a3..31266d64f6323 100644 --- a/frame/babe/src/lib.rs +++ b/frame/babe/src/lib.rs @@ -415,12 +415,14 @@ impl Module { /// In other word, this is only accurate if no slots are missed. Given missed slots, the slot /// number will grow while the block number will not. Hence, the result can be interpreted as an /// upper bound. - // -------------- IMPORTANT NOTE -------------- + // + // ## IMPORTANT NOTE + // // This implementation is linked to how [`should_epoch_change`] is working. This might need to // be updated accordingly, if the underlying mechanics of slot and epochs change. // - // WEIGHT NOTE: This function is tied to the weight of `EstimateNextSessionRotation`. If you update - // this function, you must also update the corresponding weight. + // WEIGHT NOTE: This function is tied to the weight of `EstimateNextSessionRotation`. If you + // update this function, you must also update the corresponding weight. pub fn next_expected_epoch_change(now: T::BlockNumber) -> Option { let next_slot = Self::current_epoch_start().saturating_add(T::EpochDuration::get()); next_slot @@ -748,6 +750,10 @@ impl OnTimestampSet for Module { } impl frame_support::traits::EstimateNextSessionRotation for Module { + fn average_session_length() -> T::BlockNumber { + T::EpochDuration::get().saturated_into() + } + fn estimate_next_session_rotation(now: T::BlockNumber) -> Option { Self::next_expected_epoch_change(now) } diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index 58e2af873fd91..a26f6f7f70c56 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -37,8 +37,9 @@ use sp_consensus_babe::{AuthorityId, AuthorityPair, SlotNumber}; use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof}; use sp_staking::SessionIndex; use pallet_staking::EraIndex; +use sp_election_providers::onchain; -impl_outer_origin!{ +impl_outer_origin! { pub enum Origin for Test where system = frame_system {} } @@ -179,6 +180,13 @@ parameter_types! { pub const StakingUnsignedPriority: u64 = u64::max_value() / 2; } +impl onchain::Config for Test { + type AccountId = ::AccountId; + type BlockNumber = ::BlockNumber; + type Accuracy = Perbill; + type DataProvider = Staking; +} + impl pallet_staking::Config for Test { type RewardRemainder = (); type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; @@ -201,6 +209,7 @@ impl pallet_staking::Config for Test { type MaxIterations = (); type MinSolutionScoreBump = (); type OffchainSolutionWeightLimit = (); + type ElectionProvider = onchain::OnChainSequentialPhragmen; type WeightInfo = (); } diff --git a/frame/election-provider-multi-phase/Cargo.toml b/frame/election-provider-multi-phase/Cargo.toml new file mode 100644 index 0000000000000..c70e68cd36faa --- /dev/null +++ b/frame/election-provider-multi-phase/Cargo.toml @@ -0,0 +1,67 @@ +[package] +name = "pallet-election-provider-multi-phase" +version = "2.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "PALLET two phase election providers" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +static_assertions = "1.1.0" +serde = { version = "1.0.101", optional = true } +codec = { package = "parity-scale-codec", version = "1.3.4", default-features = false, features = ["derive"] } + +frame-support = { version = "2.0.0", default-features = false, path = "../support" } +frame-system = { version = "2.0.0", default-features = false, path = "../system" } + +sp-io ={ version = "2.0.0", default-features = false, path = "../../primitives/io" } +sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } +sp-npos-elections = { version = "2.0.0", default-features = false, path = "../../primitives/npos-elections" } +sp-arithmetic = { version = "2.0.0", default-features = false, path = "../../primitives/arithmetic" } +sp-election-providers = { version = "2.0.0", default-features = false, path = "../../primitives/election-providers" } + +# Optional imports for benchmarking +frame-benchmarking = { version = "2.0.0", default-features = false, path = "../benchmarking", optional = true } +rand = { version = "0.7.3", default-features = false, optional = true, features = ["alloc", "small_rng"] } + +[dev-dependencies] +sp-io = { version = "2.0.0", path = "../../primitives/io" } +hex-literal = "0.3.1" +pallet-balances = { version = "2.0.0", path = "../balances" } +sp-core = { version = "2.0.0", path = "../../primitives/core" } +paste = "1.0.3" +substrate-test-utils = { version = "2.0.0", path = "../../test-utils" } +parking_lot = "0.11.0" +sp-tracing = { version = "2.0.0", path = "../../primitives/tracing" } +rand = { version = "0.7.3" } +frame-benchmarking = { path = "../benchmarking" } +sp-election-providers = { version = "2.0.0", features = ["runtime-benchmarks"], path = "../../primitives/election-providers" } + + +[features] +default = ["std"] +std = [ + "serde", + "codec/std", + + "frame-support/std", + "frame-system/std", + + "sp-io/std", + "sp-std/std", + "sp-runtime/std", + "sp-npos-elections/std", + "sp-arithmetic/std", + "sp-election-providers/std", +] +runtime-benchmarks = [ + "frame-benchmarking", + "rand", +] diff --git a/frame/election-provider-multi-phase/src/benchmarking.rs b/frame/election-provider-multi-phase/src/benchmarking.rs new file mode 100644 index 0000000000000..a7a4eed852850 --- /dev/null +++ b/frame/election-provider-multi-phase/src/benchmarking.rs @@ -0,0 +1,283 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Two phase election pallet benchmarking. + +use super::*; +use crate::Module as TwoPhase; + +pub use frame_benchmarking::{account, benchmarks, whitelist_account, whitelisted_caller}; +use frame_support::{assert_ok, traits::OnInitialize}; +use frame_system::RawOrigin; +use rand::{prelude::SliceRandom, rngs::SmallRng, SeedableRng}; +use sp_election_providers::Assignment; +use sp_npos_elections::ExtendedBalance; +use sp_runtime::InnerOf; +use sp_arithmetic::traits::One; +use sp_std::convert::TryInto; + +const SEED: u32 = 0; + +/// Creates a **valid** solution with exactly the given size. +/// +/// The snapshot is also created internally. +fn solution_with_size( + size: SolutionSize, + active_voters_count: u32, + desired_targets: u32, +) -> RawSolution> +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, + > as sp_std::convert::TryFrom>::Error: sp_std::fmt::Debug, +{ + assert!(size.targets >= desired_targets, "must have enough targets"); + assert!( + size.targets >= (>::LIMIT * 2) as u32, + "must have enough targets for unique votes." + ); + assert!(size.voters >= active_voters_count, "must have enough voters"); + assert!( + (>::LIMIT as u32) < desired_targets, + "must have enough winners to give them votes." + ); + + let ed: VoteWeight = T::Currency::minimum_balance().saturated_into::(); + let stake: VoteWeight = ed.max(One::one()).saturating_mul(100); + + // first generates random targets. + let targets: Vec = + (0..size.targets).map(|i| account("Targets", i, SEED)).collect(); + + let mut rng = SmallRng::seed_from_u64(999u64); + + // decide who are the winners. + let winners = targets + .as_slice() + .choose_multiple(&mut rng, desired_targets as usize) + .cloned() + .collect::>(); + + // first generate active voters who must vote for a subset of winners. + let active_voters = (0..active_voters_count) + .map(|i| { + // chose a random subset of winners. + let winner_votes = winners + .as_slice() + .choose_multiple(&mut rng, >::LIMIT) + .cloned() + .collect::>(); + let voter = account::("Voter", i, SEED); + (voter, stake, winner_votes) + }) + .collect::>(); + + // rest of the voters. They can only vote for non-winners. + let non_winners = + targets.iter().filter(|t| !winners.contains(t)).cloned().collect::>(); + let rest_voters = (active_voters_count..size.voters) + .map(|i| { + let votes = (&non_winners) + .choose_multiple(&mut rng, >::LIMIT) + .cloned() + .collect::>(); + let voter = account::("Voter", i, SEED); + (voter, stake, votes) + }) + .collect::>(); + + let mut all_voters = active_voters.clone(); + all_voters.extend(rest_voters); + all_voters.shuffle(&mut rng); + + assert_eq!(active_voters.len() as u32, active_voters_count); + assert_eq!(all_voters.len() as u32, size.voters); + assert_eq!(winners.len() as u32, desired_targets); + + >::put(RoundSnapshotMetadata { + voters_len: all_voters.len() as u32, + targets_len: targets.len() as u32, + }); + >::put(desired_targets); + >::put(RoundSnapshot { voters: all_voters.clone(), targets: targets.clone() }); + + // write the snapshot to staking or whoever is the data provider. + T::DataProvider::put_snapshot(all_voters.clone(), targets.clone()); + + let cache = helpers::generate_voter_cache::(&all_voters); + let stake_of = helpers::stake_of_fn::(&all_voters, &cache); + let voter_index = helpers::voter_index_fn::(&cache); + let target_index = helpers::target_index_fn_linear::(&targets); + let voter_at = helpers::voter_at_fn::(&all_voters); + let target_at = helpers::target_at_fn::(&targets); + + let assignments = active_voters + .iter() + .map(|(voter, _stake, votes)| { + let percent_per_edge: InnerOf> = + (100 / votes.len()).try_into().unwrap(); + Assignment { + who: voter.clone(), + distribution: votes + .iter() + .map(|t| (t.clone(), >::from_percent(percent_per_edge))) + .collect::>(), + } + }) + .collect::>(); + + let compact = + >::from_assignment(assignments, &voter_index, &target_index).unwrap(); + let score = compact.clone().score(&winners, stake_of, voter_at, target_at).unwrap(); + let round = >::round(); + RawSolution { compact, score, round } +} + +benchmarks! { + where_clause { + where ExtendedBalance: From>>, + > as sp_std::convert::TryFrom>::Error: sp_std::fmt::Debug, + ExtendedBalance: From>>, + } + + on_initialize_nothing { + assert!(>::current_phase().is_off()); + }: { + >::on_initialize(1u32.into()); + } verify { + assert!(>::current_phase().is_off()); + } + + on_initialize_open_signed { + // NOTE: this benchmark currently doesn't have any components because the length of a db + // read/write is not captured. Otherwise, it is quite influenced by how much data + // `T::ElectionDataProvider` is reading and passing on. + assert!(>::snapshot().is_none()); + assert!(>::current_phase().is_off()); + }: { + >::on_initialize_open_signed(); + } verify { + assert!(>::snapshot().is_some()); + assert!(>::current_phase().is_signed()); + } + + on_initialize_open_unsigned_with_snapshot { + assert!(>::snapshot().is_none()); + assert!(>::current_phase().is_off()); + }: { + >::on_initialize_open_unsigned(true, true, 1u32.into()); + } verify { + assert!(>::snapshot().is_some()); + assert!(>::current_phase().is_unsigned()); + } + + on_initialize_open_unsigned_without_snapshot { + // need to assume signed phase was open before + >::on_initialize_open_signed(); + assert!(>::snapshot().is_some()); + assert!(>::current_phase().is_signed()); + }: { + >::on_initialize_open_unsigned(false, true, 1u32.into()); + } verify { + assert!(>::snapshot().is_some()); + assert!(>::current_phase().is_unsigned()); + } + + #[extra] + create_snapshot { + assert!(>::snapshot().is_none()); + }: { + >::create_snapshot() + } verify { + assert!(>::snapshot().is_some()); + } + + submit_unsigned { + // number of votes in snapshot. + let v in (T::BenchmarkingConfig::VOTERS[0]) .. T::BenchmarkingConfig::VOTERS[1]; + // number of targets in snapshot. + let t in (T::BenchmarkingConfig::TARGETS[0]) .. T::BenchmarkingConfig::TARGETS[1]; + // number of assignments, i.e. compact.len(). This means the active nominators, thus must be + // a subset of `v` component. + let a in (T::BenchmarkingConfig::ACTIVE_VOTERS[0]) .. T::BenchmarkingConfig::ACTIVE_VOTERS[1]; + // number of desired targets. Must be a subset of `t` component. + let d in (T::BenchmarkingConfig::DESIRED_TARGETS[0]) .. T::BenchmarkingConfig::DESIRED_TARGETS[1]; + + let witness = SolutionSize { voters: v, targets: t }; + let raw_solution = solution_with_size::(witness, a, d); + + assert!(>::queued_solution().is_none()); + >::put(Phase::Unsigned((true, 1u32.into()))); + }: _(RawOrigin::None, raw_solution, witness) + verify { + assert!(>::queued_solution().is_some()); + } + + // This is checking a valid solution. The worse case is indeed a valid solution. + feasibility_check { + // number of votes in snapshot. + let v in (T::BenchmarkingConfig::VOTERS[0]) .. T::BenchmarkingConfig::VOTERS[1]; + // number of targets in snapshot. + let t in (T::BenchmarkingConfig::TARGETS[0]) .. T::BenchmarkingConfig::TARGETS[1]; + // number of assignments, i.e. compact.len(). This means the active nominators, thus must be + // a subset of `v` component. + let a in (T::BenchmarkingConfig::ACTIVE_VOTERS[0]) .. T::BenchmarkingConfig::ACTIVE_VOTERS[1]; + // number of desired targets. Must be a subset of `t` component. + let d in (T::BenchmarkingConfig::DESIRED_TARGETS[0]) .. T::BenchmarkingConfig::DESIRED_TARGETS[1]; + + let size = SolutionSize { voters: v, targets: t }; + let raw_solution = solution_with_size::(size, a, d); + + assert_eq!(raw_solution.compact.voter_count() as u32, a); + assert_eq!(raw_solution.compact.unique_targets().len() as u32, d); + }: { + assert_ok!(>::feasibility_check(raw_solution, ElectionCompute::Unsigned)); + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::mock::*; + + #[test] + fn test_benchmarks() { + ExtBuilder::default().build_and_execute(|| { + assert_ok!(test_benchmark_feasibility_check::()); + }); + + ExtBuilder::default().build_and_execute(|| { + assert_ok!(test_benchmark_submit_unsigned::()); + }); + + ExtBuilder::default().build_and_execute(|| { + assert_ok!(test_benchmark_on_initialize_open_unsigned_with_snapshot::()); + }); + + ExtBuilder::default().build_and_execute(|| { + assert_ok!(test_benchmark_on_initialize_open_unsigned_without_snapshot::()); + }); + + ExtBuilder::default().build_and_execute(|| { + assert_ok!(test_benchmark_on_initialize_nothing::()); + }); + + ExtBuilder::default().build_and_execute(|| { + assert_ok!(test_benchmark_create_snapshot::()); + }); + } +} diff --git a/frame/election-provider-multi-phase/src/helpers.rs b/frame/election-provider-multi-phase/src/helpers.rs new file mode 100644 index 0000000000000..da4a092653ae7 --- /dev/null +++ b/frame/election-provider-multi-phase/src/helpers.rs @@ -0,0 +1,199 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Some helper functions/macros for this crate. + +use super::{ + Config, VoteWeight, CompactVoterIndexOf, CompactTargetIndexOf, CompactAccuracyOf, + OnChainAccuracyOf, ExtendedBalance, +}; +use sp_runtime::InnerOf; +use sp_std::{collections::btree_map::BTreeMap, convert::TryInto, boxed::Box, prelude::*}; + +#[macro_export] +macro_rules! log { + ($level:tt, $patter:expr $(, $values:expr)* $(,)?) => { + frame_support::debug::$level!( + target: $crate::LOG_TARGET, + concat!("🏦 ", $patter) $(, $values)* + ) + }; +} + +/// Generate a btree-map cache of the voters and their indices. +/// +/// This can be used to efficiently build index getter closures. +pub fn generate_voter_cache( + snapshot: &Vec<(T::AccountId, VoteWeight, Vec)>, +) -> BTreeMap +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + let mut cache: BTreeMap = BTreeMap::new(); + snapshot.iter().enumerate().for_each(|(i, (x, _, _))| { + let _existed = cache.insert(x.clone(), i); + // if a duplicate exists, we only consider the last one. Defensive only, should never + // happen. + debug_assert!(_existed.is_none()); + }); + + cache +} + +/// Create a function the returns the index a voter in the snapshot. +/// +/// The returning index type is the same as the one defined in [`T::CompactSolution::Voter`]. +/// +/// ## Warning +/// +/// The snapshot must be the same is the one used to create `cache`. +pub fn voter_index_fn( + cache: &BTreeMap, +) -> Box Option> + '_> +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + Box::new(move |who| { + cache.get(who).and_then(|i| >>::try_into(*i).ok()) + }) +} + +/// Same as [`voter_index_fn`], but the returning index is converted into usize, if possible. +/// +/// ## Warning +/// +/// The snapshot must be the same is the one used to create `cache`. +pub fn voter_index_fn_usize( + cache: &BTreeMap, +) -> Box Option + '_> +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + Box::new(move |who| cache.get(who).cloned()) +} + +/// A non-optimized, linear version of [`voter_index_fn`] that does not need a cache and does a +/// linear search. +/// +/// ## Warning +/// +/// Not meant to be used in production. +pub fn voter_index_fn_linear( + snapshot: &Vec<(T::AccountId, VoteWeight, Vec)>, +) -> Box Option> + '_> +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + Box::new(move |who| { + snapshot + .iter() + .position(|(x, _, _)| x == who) + .and_then(|i| >>::try_into(i).ok()) + }) +} + +/// Create a function the returns the index a targets in the snapshot. +/// +/// The returning index type is the same as the one defined in [`T::CompactSolution::Target`]. +pub fn target_index_fn_linear( + snapshot: &Vec, +) -> Box Option> + '_> +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + Box::new(move |who| { + snapshot + .iter() + .position(|x| x == who) + .and_then(|i| >>::try_into(i).ok()) + }) +} + +/// Create a function that can map a voter index ([`CompactVoterIndexOf`]) to the actual voter +/// account using a linearly indexible snapshot. +pub fn voter_at_fn( + snapshot: &Vec<(T::AccountId, VoteWeight, Vec)>, +) -> Box) -> Option + '_> +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + Box::new(move |i| { + as TryInto>::try_into(i) + .ok() + .and_then(|i| snapshot.get(i).map(|(x, _, _)| x).cloned()) + }) +} + +/// Create a function that can map a target index ([`CompactTargetIndexOf`]) to the actual target +/// account using a linearly indexible snapshot. +pub fn target_at_fn( + snapshot: &Vec, +) -> Box) -> Option + '_> +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + Box::new(move |i| { + as TryInto>::try_into(i) + .ok() + .and_then(|i| snapshot.get(i).cloned()) + }) +} + +/// Create a function to get the stake of a voter. +/// +/// This is not optimized and uses a linear search. +pub fn stake_of_fn_linear( + snapshot: &Vec<(T::AccountId, VoteWeight, Vec)>, +) -> Box VoteWeight + '_> +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + Box::new(move |who| { + snapshot.iter().find(|(x, _, _)| x == who).map(|(_, x, _)| *x).unwrap_or_default() + }) +} + +/// Create a function to get the stake of a voter. +/// +/// ## Warning +/// +/// The cache need must be derived from the same snapshot. Zero is returned if a voter is +/// non-existent. +pub fn stake_of_fn<'a, T: Config>( + snapshot: &'a Vec<(T::AccountId, VoteWeight, Vec)>, + cache: &'a BTreeMap, +) -> Box VoteWeight + 'a> +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + Box::new(move |who| { + if let Some(index) = cache.get(who) { + snapshot.get(*index).map(|(_, x, _)| x).cloned().unwrap_or_default() + } else { + 0 + } + }) +} diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs new file mode 100644 index 0000000000000..79112297f869f --- /dev/null +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -0,0 +1,1495 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Two phase, offchain election provider pallet. +//! +//! As the name suggests, this election-provider has two distinct phases (see [`Phase`]), signed and +//! unsigned. +//! +//! ## Phases +//! +//! The timeline of pallet is as follows. At each block, +//! [`sp_election_providers::ElectionDataProvider::next_election_prediction`] is used to estimate +//! the time remaining to the next call to [`sp_election_providers::ElectionProvider::elect`]. Based +//! on this, a phase is chosen. The timeline is as follows. +//! +//! ```ignore +//! elect() +//! + <--T::SignedPhase--> + <--T::UnsignedPhase--> + +//! +-------------------------------------------------------------------+ +//! Phase::Off + Phase::Signed + Phase::Unsigned + +//! ``` +//! +//! Note that the unsigned phase starts [`pallet::Config::UnsignedPhase`] blocks before the +//! `next_election_prediction`, but only ends when a call to [`ElectionProvider::elect`] happens. +//! +//! > Given this, it is rather important for the user of this pallet to ensure it always terminates +//! election via `elect` before requesting a new one. +//! +//! Each of the phases can be disabled by essentially setting their length to zero. If both phases +//! have length zero, then the pallet essentially runs only the on-chain backup. +//! +//! ### Signed Phase +//! +//! In the signed phase, solutions (of type [`RawSolution`]) are submitted and queued on chain. A +//! deposit is reserved, based on the size of the solution, for the cost of keeping this solution +//! on-chain for a number of blocks, and the potential weight of the solution upon being checked. A +//! maximum of [`pallet::Config::MaxSignedSubmissions`] solutions are stored. The queue is always +//! sorted based on score (worse to best). +//! +//! Upon arrival of a new solution: +//! +//! 1. If the queue is not full, it is stored in the appropriate sorted index. +//! 2. If the queue is full but the submitted solution is better than one of the queued ones, the +//! worse solution is discarded, the bond of the outgoing solution is returned, and the new +//! solution is stored in the correct index. +//! 3. If the queue is full and the solution is not an improvement compared to any of the queued +//! ones, it is instantly rejected and no additional bond is reserved. +//! +//! A signed solution cannot be reversed, taken back, updated, or retracted. In other words, the +//! origin can not bail out in any way, if their solution is queued. +//! +//! Upon the end of the signed phase, the solutions are examined from best to worse (i.e. `pop()`ed +//! until drained). Each solution undergoes an expensive [`Pallet::feasibility_check`], which +//! ensures the score claimed by this score was correct, and it is valid based on the election data +//! (i.e. votes and candidates). At each step, if the current best solution passes the feasibility +//! check, it is considered to be the best one. The sender of the origin is rewarded, and the rest +//! of the queued solutions get their deposit back and are discarded, without being checked. +//! +//! The following example covers all of the cases at the end of the signed phase: +//! +//! ```ignore +//! Queue +//! +-------------------------------+ +//! |Solution(score=20, valid=false)| +--> Slashed +//! +-------------------------------+ +//! |Solution(score=15, valid=true )| +--> Rewarded, Saved +//! +-------------------------------+ +//! |Solution(score=10, valid=true )| +--> Discarded +//! +-------------------------------+ +//! |Solution(score=05, valid=false)| +--> Discarded +//! +-------------------------------+ +//! | None | +//! +-------------------------------+ +//! ``` +//! +//! Note that both of the bottom solutions end up being discarded and get their deposit back, +//! despite one of them being *invalid*. +//! +//! ## Unsigned Phase +//! +//! The unsigned phase will always follow the signed phase, with the specified duration. In this +//! phase, only validator nodes can submit solutions. A validator node who has offchain workers +//! enabled will start to mine a solution in this phase and submits it back to the chain as an +//! unsigned transaction, thus the name _unsigned_ phase. This unsigned transaction can never be +//! valid if propagated, and it acts similar to an inherent. +//! +//! Validators will only submit solutions if the one that they have computed is sufficiently better +//! than the best queued one (see [`pallet::Config::SolutionImprovementThreshold`]) and will limit +//! the weigh of the solution to [`pallet::Config::MinerMaxWeight`]. +//! +//! ### Fallback +//! +//! If we reach the end of both phases (i.e. call to [`ElectionProvider::elect`] happens) and no +//! good solution is queued, then the fallback strategy [`pallet::Config::Fallback`] is used to +//! determine what needs to be done. The on-chain election is slow, and contains no balancing or +//! reduction post-processing. See [`onchain::OnChainSequentialPhragmen`]. The +//! [`FallbackStrategy::Nothing`] should probably only be used for testing, and returns an error. +//! +//! ## Feasible Solution (correct solution) +//! +//! All submissions must undergo a feasibility check. Signed solutions are checked on by one at the +//! end of the signed phase, and the unsigned solutions are checked on the spot. A feasible solution +//! is as follows: +//! +//! 0. **all** of the used indices must be correct. +//! 1. present *exactly* correct number of winners. +//! 2. any assignment is checked to match with [`RoundSnapshot::voters`]. +//! 3. the claimed score is valid, based on the fixed point arithmetic accuracy. +//! +//! ## Accuracy +//! +//! The accuracy of the election is configured via two trait parameters. namely, +//! [`OnChainAccuracyOf`] dictates the accuracy used to compute the on-chain fallback election and +//! [`CompactAccuracyOf`] is the accuracy that the submitted solutions must adhere to. +//! +//! Note that both accuracies are of great importance. The offchain solution should be as small as +//! possible, reducing solutions size/weight. The on-chain solution can use more space for accuracy, +//! but should still be fast to prevent massively large blocks in case of a fallback. +//! +//! ## Future Plans +//! +//! **Challenge Phase**. We plan adding a third phase to the pallet, called the challenge phase. +//! This is phase in which no further solutions are processed, and the current best solution might +//! be challenged by anyone (signed or unsigned). The main plan here is to enforce the solution to +//! be PJR. Checking PJR on-chain is quite expensive, yet proving that a solution is **not** PJR is +//! rather cheap. If a queued solution is challenged: +//! +//! 1. We must surely slash whoever submitted that solution (might be a challenge for unsigned +//! solutions). +//! 2. It is probably fine to fallback to the on-chain election, as we expect this to happen rarely. +//! +//! **Bailing out**. The functionality of bailing out of a queued solution is nice. A miner can +//! submit a solution as soon as they _think_ it is high probability feasible, and do the checks +//! afterwards, and remove their solution (for a small cost of probably just transaction fees, or a +//! portion of the bond). +//! +//! **Conditionally open unsigned phase**: Currently, the unsigned phase is always opened. This is +//! useful because an honest validation will run our OCW code, which should be good enough to trump +//! a mediocre or malicious signed submission (assuming in the absence of honest signed bots). If an +//! when the signed submissions are checked against an absolute measure (e.g. PJR), then we can only +//! open the unsigned phase in extreme conditions (i.e. "not good signed solution received") to +//! spare some work in the validators +//! +//! **Allow smaller solutions and build up**: For now we only allow solutions that are exactly +//! [`DesiredTargets`], no more, no less. Over time, we can change this to a [min, max] where any +//! solution within this range is acceptable, where bigger solutions are prioritized. +//! +//! **Recursive Fallback**: Currently, the fallback is a separate enum. A different and fancier way +//! of doing this would be to have the fallback be another +//! [`sp_election_providers::ElectionProvider`]. In this case, this pallet can even have the +//! on-chain election provider as fallback, or special _noop_ fallback that simply returns an error, +//! thus replicating [`FallbackStrategy::Nothing`]. +//! +//! **Score based on size**: We should always prioritize small solutions over bigger ones, if there +//! is a tie. Even more harsh should be to enforce the bound of the `reduce` algorithm. + +#![cfg_attr(not(feature = "std"), no_std)] + +use codec::{Decode, Encode, HasCompact}; +use frame_support::{ + dispatch::DispatchResultWithPostInfo, + ensure, + traits::{Currency, Get, ReservableCurrency}, + weights::Weight, +}; +use frame_system::{ensure_none, ensure_signed, offchain::SendTransactionTypes}; +use sp_election_providers::{ElectionDataProvider, ElectionProvider, onchain}; +use sp_npos_elections::{ + assignment_ratio_to_staked_normalized, is_score_better, CompactSolution, ElectionScore, + EvaluateSupport, ExtendedBalance, PerThing128, Supports, VoteWeight, +}; +use sp_runtime::{ + transaction_validity::{ + InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity, + TransactionValidityError, ValidTransaction, + }, + DispatchError, InnerOf, PerThing, Perbill, RuntimeDebug, SaturatedConversion, +}; +use sp_std::prelude::*; +use sp_arithmetic::{ + UpperOf, + traits::{Zero, CheckedAdd}, +}; + +#[cfg(any(feature = "runtime-benchmarks", test))] +mod benchmarking; +#[cfg(test)] +mod mock; +#[macro_use] +pub mod helpers; + +const LOG_TARGET: &'static str = "election-provider"; + +// for the helper macros +#[doc(hidden)] +pub use sp_runtime::traits::UniqueSaturatedInto; +#[doc(hidden)] +pub use sp_std; + +pub mod unsigned; +pub mod weights; + +use weights::WeightInfo; + +// pub mod signed; +// use signed::SignedSubmission; + +/// The compact solution type used by this crate. +pub type CompactOf = ::CompactSolution; + +/// The voter index. Derived from [`CompactOf`]. +pub type CompactVoterIndexOf = as CompactSolution>::Voter; +/// The target index. Derived from [`CompactOf`]. +pub type CompactTargetIndexOf = as CompactSolution>::Target; +/// The accuracy of the election, when submitted from offchain. Derived from [`CompactOf`]. +pub type CompactAccuracyOf = as CompactSolution>::Accuracy; +/// The accuracy of the election, when computed on-chain. Equal to [`Config::OnChainAccuracy`]. +pub type OnChainAccuracyOf = ::OnChainAccuracy; + +struct OnChainConfig(sp_std::marker::PhantomData) +where + ExtendedBalance: From>>, + ExtendedBalance: From>>; +impl onchain::Config for OnChainConfig +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + type AccountId = T::AccountId; + type BlockNumber = T::BlockNumber; + type Accuracy = T::OnChainAccuracy; + type DataProvider = T::DataProvider; +} + +/// Configuration for the benchmarks of the pallet. +pub trait BenchmarkingConfig { + /// Range of voters. + const VOTERS: [u32; 2]; + /// Range of targets. + const TARGETS: [u32; 2]; + /// Range of active voters. + const ACTIVE_VOTERS: [u32; 2]; + /// Range of desired targets. + const DESIRED_TARGETS: [u32; 2]; +} + +impl BenchmarkingConfig for () { + const VOTERS: [u32; 2] = [4000, 6000]; + const TARGETS: [u32; 2] = [1000, 1600]; + const ACTIVE_VOTERS: [u32; 2] = [1000, 3000]; + const DESIRED_TARGETS: [u32; 2] = [400, 800]; +} + +/// Current phase of the pallet. +#[derive(PartialEq, Eq, Clone, Copy, Encode, Decode, RuntimeDebug)] +pub enum Phase { + /// Nothing, the election is not happening. + Off, + /// Signed phase is open. + Signed, + /// Unsigned phase. First element is whether it is open or not, second the starting block + /// number. + Unsigned((bool, Bn)), +} + +impl Default for Phase { + fn default() -> Self { + Phase::Off + } +} + +impl Phase { + /// Weather the phase is signed or not. + pub fn is_signed(&self) -> bool { + matches!(self, Phase::Signed) + } + + /// Weather the phase is unsigned or not. + pub fn is_unsigned(&self) -> bool { + matches!(self, Phase::Unsigned(_)) + } + + /// Weather the phase is unsigned and open or not, with specific start. + pub fn is_unsigned_open_at(&self, at: Bn) -> bool { + matches!(self, Phase::Unsigned((true, real)) if *real == at) + } + + /// Weather the phase is unsigned and open or not. + pub fn is_unsigned_open(&self) -> bool { + matches!(self, Phase::Unsigned((true, _))) + } + + /// Weather the phase is off or not. + pub fn is_off(&self) -> bool { + matches!(self, Phase::Off) + } +} + +/// A configuration for the module to indicate what should happen in the case of a fallback i.e. +/// reaching a call to `elect` with no good solution. +#[cfg_attr(test, derive(Clone))] +pub enum FallbackStrategy { + /// Run a on-chain sequential phragmen. + /// + /// This might burn the chain for a few minutes due to a stall, but is generally a safe + /// approach to maintain a sensible validator set. + OnChain, + /// Nothing. Return an error. + Nothing, +} + +/// The type of `Computation` that provided this election data. +#[derive(PartialEq, Eq, Clone, Copy, Encode, Decode, RuntimeDebug)] +pub enum ElectionCompute { + /// Election was computed on-chain. + OnChain, + /// Election was computed with a signed submission. + Signed, + /// Election was computed with an unsigned submission. + Unsigned, +} + +impl Default for ElectionCompute { + fn default() -> Self { + ElectionCompute::OnChain + } +} + +/// A raw, unchecked solution. +/// +/// This is what will get submitted to the chain. +/// +/// Such a solution should never become effective in anyway before being checked by the +/// [`Pallet::feasibility_check`] +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] +pub struct RawSolution { + /// Compact election edges. + compact: C, + /// The _claimed_ score of the solution. + score: ElectionScore, + /// The round at which this solution should be submitted. + round: u32, +} + +impl Default for RawSolution { + fn default() -> Self { + // Round 0 is always invalid, only set this to 1. + Self { round: 1, compact: Default::default(), score: Default::default() } + } +} + +/// A checked solution, ready to be enacted. +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, Default)] +pub struct ReadySolution { + /// The final supports of the solution. + /// + /// This is target-major vector, storing each winners, total backing, and each individual + /// backer. + supports: Supports, + /// The score of the solution. + /// + /// This is needed to potentially challenge the solution. + score: ElectionScore, + /// How this election was computed. + compute: ElectionCompute, +} + +/// Solution size of the election. +/// +/// This is needed for proper weight calculation. +#[derive(PartialEq, Eq, Clone, Copy, Encode, Decode, RuntimeDebug, Default)] +pub struct SolutionSize { + /// Number of all voters. + /// + /// This must match the on-chain snapshot. + #[codec(compact)] + voters: u32, + /// Number of all targets. + /// + /// This must match the on-chain snapshot. + #[codec(compact)] + targets: u32, +} + +/// A snapshot of all the data that is needed for en entire round. They are provided by +/// [`ElectionDataProvider`] at the beginning of the signed phase (or the unsigned phase, if signed +/// phase is non-existent) and are kept around until the round is finished. +/// +/// These are stored together because they are often times accessed together. +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, Default)] +pub struct RoundSnapshot { + /// All of the voters. + pub voters: Vec<(A, VoteWeight, Vec)>, + /// All of the targets. + pub targets: Vec, +} + +/// Some metadata related to snapshot. +/// +/// In this pallet, there are cases where we want to read the whole snapshot (voters, targets, +/// desired), and cases that we are interested in just the length of these values. The former favors +/// the snapshot to be stored in one struct (as it is now) while the latter prefers them to be +/// separate to enable the use of `decode_len`. This approach is a middle ground, storing the +/// snapshot as one struct, whilst storing the lengths separately. +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, Default)] +pub struct RoundSnapshotMetadata { + /// The length of voters. + voters_len: u32, + /// The length of targets. + targets_len: u32, +} + +/// Internal errors of the pallet. +/// +/// Note that this is different from [`pallet::Error`]. +#[derive(RuntimeDebug, Eq, PartialEq)] +pub enum ElectionError { + /// A feasibility error. + Feasibility(FeasibilityError), + /// An error in the on-chain fallback. + OnChainFallback(onchain::Error), + /// No fallback is configured + NoFallbackConfigured, + /// An internal error in the NPoS elections crate. + NposElections(sp_npos_elections::Error), + /// Snapshot data was unavailable unexpectedly. + SnapshotUnAvailable, + /// Submitting a transaction to the pool failed. + /// + /// This can only happen in the unsigned phase. + PoolSubmissionFailed, +} + +impl From for ElectionError { + fn from(e: onchain::Error) -> Self { + ElectionError::OnChainFallback(e) + } +} + +impl From for ElectionError { + fn from(e: sp_npos_elections::Error) -> Self { + ElectionError::NposElections(e) + } +} + +impl From for ElectionError { + fn from(e: FeasibilityError) -> Self { + ElectionError::Feasibility(e) + } +} + +/// Errors that can happen in the feasibility check. +#[derive(RuntimeDebug, Eq, PartialEq)] +pub enum FeasibilityError { + /// Wrong number of winners presented. + WrongWinnerCount, + /// The snapshot is not available. + /// + /// This must be an internal error of the chain. + SnapshotUnavailable, + /// Internal error from the election crate. + NposElection(sp_npos_elections::Error), + /// A vote is invalid. + InvalidVote, + /// A voter is invalid. + InvalidVoter, + /// A winner is invalid. + InvalidWinner, + /// The given score was invalid. + InvalidScore, + /// The provided round is incorrect. + InvalidRound, +} + +impl From for FeasibilityError { + fn from(e: sp_npos_elections::Error) -> Self { + FeasibilityError::NposElection(e) + } +} + +pub use pallet::*; +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::config] + pub trait Config: frame_system::Config + SendTransactionTypes> + where + ExtendedBalance: From>>, + ExtendedBalance: From>>, + { + type Event: From> + + Into<::Event> + + IsType<::Event>; + + /// Currency type. + type Currency: ReservableCurrency + Currency; + + /// Duration of the unsigned phase. + #[pallet::constant] + type UnsignedPhase: Get; + /// Duration of the signed phase. + #[pallet::constant] + type SignedPhase: Get; + + /// The minimum amount of improvement to the solution score that defines a solution as + /// "better". + #[pallet::constant] + type SolutionImprovementThreshold: Get; + + /// The priority of the unsigned transaction submitted in the unsigned-phase + type UnsignedPriority: Get; + /// Maximum number of iteration of balancing that will be executed in the embedded miner of + /// the pallet. + type MinerMaxIterations: Get; + /// Maximum weight that the miner should consume. + /// + /// The miner will ensure that the total weight of the unsigned solution will not exceed + /// this values, based on [`WeightInfo::submit_unsigned`]. + type MinerMaxWeight: Get; + + /// Something that will provide the election data. + type DataProvider: ElectionDataProvider; + + /// The compact solution type + type CompactSolution: codec::Codec + + Default + + PartialEq + + Eq + + Clone + + sp_std::fmt::Debug + + CompactSolution; + + /// Accuracy used for fallback on-chain election. + type OnChainAccuracy: PerThing128; + + /// Configuration for the fallback + type Fallback: Get; + + /// The configuration of benchmarking. + type BenchmarkingConfig: BenchmarkingConfig; + + /// The weight of the pallet. + type WeightInfo: WeightInfo; + } + + #[pallet::hooks] + impl Hooks> for Pallet + where + ExtendedBalance: From>>, + ExtendedBalance: From>>, + { + fn on_initialize(now: T::BlockNumber) -> Weight { + let next_election = T::DataProvider::next_election_prediction(now).max(now); + + let signed_deadline = T::SignedPhase::get() + T::UnsignedPhase::get(); + let unsigned_deadline = T::UnsignedPhase::get(); + + let remaining = next_election - now; + let current_phase = Self::current_phase(); + + match current_phase { + Phase::Off if remaining <= signed_deadline && remaining > unsigned_deadline => { + Self::on_initialize_open_signed(); + log!(info, "Starting signed phase at #{:?} , round {}.", now, Self::round()); + T::WeightInfo::on_initialize_open_signed() + } + Phase::Signed | Phase::Off + if remaining <= unsigned_deadline && remaining > 0u32.into() => + { + let (need_snapshot, enabled, additional) = if current_phase == Phase::Signed { + // followed by a signed phase: close the signed phase, no need for snapshot. + // TODO + (false, true, Weight::zero()) + } else { + // no signed phase + (true, true, Weight::zero()) + }; + + Self::on_initialize_open_unsigned(need_snapshot, enabled, now); + log!(info, "Starting unsigned phase({}) at #{:?}.", enabled, now); + let base_weight = if need_snapshot { + T::WeightInfo::on_initialize_open_unsigned_with_snapshot() } + else { + T::WeightInfo::on_initialize_open_unsigned_without_snapshot() + }; + base_weight.saturating_add(additional) + } + _ => T::WeightInfo::on_initialize_nothing(), + } + } + + fn offchain_worker(n: T::BlockNumber) { + // We only run the OCW in the fist block of the unsigned phase. + if Self::current_phase().is_unsigned_open_at(n) { + match Self::set_check_offchain_execution_status(n) { + Ok(_) => match Self::mine_and_submit() { + Ok(_) => { + log!(info, "successfully submitted a solution via OCW at block {:?}", n) + } + Err(e) => log!(error, "error while submitting transaction in OCW: {:?}", e), + }, + Err(why) => log!(error, "Error in unsigned offchain worker: {:?}", why), + } + } + } + + fn integrity_test() { + use sp_std::mem::size_of; + // The index type of both voters and targets need to be smaller than that of usize (very + // unlikely to be the case, but anyhow). + assert!(size_of::>() <= size_of::()); + assert!(size_of::>() <= size_of::()); + + // ---------------------------- + // based on the requirements of [`sp_npos_elections::Assignment::try_normalize`]. + let max_vote: usize = as CompactSolution>::LIMIT; + + // 1. Maximum sum of [ChainAccuracy; 16] must fit into `UpperOf`.. + let maximum_chain_accuracy: Vec>> = + (0..max_vote).map(|_| >::one().deconstruct().into()).collect(); + let _: UpperOf> = maximum_chain_accuracy + .iter() + .fold(Zero::zero(), |acc, x| acc.checked_add(x).unwrap()); + + // 2. Maximum sum of [CompactAccuracy; 16] must fit into `UpperOf`. + let maximum_chain_accuracy: Vec>> = + (0..max_vote).map(|_| >::one().deconstruct().into()).collect(); + let _: UpperOf> = maximum_chain_accuracy + .iter() + .fold(Zero::zero(), |acc, x| acc.checked_add(x).unwrap()); + } + } + + #[pallet::call] + impl Pallet + where + ExtendedBalance: From>>, + ExtendedBalance: From>>, + { + /// Submit a solution for the unsigned phase. + /// + /// The dispatch origin fo this call must be __none__. + /// + /// This submission is checked on the fly, thus it is likely yo be more limited and smaller. + /// Moreover, this unsigned solution is only validated when submitted to the pool from the + /// local process. Effectively, this means that only active validators can submit this + /// transaction when authoring a block. + /// + /// To prevent any incorrect solution (and thus wasted time/weight), this transaction will + /// panic if the solution submitted by the validator is invalid, effectively putting their + /// authoring reward at risk. + /// + /// No deposit or reward is associated with this. + #[pallet::weight(T::WeightInfo::submit_unsigned( + witness.voters, + witness.targets, + solution.compact.voter_count() as u32, + solution.compact.unique_targets().len() as u32 + ))] + pub fn submit_unsigned( + origin: OriginFor, + solution: RawSolution>, + witness: SolutionSize, + ) -> DispatchResultWithPostInfo { + ensure_none(origin)?; + let error_message = "Invalid unsigned submission must produce invalid block and \ + deprive validator from their authoring reward."; + + // check phase and score. + // NOTE: since we do this in pre-dispatch, we can just ignore it here. + Self::unsigned_pre_dispatch_checks(&solution).expect(error_message); + + // ensure witness was correct. + let RoundSnapshotMetadata { voters_len, targets_len } = + Self::snapshot_metadata().expect(error_message); + + // NOTE: we are asserting, not `ensure`ing -- we want to panic here. + assert!(voters_len as u32 == witness.voters, error_message); + assert!(targets_len as u32 == witness.targets, error_message); + + let ready = + Self::feasibility_check(solution, ElectionCompute::Unsigned).expect(error_message); + + // store the newly received solution. + log!(info, "queued unsigned solution with score {:?}", ready.score); + >::put(ready); + Self::deposit_event(Event::SolutionStored(ElectionCompute::Unsigned)); + + Ok(None.into()) + } + } + + #[pallet::event] + #[pallet::metadata(::AccountId = "AccountId")] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event + where + ExtendedBalance: From>>, + ExtendedBalance: From>>, + { + /// A solution was stored with the given compute. + /// + /// If the solution is signed, this means that it hasn't yet been processed. If the + /// solution is unsigned, this means that it has also been processed. + SolutionStored(ElectionCompute), + /// The election has been finalized, with `Some` of the given computation, or else if the + /// election failed, `None`. + ElectionFinalized(Option), + /// An account has been rewarded for their signed submission being finalized. + Rewarded(::AccountId), + /// An account has been slashed for submitting an invalid signed submission. + Slashed(::AccountId), + /// The signed phase of the given round has started. + SignedPhaseStarted(u32), + /// The unsigned phase of the given round has started. + UnsignedPhaseStarted(u32), + } + + #[pallet::error] + pub enum Error { + /// Submission was too early. + EarlySubmission, + /// Wrong number of winners presented. + WrongWinnerCount, + /// Submission was too weak, score-wise. + WeakSubmission, + /// The queue was full, and the solution was not better than any of the existing ones. + QueueFull, + /// The origin failed to pay the deposit. + CannotPayDeposit, + /// witness data to dispatchable is invalid. + InvalidWitness, + /// The signed submission consumes too much weight + TooMuchWeight, + } + + #[pallet::origin] + pub struct Origin(PhantomData); + + #[pallet::validate_unsigned] + impl ValidateUnsigned for Pallet + where + ExtendedBalance: From>>, + ExtendedBalance: From>>, + { + type Call = Call; + fn validate_unsigned(source: TransactionSource, call: &Self::Call) -> TransactionValidity { + if let Call::submit_unsigned(solution, _) = call { + // discard solution not coming from the local OCW. + match source { + TransactionSource::Local | TransactionSource::InBlock => { /* allowed */ } + _ => { + return InvalidTransaction::Call.into(); + } + } + + let _ = Self::unsigned_pre_dispatch_checks(solution) + .map_err(|err| { + log!(error, "unsigned transaction validation failed due to {:?}", err); + err + }) + .map_err(dispatch_error_to_invalid)?; + + ValidTransaction::with_tag_prefix("OffchainElection") + // The higher the score[0], the better a solution is. + .priority( + T::UnsignedPriority::get() + .saturating_add(solution.score[0].saturated_into()), + ) + // used to deduplicate unsigned solutions: each validator should produce one + // solution per round at most, and solutions are not propagate. + .and_provides(solution.round) + // transaction should stay in the pool for the duration of the unsigned phase. + .longevity(T::UnsignedPhase::get().saturated_into::()) + // We don't propagate this. This can never the validated at a remote node. + .propagate(false) + .build() + } else { + InvalidTransaction::Call.into() + } + } + + fn pre_dispatch(call: &Self::Call) -> Result<(), TransactionValidityError> { + if let Call::submit_unsigned(solution, _) = call { + Self::unsigned_pre_dispatch_checks(solution) + .map_err(dispatch_error_to_invalid) + .map_err(Into::into) + } else { + Err(InvalidTransaction::Call.into()) + } + } + } + + #[pallet::type_value] + pub fn DefaultForRound() -> u32 { + 1 + } + + /// Internal counter for the number of rounds. + /// + /// This is useful for de-duplication of transactions submitted to the pool, and general + /// diagnostics of the module. + /// + /// This is merely incremented once per every time that an upstream `elect` is called. + #[pallet::storage] + #[pallet::getter(fn round)] + pub type Round = StorageValue<_, u32, ValueQuery, DefaultForRound>; + + /// Current phase. + #[pallet::storage] + #[pallet::getter(fn current_phase)] + pub type CurrentPhase = StorageValue<_, Phase, ValueQuery>; + + /// Current best solution, signed or unsigned. + #[pallet::storage] + #[pallet::getter(fn queued_solution)] + pub type QueuedSolution = StorageValue<_, ReadySolution>; + + /// Snapshot data of the round. + /// + /// This is created at the beginning of the signed phase and cleared upon calling `elect`. + #[pallet::storage] + #[pallet::getter(fn snapshot)] + pub type Snapshot = StorageValue<_, RoundSnapshot>; + + /// Desired number of targets to elect for this round. + /// + /// Only exists when [`Snapshot`] is present. + #[pallet::storage] + #[pallet::getter(fn desired_targets)] + pub type DesiredTargets = StorageValue<_, u32>; + + /// The metadata of the [`RoundSnapshot`] + /// + /// Only exists when [`Snapshot`] is present. + #[pallet::storage] + #[pallet::getter(fn snapshot_metadata)] + pub type SnapshotMetadata = StorageValue<_, RoundSnapshotMetadata>; + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(PhantomData); +} + +impl Pallet +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + /// Logic for `::on_initialize` when signed phase is being opened. + /// + /// This is decoupled for easy weight calculation. + pub fn on_initialize_open_signed() { + >::put(Phase::Signed); + Self::create_snapshot(); + Self::deposit_event(Event::SignedPhaseStarted(Self::round())); + } + + /// Logic for `>::on_initialize` when unsigned phase is being opened. + /// + /// This is decoupled for easy weight calculation. Note that the default weight benchmark of + /// this function will assume an empty signed queue for `finalize_signed_phase`. + pub fn on_initialize_open_unsigned( + need_snapshot: bool, + enabled: bool, + now: T::BlockNumber, + ) { + if need_snapshot { + // if not being followed by a signed phase, then create the snapshots. + debug_assert!(Self::snapshot().is_none()); + Self::create_snapshot(); + } + + // for now always start the unsigned phase. + >::put(Phase::Unsigned((enabled, now))); + Self::deposit_event(Event::UnsignedPhaseStarted(Self::round())); + } + + /// Creates the snapshot. Writes new data to: + /// + /// 1. [`SnapshotMetadata`] + /// 2. [`RoundSnapshot`] + /// 3. [`DesiredTargets`] + pub fn create_snapshot() { + // if any of them don't exist, create all of them. This is a bit conservative. + let targets = T::DataProvider::targets(); + let voters = T::DataProvider::voters(); + let desired_targets = T::DataProvider::desired_targets(); + + >::put(RoundSnapshotMetadata { + voters_len: voters.len() as u32, + targets_len: targets.len() as u32, + }); + >::put(desired_targets); + >::put(RoundSnapshot { voters, targets }); + } + + /// Checks the feasibility of a solution. + /// + /// This checks the solution for the following: + /// + /// 0. **all** of the used indices must be correct. + /// 1. present correct number of winners. + /// 2. any assignment is checked to match with [Snapshot::voters]. + /// 3. for each assignment, the check of `ElectionDataProvider` is also examined. + /// 4. the claimed score is valid. + fn feasibility_check( + solution: RawSolution>, + compute: ElectionCompute, + ) -> Result, FeasibilityError> { + let RawSolution { compact, score, round } = solution; + + // first, check round. + ensure!(Self::round() == round, FeasibilityError::InvalidRound); + + // winners are not directly encoded in the solution. + let winners = compact.unique_targets(); + + let desired_targets = + Self::desired_targets().ok_or(FeasibilityError::SnapshotUnavailable)?; + + // NOTE: this is a bit of duplicate, but we keep it around for veracity. The unsigned path + // already checked this in `unsigned_per_dispatch_checks`. The signed path *could* check it + // upon arrival. + ensure!(winners.len() as u32 == desired_targets, FeasibilityError::WrongWinnerCount,); + + // read the entire snapshot. + let RoundSnapshot { voters: snapshot_voters, targets: snapshot_targets } = + Self::snapshot().ok_or(FeasibilityError::SnapshotUnavailable)?; + + // ----- Start building. First, we need some closures. + let cache = helpers::generate_voter_cache::(&snapshot_voters); + let voter_at = helpers::voter_at_fn::(&snapshot_voters); + let target_at = helpers::target_at_fn::(&snapshot_targets); + let voter_index = helpers::voter_index_fn_usize::(&cache); + + // first, make sure that all the winners are sane. + let winners = winners + .into_iter() + .map(|i| target_at(i).ok_or(FeasibilityError::InvalidWinner)) + .collect::, FeasibilityError>>()?; + + // Then convert compact -> Assignment. This will fail if any of the indices are gibberish. + // that winner indices are already checked. + let assignments = compact + .into_assignment(voter_at, target_at) + .map_err::(Into::into)?; + + // Ensure that assignments is correct. + let _ = assignments + .iter() + .map(|ref assignment| { + // check that assignment.who is actually a voter (defensive-only). + // NOTE: while using the index map from `voter_index` is better than a blind linear + // search, this *still* has room for optimization. Note that we had the index when + // we did `compact -> assignment` and we lost it. Ideal is to keep the index around. + + // defensive-only: must exist in the snapshot. + let snapshot_index = + voter_index(&assignment.who).ok_or(FeasibilityError::InvalidVoter)?; + // defensive-only: index comes from the snapshot, must exist. + let (_voter, _stake, targets) = + snapshot_voters.get(snapshot_index).ok_or(FeasibilityError::InvalidVoter)?; + + // check that all of the targets are valid based on the snapshot. + if assignment.distribution.iter().any(|(d, _)| !targets.contains(d)) { + return Err(FeasibilityError::InvalidVote); + } + Ok(()) + }) + .collect::>()?; + + // ----- Start building support. First, we need one more closure. + let stake_of = helpers::stake_of_fn::(&snapshot_voters, &cache); + + // This might fail if the normalization fails. Very unlikely. See `integrity_test`. + let staked_assignments = assignment_ratio_to_staked_normalized(assignments, stake_of) + .map_err::(Into::into)?; + // This might fail if one of the voter edges is pointing to a non-winner, which is not + // really possible anymore because all the winners come from the same `compact`. + let supports = sp_npos_elections::to_supports(&winners, &staked_assignments) + .map_err::(Into::into)?; + + // Finally, check that the claimed score was indeed correct. + let known_score = (&supports).evaluate(); + ensure!(known_score == score, FeasibilityError::InvalidScore); + + Ok(ReadySolution { supports, compute, score }) + } + + /// Perform the tasks to be done after a new `elect` has been triggered: + /// + /// 1. Increment round. + /// 2. Change phase to [`Phase::Off`] + /// 3. Clear all snapshot data. + fn post_elect() { + // inc round + >::mutate(|r| *r = *r + 1); + + // change phase + >::put(Phase::Off); + + // kill snapshots + >::kill(); + >::kill(); + >::kill(); + } + + /// On-chain fallback of election. + fn onchain_fallback() -> Result, ElectionError> + where + ExtendedBalance: From<::Inner>, + { + > as ElectionProvider< + T::AccountId, + T::BlockNumber, + >>::elect() + .map_err(Into::into) + } + + fn do_elect() -> Result, ElectionError> { + // NOTE: SignedSubmission is guaranteed to be drained by the end of the signed phase too, + // thus no need for a manual cleanup: + // TODO + // debug_assert!(Self::signed_submissions().is_empty()); + >::take() + .map_or_else( + || match T::Fallback::get() { + FallbackStrategy::OnChain => Self::onchain_fallback() + .map(|r| (r, ElectionCompute::OnChain)) + .map_err(Into::into), + FallbackStrategy::Nothing => Err(ElectionError::NoFallbackConfigured), + }, + |ReadySolution { supports, compute, .. }| Ok((supports, compute)), + ) + .map(|(supports, compute)| { + Self::deposit_event(Event::ElectionFinalized(Some(compute))); + log!(info, "Finalized election round with compute {:?}.", compute); + supports + }) + .map_err(|err| { + Self::deposit_event(Event::ElectionFinalized(None)); + log!(warn, "Failed to finalize election round. reason {:?}", err); + err + }) + } +} + +impl ElectionProvider for Pallet +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + type Error = ElectionError; + type DataProvider = T::DataProvider; + + fn elect() -> Result, Self::Error> { + let outcome = Self::do_elect(); + // cleanup. + Self::post_elect(); + outcome + } +} + +/// convert a DispatchError to a custom InvalidTransaction with the inner code being the error +/// number. +pub fn dispatch_error_to_invalid(error: DispatchError) -> InvalidTransaction { + let error_number = match error { + DispatchError::Module { error, .. } => error, + _ => 0, + }; + InvalidTransaction::Custom(error_number) +} + +#[cfg(test)] +mod feasibility_check { + //! All of the tests here should be dedicated to only testing the feasibility check and nothing + //! more. The best way to audit and review these tests is to try and come up with a solution + //! that is invalid, but gets through the system as valid. + + use super::{mock::*, *}; + + const COMPUTE: ElectionCompute = ElectionCompute::OnChain; + + #[test] + fn snapshot_is_there() { + ExtBuilder::default().build_and_execute(|| { + roll_to(::get() - ::get() - ::get()); + assert!(TwoPhase::current_phase().is_signed()); + let solution = raw_solution(); + + // for whatever reason it might be: + >::kill(); + + assert_noop!( + TwoPhase::feasibility_check(solution, COMPUTE), + FeasibilityError::SnapshotUnavailable + ); + }) + } + + #[test] + fn round() { + ExtBuilder::default().build_and_execute(|| { + roll_to(::get() - ::get() - ::get()); + assert!(TwoPhase::current_phase().is_signed()); + + let mut solution = raw_solution(); + solution.round += 1; + assert_noop!( + TwoPhase::feasibility_check(solution, COMPUTE), + FeasibilityError::InvalidRound + ); + }) + } + + #[test] + fn desired_targets() { + ExtBuilder::default().desired_targets(8).build_and_execute(|| { + roll_to(::get() - ::get() - ::get()); + assert!(TwoPhase::current_phase().is_signed()); + + let solution = raw_solution(); + + assert_eq!(solution.compact.unique_targets().len(), 4); + assert_eq!(TwoPhase::desired_targets().unwrap(), 8); + + assert_noop!( + TwoPhase::feasibility_check(solution, COMPUTE), + FeasibilityError::WrongWinnerCount + ); + }) + } + + #[test] + fn winner_indices() { + ExtBuilder::default().desired_targets(2).build_and_execute(|| { + roll_to(::get() - ::get() - ::get()); + assert!(TwoPhase::current_phase().is_signed()); + + let mut solution = raw_solution(); + assert_eq!(TwoPhase::snapshot().unwrap().targets.len(), 4); + // ----------------------------------------------------^^ valid range is [0..3]. + + // swap all votes from 3 to 4. This will ensure that the number of unique winners + // will still be 4, but one of the indices will be gibberish. Requirement is to make + // sure 3 a winner, which we don't do here. + solution + .compact + .votes1 + .iter_mut() + .filter(|(_, t)| *t == 3u16) + .for_each(|(_, t)| *t += 1); + solution.compact.votes2.iter_mut().for_each(|(_, (t0, _), t1)| { + if *t0 == 3u16 { + *t0 += 1 + }; + if *t1 == 3u16 { + *t1 += 1 + }; + }); + assert_noop!( + TwoPhase::feasibility_check(solution, COMPUTE), + FeasibilityError::InvalidWinner + ); + }) + } + + #[test] + fn voter_indices() { + // should be caught in `compact.into_assignment`. + ExtBuilder::default().desired_targets(2).build_and_execute(|| { + roll_to(::get() - ::get() - ::get()); + assert!(TwoPhase::current_phase().is_signed()); + + let mut solution = raw_solution(); + assert_eq!(TwoPhase::snapshot().unwrap().voters.len(), 8); + // ----------------------------------------------------^^ valid range is [0..7]. + + // check that there is a index 7 in votes1, and flip to 8. + assert!( + solution + .compact + .votes1 + .iter_mut() + .filter(|(v, _)| *v == 7u32) + .map(|(v, _)| *v = 8) + .count() > 0 + ); + assert_noop!( + TwoPhase::feasibility_check(solution, COMPUTE), + FeasibilityError::NposElection(sp_npos_elections::Error::CompactInvalidIndex), + ); + }) + } + + #[test] + fn voter_votes() { + ExtBuilder::default().desired_targets(2).build_and_execute(|| { + roll_to(::get() - ::get() - ::get()); + assert!(TwoPhase::current_phase().is_signed()); + + let mut solution = raw_solution(); + assert_eq!(TwoPhase::snapshot().unwrap().voters.len(), 8); + // ----------------------------------------------------^^ valid range is [0..7]. + + // first, check that voter at index 7 (40) actually voted for 3 (40) -- this is self + // vote. Then, change the vote to 2 (30). + assert_eq!( + solution + .compact + .votes1 + .iter_mut() + .filter(|(v, t)| *v == 7 && *t == 3) + .map(|(_, t)| *t = 2) + .count(), + 1, + ); + assert_noop!( + TwoPhase::feasibility_check(solution, COMPUTE), + FeasibilityError::InvalidVote, + ); + }) + } + + #[test] + fn score() { + ExtBuilder::default().desired_targets(2).build_and_execute(|| { + roll_to(::get() - ::get() - ::get()); + assert!(TwoPhase::current_phase().is_signed()); + + let mut solution = raw_solution(); + assert_eq!(TwoPhase::snapshot().unwrap().voters.len(), 8); + + // simply faff with the score. + solution.score[0] += 1; + + assert_noop!( + TwoPhase::feasibility_check(solution, COMPUTE), + FeasibilityError::InvalidScore, + ); + }) + } +} + +#[cfg(test)] +mod tests { + use super::{mock::*, Event, *}; + use sp_election_providers::ElectionProvider; + use sp_npos_elections::Support; + + #[test] + fn phase_rotation_works() { + ExtBuilder::default().build_and_execute(|| { + // 0 ------- 15 ------- 25 ------- 30 ------- ------- 45 ------- 55 ------- 60 + // | | | | + // Signed Unsigned Signed Unsigned + + assert_eq!(System::block_number(), 0); + assert_eq!(TwoPhase::current_phase(), Phase::Off); + assert_eq!(TwoPhase::round(), 1); + + roll_to(4); + assert_eq!(TwoPhase::current_phase(), Phase::Off); + assert!(TwoPhase::snapshot().is_none()); + assert_eq!(TwoPhase::round(), 1); + + roll_to(15); + assert_eq!(TwoPhase::current_phase(), Phase::Signed); + assert_eq!(two_phase_events(), vec![Event::SignedPhaseStarted(1)]); + assert!(TwoPhase::snapshot().is_some()); + assert_eq!(TwoPhase::round(), 1); + + roll_to(24); + assert_eq!(TwoPhase::current_phase(), Phase::Signed); + assert!(TwoPhase::snapshot().is_some()); + assert_eq!(TwoPhase::round(), 1); + + roll_to(25); + assert_eq!(TwoPhase::current_phase(), Phase::Unsigned((true, 25))); + assert_eq!( + two_phase_events(), + vec![Event::SignedPhaseStarted(1), Event::UnsignedPhaseStarted(1)], + ); + assert!(TwoPhase::snapshot().is_some()); + + roll_to(29); + assert_eq!(TwoPhase::current_phase(), Phase::Unsigned((true, 25))); + assert!(TwoPhase::snapshot().is_some()); + + roll_to(30); + assert_eq!(TwoPhase::current_phase(), Phase::Unsigned((true, 25))); + assert!(TwoPhase::snapshot().is_some()); + + // we close when upstream tells us to elect. + roll_to(32); + assert_eq!(TwoPhase::current_phase(), Phase::Unsigned((true, 25))); + assert!(TwoPhase::snapshot().is_some()); + + TwoPhase::elect().unwrap(); + + assert!(TwoPhase::current_phase().is_off()); + assert!(TwoPhase::snapshot().is_none()); + assert_eq!(TwoPhase::round(), 2); + + roll_to(44); + assert!(TwoPhase::current_phase().is_off()); + + roll_to(45); + assert!(TwoPhase::current_phase().is_signed()); + + roll_to(55); + assert!(TwoPhase::current_phase().is_unsigned_open_at(55)); + }) + } + + #[test] + fn signed_phase_void() { + ExtBuilder::default().phases(0, 10).build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_off()); + + roll_to(19); + assert!(TwoPhase::current_phase().is_off()); + + roll_to(20); + assert!(TwoPhase::current_phase().is_unsigned_open_at(20)); + assert!(TwoPhase::snapshot().is_some()); + + roll_to(30); + assert!(TwoPhase::current_phase().is_unsigned_open_at(20)); + + TwoPhase::elect().unwrap(); + + assert!(TwoPhase::current_phase().is_off()); + assert!(TwoPhase::snapshot().is_none()); + }); + } + + #[test] + fn unsigned_phase_void() { + ExtBuilder::default().phases(10, 0).build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_off()); + + roll_to(19); + assert!(TwoPhase::current_phase().is_off()); + + roll_to(20); + assert!(TwoPhase::current_phase().is_signed()); + assert!(TwoPhase::snapshot().is_some()); + + roll_to(30); + assert!(TwoPhase::current_phase().is_signed()); + + let _ = TwoPhase::elect().unwrap(); + + assert!(TwoPhase::current_phase().is_off()); + assert!(TwoPhase::snapshot().is_none()); + }); + } + + #[test] + fn both_phases_void() { + ExtBuilder::default().phases(0, 0).build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_off()); + + roll_to(19); + assert!(TwoPhase::current_phase().is_off()); + + roll_to(20); + assert!(TwoPhase::current_phase().is_off()); + + roll_to(30); + assert!(TwoPhase::current_phase().is_off()); + + // this module is now only capable of doing on-chain backup. + let _ = TwoPhase::elect().unwrap(); + + assert!(TwoPhase::current_phase().is_off()); + }); + } + + #[test] + fn early_termination() { + // an early termination in the signed phase, with no queued solution. + ExtBuilder::default().build_and_execute(|| { + // signed phase started at block 15 and will end at 25. + roll_to(14); + assert_eq!(TwoPhase::current_phase(), Phase::Off); + + roll_to(15); + assert_eq!(two_phase_events(), vec![Event::SignedPhaseStarted(1)]); + assert_eq!(TwoPhase::current_phase(), Phase::Signed); + assert_eq!(TwoPhase::round(), 1); + + // an unexpected call to elect. + roll_to(20); + TwoPhase::elect().unwrap(); + + // we surely can't have any feasible solutions. This will cause an on-chain election. + assert_eq!( + two_phase_events(), + vec![ + Event::SignedPhaseStarted(1), + Event::ElectionFinalized(Some(ElectionCompute::OnChain)) + ], + ); + // all storage items must be cleared. + assert_eq!(TwoPhase::round(), 2); + assert!(TwoPhase::snapshot().is_none()); + assert!(TwoPhase::snapshot_metadata().is_none()); + assert!(TwoPhase::desired_targets().is_none()); + assert!(TwoPhase::queued_solution().is_none()); + }) + } + + #[test] + fn fallback_strategy_works() { + ExtBuilder::default().fallabck(FallbackStrategy::OnChain).build_and_execute(|| { + roll_to(15); + assert_eq!(TwoPhase::current_phase(), Phase::Signed); + + roll_to(25); + assert_eq!(TwoPhase::current_phase(), Phase::Unsigned((true, 25))); + + // zilch solutions thus far. + let supports = TwoPhase::elect().unwrap(); + + assert_eq!( + supports, + vec![ + (30, Support { total: 40, voters: vec![(2, 5), (4, 5), (30, 30)] }), + (40, Support { total: 60, voters: vec![(2, 5), (3, 10), (4, 5), (40, 40)] }) + ] + ) + }); + + ExtBuilder::default().fallabck(FallbackStrategy::Nothing).build_and_execute(|| { + roll_to(15); + assert_eq!(TwoPhase::current_phase(), Phase::Signed); + + roll_to(25); + assert_eq!(TwoPhase::current_phase(), Phase::Unsigned((true, 25))); + + // zilch solutions thus far. + assert_eq!(TwoPhase::elect().unwrap_err(), ElectionError::NoFallbackConfigured); + }) + } + + #[test] + fn number_of_voters_allowed_2sec_block() { + // Just a rough estimate with the substrate weights. + assert!(!MockWeightInfo::get()); + + let all_voters: u32 = 100_000; + let all_targets: u32 = 2_000; + let desired: u32 = 1_000; + let weight_with = |active| { + ::WeightInfo::submit_unsigned( + all_voters, + all_targets, + active, + desired, + ) + }; + + let mut active = 1; + while weight_with(active) + <= ::BlockWeights::get().max_block + { + active += 1; + } + + println!("can support {} voters to yield a weight of {}", active, weight_with(active)); + } +} diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs new file mode 100644 index 0000000000000..71102f3e6e526 --- /dev/null +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -0,0 +1,386 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate as two_phase; +pub use frame_support::{assert_noop, assert_ok}; +use frame_support::{ + parameter_types, + traits::{Hooks}, + weights::Weight, +}; +use parking_lot::RwLock; +use sp_core::{ + offchain::{ + testing::{PoolState, TestOffchainExt, TestTransactionPoolExt}, + OffchainExt, TransactionPoolExt, + }, + H256, +}; +use sp_election_providers::ElectionDataProvider; +use sp_npos_elections::{ + assignment_ratio_to_staked_normalized, seq_phragmen, to_supports, to_without_backing, + CompactSolution, ElectionResult, EvaluateSupport, +}; +use sp_runtime::{ + testing::Header, + traits::{BlakeTwo256, Block as BlockT, IdentityLookup}, + PerU16, +}; +use std::sync::Arc; + +pub type Block = sp_runtime::generic::Block; +pub type UncheckedExtrinsic = sp_runtime::generic::UncheckedExtrinsic; + +frame_support::construct_runtime!( + pub enum Runtime where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic + { + System: frame_system::{Module, Call, Event}, + Balances: pallet_balances::{Module, Call, Event, Config}, + TwoPhase: two_phase::{Module, Call, Event}, + } +); + +pub(crate) type Balance = u64; +pub(crate) type AccountId = u64; + +sp_npos_elections::generate_solution_type!( + #[compact] + pub struct TestCompact::(16) +); + +/// All events of this pallet. +pub(crate) fn two_phase_events() -> Vec> { + System::events() + .into_iter() + .map(|r| r.event) + .filter_map(|e| if let Event::two_phase(inner) = e { Some(inner) } else { None }) + .collect::>() +} + +/// To from `now` to block `n`. +pub fn roll_to(n: u64) { + let now = System::block_number(); + for i in now + 1..=n { + System::set_block_number(i); + TwoPhase::on_initialize(i); + } +} + +pub fn roll_to_with_ocw(n: u64) { + let now = System::block_number(); + for i in now + 1..=n { + System::set_block_number(i); + TwoPhase::on_initialize(i); + TwoPhase::offchain_worker(i); + } +} + +/// Get the free and reserved balance of some account. +pub fn balances(who: &AccountId) -> (Balance, Balance) { + (Balances::free_balance(who), Balances::reserved_balance(who)) +} + +/// Spit out a verifiable raw solution. +/// +/// This is a good example of what an offchain miner would do. +pub fn raw_solution() -> RawSolution> { + let RoundSnapshot { voters, targets } = TwoPhase::snapshot().unwrap(); + let desired_targets = TwoPhase::desired_targets().unwrap(); + + // closures + let cache = helpers::generate_voter_cache::(&voters); + let voter_index = helpers::voter_index_fn_linear::(&voters); + let target_index = helpers::target_index_fn_linear::(&targets); + let stake_of = helpers::stake_of_fn::(&voters, &cache); + + let ElectionResult { winners, assignments } = seq_phragmen::<_, CompactAccuracyOf>( + desired_targets as usize, + targets.clone(), + voters.clone(), + None, + ) + .unwrap(); + + let winners = to_without_backing(winners); + + let score = { + let staked = assignment_ratio_to_staked_normalized(assignments.clone(), &stake_of).unwrap(); + to_supports(&winners, &staked).unwrap().evaluate() + }; + let compact = + >::from_assignment(assignments, &voter_index, &target_index).unwrap(); + + let round = TwoPhase::round(); + RawSolution { compact, score, round } +} + +pub fn witness() -> SolutionSize { + TwoPhase::snapshot() + .map(|snap| SolutionSize { + voters: snap.voters.len() as u32, + targets: snap.targets.len() as u32, + }) + .unwrap_or_default() +} + +impl frame_system::Config for Runtime { + type SS58Prefix = (); + type BaseCallFilter = (); + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Call = Call; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = AccountId; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = (); + type DbWeight = (); + type BlockLength = (); + type BlockWeights = BlockWeights; + type Version = (); + type PalletInfo = (); + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); +} + +const NORMAL_DISPATCH_RATIO: Perbill = Perbill::from_percent(75); +parameter_types! { + pub const ExistentialDeposit: u64 = 1; + pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights + ::with_sensible_defaults(2 * frame_support::weights::constants::WEIGHT_PER_SECOND, NORMAL_DISPATCH_RATIO); +} + +impl pallet_balances::Config for Runtime { + type Balance = Balance; + type Event = Event; + type DustRemoval = (); + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; + type MaxLocks = (); + type WeightInfo = (); +} + +parameter_types! { + pub static Targets: Vec = vec![10, 20, 30, 40]; + pub static Voters: Vec<(AccountId, VoteWeight, Vec)> = vec![ + (1, 10, vec![10, 20]), + (2, 10, vec![30, 40]), + (3, 10, vec![40]), + (4, 10, vec![10, 20, 30, 40]), + // self votes. + (10, 10, vec![10]), + (20, 20, vec![20]), + (30, 30, vec![30]), + (40, 40, vec![40]), + ]; + + pub static Fallback: FallbackStrategy = FallbackStrategy::OnChain; + pub static DesiredTargets: u32 = 2; + pub static SignedPhase: u64 = 10; + pub static UnsignedPhase: u64 = 5; + pub static MaxSignedSubmissions: u32 = 5; + + pub static MinerMaxIterations: u32 = 5; + pub static UnsignedPriority: u64 = 100; + pub static SolutionImprovementThreshold: Perbill = Perbill::zero(); + pub static MinerMaxWeight: Weight = BlockWeights::get().max_block; + pub static MockWeightInfo: bool = false; + + + pub static EpochLength: u64 = 30; +} + +// Hopefully this won't be too much of a hassle to maintain. +pub struct DualMockWeightInfo; +impl two_phase::weights::WeightInfo for DualMockWeightInfo { + fn on_initialize_nothing() -> Weight { + if MockWeightInfo::get() { + Zero::zero() + } else { + <() as two_phase::weights::WeightInfo>::on_initialize_nothing() + } + } + fn on_initialize_open_signed() -> Weight { + if MockWeightInfo::get() { + Zero::zero() + } else { + <() as two_phase::weights::WeightInfo>::on_initialize_open_signed() + } + } + fn on_initialize_open_unsigned_with_snapshot() -> Weight { + if MockWeightInfo::get() { + Zero::zero() + } else { + <() as two_phase::weights::WeightInfo>::on_initialize_open_unsigned_with_snapshot() + } + } + fn on_initialize_open_unsigned_without_snapshot() -> Weight { + if MockWeightInfo::get() { + Zero::zero() + } else { + <() as two_phase::weights::WeightInfo>::on_initialize_open_unsigned_without_snapshot() + } + } + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32) -> Weight { + if MockWeightInfo::get() { + // 10 base + // 5 per edge. + (10 as Weight).saturating_add((5 as Weight).saturating_mul(a as Weight)) + } else { + <() as two_phase::weights::WeightInfo>::submit_unsigned(v, t, a, d) + } + } + fn feasibility_check(v: u32, t: u32, a: u32, d: u32) -> Weight { + if MockWeightInfo::get() { + // 10 base + // 5 per edge. + (10 as Weight).saturating_add((5 as Weight).saturating_mul(a as Weight)) + } else { + <() as two_phase::weights::WeightInfo>::feasibility_check(v, t, a, d) + } + } +} + +impl crate::Config for Runtime { + type Event = Event; + type Currency = Balances; + type SignedPhase = SignedPhase; + type UnsignedPhase = UnsignedPhase; + type SolutionImprovementThreshold = SolutionImprovementThreshold; + type MinerMaxIterations = MinerMaxIterations; + type MinerMaxWeight = MinerMaxWeight; + type UnsignedPriority = UnsignedPriority; + type DataProvider = StakingMock; + type WeightInfo = DualMockWeightInfo; + type BenchmarkingConfig = (); + type OnChainAccuracy = Perbill; + type Fallback = Fallback; + type CompactSolution = TestCompact; +} + +impl frame_system::offchain::SendTransactionTypes for Runtime +where + Call: From, +{ + type OverarchingCall = Call; + type Extrinsic = Extrinsic; +} + +pub type Extrinsic = sp_runtime::testing::TestXt; + +#[derive(Default)] +pub struct ExtBuilder {} + +pub struct StakingMock; +impl ElectionDataProvider for StakingMock { + fn targets() -> Vec { + Targets::get() + } + fn voters() -> Vec<(AccountId, VoteWeight, Vec)> { + Voters::get() + } + fn desired_targets() -> u32 { + DesiredTargets::get() + } + fn next_election_prediction(now: u64) -> u64 { + now + EpochLength::get() - now % EpochLength::get() + } +} + +impl ExtBuilder { + pub fn unsigned_priority(self, p: u64) -> Self { + ::set(p); + self + } + pub fn solution_improvement_threshold(self, p: Perbill) -> Self { + ::set(p); + self + } + pub fn phases(self, signed: u64, unsigned: u64) -> Self { + ::set(signed); + ::set(unsigned); + self + } + pub fn fallabck(self, fallback: FallbackStrategy) -> Self { + ::set(fallback); + self + } + pub fn miner_weight(self, weight: Weight) -> Self { + ::set(weight); + self + } + pub fn mock_weight_info(self, mock: bool) -> Self { + ::set(mock); + self + } + pub fn desired_targets(self, t: u32) -> Self { + ::set(t); + self + } + pub fn add_voter(self, who: AccountId, stake: Balance, targets: Vec) -> Self { + VOTERS.with(|v| v.borrow_mut().push((who, stake, targets))); + self + } + pub fn build(self) -> sp_io::TestExternalities { + sp_tracing::try_init_simple(); + let mut storage = + frame_system::GenesisConfig::default().build_storage::().unwrap(); + + let _ = pallet_balances::GenesisConfig:: { + balances: vec![ + // bunch of account for submitting stuff only. + (99, 100), + (999, 100), + (9999, 100), + ], + } + .assimilate_storage(&mut storage); + + sp_io::TestExternalities::from(storage) + } + + pub fn build_offchainify( + self, + iters: u32, + ) -> (sp_io::TestExternalities, Arc>) { + let mut ext = self.build(); + let (offchain, offchain_state) = TestOffchainExt::new(); + let (pool, pool_state) = TestTransactionPoolExt::new(); + + let mut seed = [0_u8; 32]; + seed[0..4].copy_from_slice(&iters.to_le_bytes()); + offchain_state.write().seed = seed; + + ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(TransactionPoolExt::new(pool)); + + (ext, pool_state) + } + + pub fn build_and_execute(self, test: impl FnOnce() -> ()) { + self.build().execute_with(test) + } +} diff --git a/frame/election-provider-multi-phase/src/unsigned.rs b/frame/election-provider-multi-phase/src/unsigned.rs new file mode 100644 index 0000000000000..e922dc32dfa34 --- /dev/null +++ b/frame/election-provider-multi-phase/src/unsigned.rs @@ -0,0 +1,799 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The unsigned phase implementation. + +use crate::*; +use frame_support::dispatch::DispatchResult; +use frame_system::offchain::SubmitTransaction; +use sp_npos_elections::{seq_phragmen, CompactSolution, ElectionResult}; +use sp_runtime::{offchain::storage::StorageValueRef, traits::TrailingZeroInput}; +use sp_std::cmp::Ordering; + +/// Storage key used to store the persistent offchain worker status. +pub(crate) const OFFCHAIN_HEAD_DB: &[u8] = b"parity/multi-phase-unsigned-election/"; +/// The repeat threshold of the offchain worker. This means we won't run the offchain worker twice +/// within a window of 5 blocks. +pub(crate) const OFFCHAIN_REPEAT: u32 = 5; + +impl Pallet +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + /// Min a new npos solution. + pub fn mine_solution( + iters: usize, + ) -> Result<(RawSolution>, SolutionSize), ElectionError> { + let RoundSnapshot { voters, targets } = + Self::snapshot().ok_or(ElectionError::SnapshotUnAvailable)?; + let desired_targets = Self::desired_targets().ok_or(ElectionError::SnapshotUnAvailable)?; + + seq_phragmen::<_, CompactAccuracyOf>( + desired_targets as usize, + targets, + voters, + Some((iters, 0)), + ) + .map_err(Into::into) + .and_then(|election_result| { + if election_result.winners.len() as u32 == desired_targets { + Ok(election_result) + } else { + Err(ElectionError::Feasibility(FeasibilityError::WrongWinnerCount)) + } + }) + .and_then(Self::prepare_election_result) + } + + /// Convert a raw solution from [`sp_npos_elections::ElectionResult`] to [`RawSolution`], which + /// is ready to be submitted to the chain. + /// + /// Will always reduce the solution as well. + pub fn prepare_election_result( + election_result: ElectionResult>, + ) -> Result<(RawSolution>, SolutionSize), ElectionError> { + // storage items. Note: we have already read this from storage, they must be in cache. + let RoundSnapshot { voters, targets } = + Self::snapshot().ok_or(ElectionError::SnapshotUnAvailable)?; + let desired_targets = Self::desired_targets().ok_or(ElectionError::SnapshotUnAvailable)?; + + // closures. + let cache = helpers::generate_voter_cache::(&voters); + let voter_index = helpers::voter_index_fn::(&cache); + let target_index = helpers::target_index_fn_linear::(&targets); + let voter_at = helpers::voter_at_fn::(&voters); + let target_at = helpers::target_at_fn::(&targets); + let stake_of = helpers::stake_of_fn::(&voters, &cache); + + let ElectionResult { assignments, winners } = election_result; + + // convert to staked and reduce. + let mut staked = + sp_npos_elections::assignment_ratio_to_staked_normalized(assignments, &stake_of) + .map_err::(Into::into)?; + sp_npos_elections::reduce(&mut staked); + + // convert back to ration and make compact. + let ratio = sp_npos_elections::assignment_staked_to_ratio_normalized(staked)?; + let compact = >::from_assignment(ratio, &voter_index, &target_index)?; + + let size = SolutionSize { voters: voters.len() as u32, targets: targets.len() as u32 }; + let maximum_allowed_voters = Self::maximum_voter_for_weight::( + desired_targets, + size, + T::MinerMaxWeight::get(), + ); + log!( + debug, + "miner: current compact solution voters = {}, maximum_allowed = {}", + compact.voter_count(), + maximum_allowed_voters, + ); + let compact = Self::trim_compact(maximum_allowed_voters, compact, &voter_index)?; + + // re-calc score. + let winners = sp_npos_elections::to_without_backing(winners); + let score = compact.clone().score(&winners, stake_of, voter_at, target_at)?; + + let round = Self::round(); + Ok((RawSolution { compact, score, round }, size)) + } + + /// Get a random number of iterations to run the balancing in the OCW. + /// + /// Uses the offchain seed to generate a random number, maxed with `T::MinerMaxIterations`. + pub fn get_balancing_iters() -> usize { + match T::MinerMaxIterations::get() { + 0 => 0, + max @ _ => { + let seed = sp_io::offchain::random_seed(); + let random = ::decode(&mut TrailingZeroInput::new(seed.as_ref())) + .expect("input is padded with zeroes; qed") + % max.saturating_add(1); + random as usize + } + } + } + + /// Greedily reduce the size of the a solution to fit into the block, w.r.t. weight. + /// + /// The weight of the solution is foremost a function of the number of voters (i.e. + /// `compact.len()`). Aside from this, the other components of the weight are invariant. The + /// number of winners shall not be changed (otherwise the solution is invalid) and the + /// `ElectionSize` is merely a representation of the total number of stakers. + /// + /// Thus, we reside to stripping away some voters. This means only changing the `compact` + /// struct. + /// + /// Note that the solution is already computed, and the winners are elected based on the merit + /// of the entire stake in the system. Nonetheless, some of the voters will be removed further + /// down the line. + /// + /// Indeed, the score must be computed **after** this step. If this step reduces the score too + /// much, then the solution will be discarded. + pub fn trim_compact( + maximum_allowed_voters: u32, + mut compact: CompactOf, + nominator_index: FN, + ) -> Result, ElectionError> + where + for<'r> FN: Fn(&'r T::AccountId) -> Option>, + { + match compact.voter_count().checked_sub(maximum_allowed_voters as usize) { + Some(to_remove) if to_remove > 0 => { + // grab all voters and sort them by least stake. + let RoundSnapshot { voters, .. } = + Self::snapshot().ok_or(ElectionError::SnapshotUnAvailable)?; + let mut voters_sorted = voters + .into_iter() + .map(|(who, stake, _)| (who.clone(), stake)) + .collect::>(); + voters_sorted.sort_by_key(|(_, y)| *y); + + // start removing from the least stake. Iterate until we know enough have been + // removed. + let mut removed = 0; + for (maybe_index, _stake) in + voters_sorted.iter().map(|(who, stake)| (nominator_index(&who), stake)) + { + let index = maybe_index.ok_or(ElectionError::SnapshotUnAvailable)?; + if compact.remove_voter(index) { + removed += 1 + } + + if removed >= to_remove { + break; + } + } + + Ok(compact) + } + _ => { + // nada, return as-is + Ok(compact) + } + } + } + + /// Find the maximum `len` that a compact can have in order to fit into the block weight. + /// + /// This only returns a value between zero and `size.nominators`. + pub fn maximum_voter_for_weight( + desired_winners: u32, + size: SolutionSize, + max_weight: Weight, + ) -> u32 { + if size.voters < 1 { + return size.voters; + } + + let max_voters = size.voters.max(1); + let mut voters = max_voters; + + // helper closures. + let weight_with = |active_voters: u32| -> Weight { + W::submit_unsigned(size.voters, size.targets, active_voters, desired_winners) + }; + + let next_voters = |current_weight: Weight, voters: u32, step: u32| -> Result { + match current_weight.cmp(&max_weight) { + Ordering::Less => { + let next_voters = voters.checked_add(step); + match next_voters { + Some(voters) if voters < max_voters => Ok(voters), + _ => Err(()), + } + } + Ordering::Greater => voters.checked_sub(step).ok_or(()), + Ordering::Equal => Ok(voters), + } + }; + + // First binary-search the right amount of voters + let mut step = voters / 2; + let mut current_weight = weight_with(voters); + while step > 0 { + match next_voters(current_weight, voters, step) { + // proceed with the binary search + Ok(next) if next != voters => { + voters = next; + } + // we are out of bounds, break out of the loop. + Err(()) => { + break; + } + // we found the right value - early exit the function. + Ok(next) => return next, + } + step = step / 2; + current_weight = weight_with(voters); + } + + // Time to finish. We might have reduced less than expected due to rounding error. Increase + // one last time if we have any room left, the reduce until we are sure we are below limit. + while voters + 1 <= max_voters && weight_with(voters + 1) < max_weight { + voters += 1; + } + while voters.checked_sub(1).is_some() && weight_with(voters) > max_weight { + voters -= 1; + } + + debug_assert!( + weight_with(voters.min(size.voters)) <= max_weight, + "weight_with({}) <= {}", + voters.min(size.voters), + max_weight, + ); + voters.min(size.voters) + } + + /// Checks if an execution of the offchain worker is permitted at the given block number, or + /// not. + /// + /// This essentially makes sure that we don't run on previous blocks in case of a re-org, and we + /// don't run twice within a window of length [`OFFCHAIN_REPEAT`]. + /// + /// Returns `Ok(())` if offchain worker should happen, `Err(reason)` otherwise. + pub(crate) fn set_check_offchain_execution_status( + now: T::BlockNumber, + ) -> Result<(), &'static str> { + let storage = StorageValueRef::persistent(&OFFCHAIN_HEAD_DB); + let threshold = T::BlockNumber::from(OFFCHAIN_REPEAT); + + let mutate_stat = + storage.mutate::<_, &'static str, _>(|maybe_head: Option>| { + match maybe_head { + Some(Some(head)) if now < head => Err("fork."), + Some(Some(head)) if now >= head && now <= head + threshold => { + Err("recently executed.") + } + Some(Some(head)) if now > head + threshold => { + // we can run again now. Write the new head. + Ok(now) + } + _ => { + // value doesn't exists. Probably this node just booted up. Write, and run + Ok(now) + } + } + }); + + match mutate_stat { + // all good + Ok(Ok(_)) => Ok(()), + // failed to write. + Ok(Err(_)) => Err("failed to write to offchain db."), + // fork etc. + Err(why) => Err(why), + } + } + + /// Mine a new solution, and submit it back to the chain as an unsigned transaction. + pub(crate) fn mine_and_submit() -> Result<(), ElectionError> { + let balancing = Self::get_balancing_iters(); + let (raw_solution, witness) = Self::mine_solution(balancing)?; + + // submit the raw solution to the pool. + let call = Call::submit_unsigned(raw_solution, witness).into(); + + SubmitTransaction::>::submit_unsigned_transaction(call) + .map_err(|_| ElectionError::PoolSubmissionFailed) + } + + pub(crate) fn unsigned_pre_dispatch_checks( + solution: &RawSolution>, + ) -> DispatchResult { + // ensure solution is timely. Don't panic yet. This is a cheap check. + ensure!(Self::current_phase().is_unsigned_open(), Error::::EarlySubmission); + + // ensure correct number of winners. + ensure!( + Self::desired_targets().unwrap_or_default() + == solution.compact.unique_targets().len() as u32, + Error::::WrongWinnerCount, + ); + + // ensure score is being improved. Panic henceforth. + ensure!( + Self::queued_solution().map_or(true, |q: ReadySolution<_>| is_score_better::( + solution.score, + q.score, + T::SolutionImprovementThreshold::get() + )), + Error::::WeakSubmission + ); + + Ok(()) + } +} + +#[cfg(test)] +mod max_weight { + #![allow(unused_variables)] + use super::{mock::*, *}; + + struct TestWeight; + impl crate::weights::WeightInfo for TestWeight { + fn on_initialize_nothing() -> Weight { + unreachable!() + } + fn on_initialize_open_signed() -> Weight { + unreachable!() + } + fn on_initialize_open_unsigned_with_snapshot() -> Weight { + unreachable!() + } + fn on_initialize_open_unsigned_without_snapshot() -> Weight { + unreachable!() + } + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32) -> Weight { + (0 * v + 0 * t + 1000 * a + 0 * d) as Weight + } + fn feasibility_check(v: u32, _t: u32, a: u32, d: u32) -> Weight { + unreachable!() + } + } + + #[test] + fn find_max_voter_binary_search_works() { + let w = SolutionSize { voters: 10, targets: 0 }; + + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 0), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 999), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1000), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1001), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1990), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1999), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2000), 2); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2001), 2); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2010), 2); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2990), 2); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2999), 2); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 3000), 3); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 3333), 3); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 5500), 5); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 7777), 7); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 9999), 9); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 10_000), 10); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 10_999), 10); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 11_000), 10); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 22_000), 10); + + let w = SolutionSize { voters: 1, targets: 0 }; + + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 0), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 999), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1000), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1001), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1990), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1999), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2000), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2001), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2010), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 3333), 1); + + let w = SolutionSize { voters: 2, targets: 0 }; + + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 0), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 999), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1000), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1001), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1999), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2000), 2); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2001), 2); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2010), 2); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 3333), 2); + } +} + +#[cfg(test)] +mod tests { + use super::{ + mock::{Origin, *}, + Call, *, + }; + use frame_support::{dispatch::Dispatchable, traits::OffchainWorker}; + use mock::Call as OuterCall; + use sp_election_providers::Assignment; + use sp_runtime::{traits::ValidateUnsigned, PerU16}; + + #[test] + fn validate_unsigned_retracts_wrong_phase() { + ExtBuilder::default().desired_targets(0).build_and_execute(|| { + let solution = RawSolution:: { score: [5, 0, 0], ..Default::default() }; + let call = Call::submit_unsigned(solution.clone(), witness()); + + // initial + assert_eq!(TwoPhase::current_phase(), Phase::Off); + assert!(matches!( + ::validate_unsigned(TransactionSource::Local, &call) + .unwrap_err(), + TransactionValidityError::Invalid(InvalidTransaction::Custom(0)) + )); + assert!(matches!( + ::pre_dispatch(&call).unwrap_err(), + TransactionValidityError::Invalid(InvalidTransaction::Custom(0)) + )); + + // signed + roll_to(15); + assert_eq!(TwoPhase::current_phase(), Phase::Signed); + assert!(matches!( + ::validate_unsigned(TransactionSource::Local, &call) + .unwrap_err(), + TransactionValidityError::Invalid(InvalidTransaction::Custom(0)) + )); + assert!(matches!( + ::pre_dispatch(&call).unwrap_err(), + TransactionValidityError::Invalid(InvalidTransaction::Custom(0)) + )); + + // unsigned + roll_to(25); + assert!(TwoPhase::current_phase().is_unsigned()); + + assert!(::validate_unsigned( + TransactionSource::Local, + &call + ) + .is_ok()); + assert!(::pre_dispatch(&call).is_ok()); + }) + } + + #[test] + fn validate_unsigned_retracts_low_score() { + ExtBuilder::default().desired_targets(0).build_and_execute(|| { + roll_to(25); + assert!(TwoPhase::current_phase().is_unsigned()); + + let solution = RawSolution:: { score: [5, 0, 0], ..Default::default() }; + let call = Call::submit_unsigned(solution.clone(), witness()); + + // initial + assert!(::validate_unsigned( + TransactionSource::Local, + &call + ) + .is_ok()); + assert!(::pre_dispatch(&call).is_ok()); + + // set a better score + let ready = ReadySolution { score: [10, 0, 0], ..Default::default() }; + >::put(ready); + + // won't work anymore. + assert!(matches!( + ::validate_unsigned(TransactionSource::Local, &call) + .unwrap_err(), + TransactionValidityError::Invalid(InvalidTransaction::Custom(2)) + )); + assert!(matches!( + ::pre_dispatch(&call).unwrap_err(), + TransactionValidityError::Invalid(InvalidTransaction::Custom(2)) + )); + }) + } + + #[test] + fn validate_unsigned_retracts_incorrect_winner_count() { + ExtBuilder::default().desired_targets(1).build_and_execute(|| { + roll_to(25); + assert!(TwoPhase::current_phase().is_unsigned()); + + let solution = RawSolution:: { score: [5, 0, 0], ..Default::default() }; + let call = Call::submit_unsigned(solution.clone(), witness()); + assert_eq!(solution.compact.unique_targets().len(), 0); + + // won't work anymore. + assert!(matches!( + ::validate_unsigned(TransactionSource::Local, &call) + .unwrap_err(), + TransactionValidityError::Invalid(InvalidTransaction::Custom(1)) + )); + }) + } + + #[test] + fn priority_is_set() { + ExtBuilder::default().unsigned_priority(20).desired_targets(0).build_and_execute(|| { + roll_to(25); + assert!(TwoPhase::current_phase().is_unsigned()); + + let solution = RawSolution:: { score: [5, 0, 0], ..Default::default() }; + let call = Call::submit_unsigned(solution.clone(), witness()); + + assert_eq!( + ::validate_unsigned(TransactionSource::Local, &call) + .unwrap() + .priority, + 25 + ); + }) + } + + #[test] + #[should_panic(expected = "Invalid unsigned submission must produce invalid block and \ + deprive validator from their authoring reward.: \ + DispatchError::Module { index: 0, error: 1, message: \ + Some(\"WrongWinnerCount\") }")] + fn unfeasible_solution_panics() { + ExtBuilder::default().build_and_execute(|| { + roll_to(25); + assert!(TwoPhase::current_phase().is_unsigned()); + + // This is in itself an invalid BS solution. + let solution = RawSolution:: { score: [5, 0, 0], ..Default::default() }; + let call = Call::submit_unsigned(solution.clone(), witness()); + let outer_call: OuterCall = call.into(); + let _ = outer_call.dispatch(Origin::none()); + }) + } + + #[test] + #[should_panic(expected = "Invalid unsigned submission must produce invalid block and \ + deprive validator from their authoring reward.")] + fn wrong_witness_panics() { + ExtBuilder::default().build_and_execute(|| { + roll_to(25); + assert!(TwoPhase::current_phase().is_unsigned()); + + // This solution is unfeasible as well, but we won't even get there. + let solution = RawSolution:: { score: [5, 0, 0], ..Default::default() }; + + let mut correct_witness = witness(); + correct_witness.voters += 1; + correct_witness.targets -= 1; + let call = Call::submit_unsigned(solution.clone(), correct_witness); + let outer_call: OuterCall = call.into(); + let _ = outer_call.dispatch(Origin::none()); + }) + } + + #[test] + fn miner_works() { + ExtBuilder::default().build_and_execute(|| { + roll_to(25); + assert!(TwoPhase::current_phase().is_unsigned()); + + // ensure we have snapshots in place. + assert!(TwoPhase::snapshot().is_some()); + assert_eq!(TwoPhase::desired_targets().unwrap(), 2); + + // mine seq_phragmen solution with 2 iters. + let (solution, witness) = TwoPhase::mine_solution(2).unwrap(); + + // ensure this solution is valid. + assert!(TwoPhase::queued_solution().is_none()); + assert_ok!(TwoPhase::submit_unsigned(Origin::none(), solution, witness)); + assert!(TwoPhase::queued_solution().is_some()); + }) + } + + #[test] + fn miner_trims_weight() { + ExtBuilder::default().miner_weight(100).mock_weight_info(true).build_and_execute(|| { + roll_to(25); + assert!(TwoPhase::current_phase().is_unsigned()); + + let (solution, witness) = TwoPhase::mine_solution(2).unwrap(); + let solution_weight = ::WeightInfo::submit_unsigned( + witness.voters, + witness.targets, + solution.compact.voter_count() as u32, + solution.compact.unique_targets().len() as u32, + ); + // default solution will have 5 edges (5 * 5 + 10) + assert_eq!(solution_weight, 35); + assert_eq!(solution.compact.voter_count(), 5); + + // now reduce the max weight + ::set(25); + + let (solution, witness) = TwoPhase::mine_solution(2).unwrap(); + let solution_weight = ::WeightInfo::submit_unsigned( + witness.voters, + witness.targets, + solution.compact.voter_count() as u32, + solution.compact.unique_targets().len() as u32, + ); + // default solution will have 5 edges (5 * 5 + 10) + assert_eq!(solution_weight, 25); + assert_eq!(solution.compact.voter_count(), 3); + }) + } + + #[test] + fn miner_will_not_submit_if_not_enough_winners() { + ExtBuilder::default().desired_targets(8).build_and_execute(|| { + roll_to(25); + assert!(TwoPhase::current_phase().is_unsigned()); + + // mine seq_phragmen solution with 2 iters. + assert_eq!( + TwoPhase::mine_solution(2).unwrap_err(), + ElectionError::Feasibility(FeasibilityError::WrongWinnerCount), + ); + }) + } + + #[test] + fn unsigned_per_dispatch_checks_can_only_submit_threshold_better() { + ExtBuilder::default() + .desired_targets(1) + .add_voter(7, 2, vec![10]) + .add_voter(8, 5, vec![10]) + .solution_improvement_threshold(Perbill::from_percent(50)) + .build_and_execute(|| { + roll_to(25); + assert!(TwoPhase::current_phase().is_unsigned()); + assert_eq!(TwoPhase::desired_targets().unwrap(), 1); + + // an initial solution + let result = ElectionResult { + // note: This second element of backing stake is not important here. + winners: vec![(10, 10)], + assignments: vec![Assignment { + who: 10, + distribution: vec![(10, PerU16::one())], + }], + }; + let (solution, witness) = TwoPhase::prepare_election_result(result).unwrap(); + assert_ok!(TwoPhase::unsigned_pre_dispatch_checks(&solution)); + assert_ok!(TwoPhase::submit_unsigned(Origin::none(), solution, witness)); + assert_eq!(TwoPhase::queued_solution().unwrap().score[0], 10); + + // trial 1: a solution who's score is only 2, i.e. 20% better in the first element. + let result = ElectionResult { + winners: vec![(10, 12)], + assignments: vec![ + Assignment { who: 10, distribution: vec![(10, PerU16::one())] }, + Assignment { + who: 7, + // note: this percent doesn't even matter, in compact it is 100%. + distribution: vec![(10, PerU16::one())], + }, + ], + }; + let (solution, _) = TwoPhase::prepare_election_result(result).unwrap(); + // 12 is not 50% more than 10 + assert_eq!(solution.score[0], 12); + assert_noop!( + TwoPhase::unsigned_pre_dispatch_checks(&solution), + Error::::WeakSubmission, + ); + // submitting this will actually panic. + + // trial 2: a solution who's score is only 7, i.e. 70% better in the first element. + let result = ElectionResult { + winners: vec![(10, 12)], + assignments: vec![ + Assignment { who: 10, distribution: vec![(10, PerU16::one())] }, + Assignment { who: 7, distribution: vec![(10, PerU16::one())] }, + Assignment { + who: 8, + // note: this percent doesn't even matter, in compact it is 100%. + distribution: vec![(10, PerU16::one())], + }, + ], + }; + let (solution, witness) = TwoPhase::prepare_election_result(result).unwrap(); + assert_eq!(solution.score[0], 17); + + // and it is fine + assert_ok!(TwoPhase::unsigned_pre_dispatch_checks(&solution)); + assert_ok!(TwoPhase::submit_unsigned(Origin::none(), solution, witness)); + }) + } + + #[test] + fn ocw_check_prevent_duplicate() { + let (mut ext, _) = ExtBuilder::default().build_offchainify(0); + ext.execute_with(|| { + roll_to(25); + assert!(TwoPhase::current_phase().is_unsigned()); + + // first execution -- okay. + assert!(TwoPhase::set_check_offchain_execution_status(25).is_ok()); + + // next block: rejected. + assert!(TwoPhase::set_check_offchain_execution_status(26).is_err()); + + // allowed after `OFFCHAIN_REPEAT` + assert!(TwoPhase::set_check_offchain_execution_status((26 + OFFCHAIN_REPEAT).into()) + .is_ok()); + + // a fork like situation: re-execute last 3. + assert!(TwoPhase::set_check_offchain_execution_status( + (26 + OFFCHAIN_REPEAT - 3).into() + ) + .is_err()); + assert!(TwoPhase::set_check_offchain_execution_status( + (26 + OFFCHAIN_REPEAT - 2).into() + ) + .is_err()); + assert!(TwoPhase::set_check_offchain_execution_status( + (26 + OFFCHAIN_REPEAT - 1).into() + ) + .is_err()); + }) + } + + #[test] + fn ocw_only_runs_when_signed_open_now() { + let (mut ext, pool) = ExtBuilder::default().build_offchainify(0); + ext.execute_with(|| { + roll_to(25); + assert_eq!(TwoPhase::current_phase(), Phase::Unsigned((true, 25))); + + // we must clear the offchain storage to ensure the offchain execution check doesn't get + // in the way. + let mut storage = StorageValueRef::persistent(&OFFCHAIN_HEAD_DB); + + TwoPhase::offchain_worker(24); + assert!(pool.read().transactions.len().is_zero()); + storage.clear(); + + TwoPhase::offchain_worker(26); + assert!(pool.read().transactions.len().is_zero()); + storage.clear(); + + // submits! + TwoPhase::offchain_worker(25); + assert!(!pool.read().transactions.len().is_zero()); + }) + } + + #[test] + fn ocw_can_submit_to_pool() { + let (mut ext, pool) = ExtBuilder::default().build_offchainify(0); + ext.execute_with(|| { + roll_to_with_ocw(25); + assert_eq!(TwoPhase::current_phase(), Phase::Unsigned((true, 25))); + // OCW must have submitted now + + let encoded = pool.read().transactions[0].clone(); + let extrinsic: Extrinsic = Decode::decode(&mut &*encoded).unwrap(); + let call = extrinsic.call; + assert!(matches!(call, OuterCall::TwoPhase(Call::submit_unsigned(_, _)))); + }) + } +} diff --git a/frame/election-provider-multi-phase/src/weights.rs b/frame/election-provider-multi-phase/src/weights.rs new file mode 100644 index 0000000000000..6070b771593ce --- /dev/null +++ b/frame/election-provider-multi-phase/src/weights.rs @@ -0,0 +1,147 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for pallet_two_phase_election_provider +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.1 +//! DATE: 2021-01-14, STEPS: [50, ], REPEAT: 20, LOW RANGE: [], HIGH RANGE: [] +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 + +// Executed Command: +// target/release/substrate +// benchmark +// --chain=dev +// --steps=50 +// --repeat=20 +// --pallet=pallet_two_phase_election_provider +// --extrinsic=* +// --execution=wasm +// --wasm-execution=compiled +// --heap-pages=4096 +// --output=./frame/two-phase-election-provider/src/weights.rs +// --template=./.maintain/frame-weight-template.hbs + +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{ + traits::Get, + weights::{Weight, constants::RocksDbWeight}, +}; +use sp_std::marker::PhantomData; + +/// Weight functions needed for pallet_two_phase_election_provider. +pub trait WeightInfo { + fn on_initialize_nothing() -> Weight; + fn on_initialize_open_signed() -> Weight; + fn on_initialize_open_unsigned_without_snapshot() -> Weight; + fn on_initialize_open_unsigned_with_snapshot() -> Weight; + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32) -> Weight; + fn feasibility_check(v: u32, t: u32, a: u32, d: u32) -> Weight; +} + +/// Weights for pallet_two_phase_election_provider using the Substrate node and recommended +/// hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + fn on_initialize_nothing() -> Weight { + (21_280_000 as Weight).saturating_add(T::DbWeight::get().reads(7 as Weight)) + } + fn on_initialize_open_signed() -> Weight { + (74_221_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + fn on_initialize_open_unsigned_with_snapshot() -> Weight { + (76_100_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + fn on_initialize_open_unsigned_without_snapshot() -> Weight { + (76_100_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32) -> Weight { + (0 as Weight) + // Standard Error: 21_000 + .saturating_add((2_606_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 21_000 + .saturating_add((11_405_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 108_000 + .saturating_add((2_651_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(6 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn feasibility_check(v: u32, t: u32, a: u32, d: u32) -> Weight { + (0 as Weight) + // Standard Error: 12_000 + .saturating_add((2_788_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 41_000 + .saturating_add((601_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 12_000 + .saturating_add((9_722_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 61_000 + .saturating_add((3_706_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + } +} + +// For backwards compatibility and tests +impl WeightInfo for () { + fn on_initialize_nothing() -> Weight { + (21_280_000 as Weight).saturating_add(RocksDbWeight::get().reads(7 as Weight)) + } + fn on_initialize_open_signed() -> Weight { + (74_221_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + fn on_initialize_open_unsigned_with_snapshot() -> Weight { + (76_100_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + fn on_initialize_open_unsigned_without_snapshot() -> Weight { + (76_100_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32) -> Weight { + (0 as Weight) + // Standard Error: 21_000 + .saturating_add((2_606_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 21_000 + .saturating_add((11_405_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 108_000 + .saturating_add((2_651_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(6 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn feasibility_check(v: u32, t: u32, a: u32, d: u32) -> Weight { + (0 as Weight) + // Standard Error: 12_000 + .saturating_add((2_788_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 41_000 + .saturating_add((601_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 12_000 + .saturating_add((9_722_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 61_000 + .saturating_add((3_706_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + } +} diff --git a/frame/grandpa/Cargo.toml b/frame/grandpa/Cargo.toml index c6a76de23e454..d12eb6060a1a9 100644 --- a/frame/grandpa/Cargo.toml +++ b/frame/grandpa/Cargo.toml @@ -38,6 +38,7 @@ pallet-offences = { version = "2.0.0", path = "../offences" } pallet-staking = { version = "2.0.0", path = "../staking" } pallet-staking-reward-curve = { version = "2.0.0", path = "../staking/reward-curve" } pallet-timestamp = { version = "2.0.0", path = "../timestamp" } +sp-election-providers = { version = "2.0.0", path = "../../primitives/election-providers" } [features] default = ["std"] diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index bf4ce5a519e7c..2c63424c5abd8 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -40,6 +40,7 @@ use sp_runtime::{ DigestItem, Perbill, }; use sp_staking::SessionIndex; +use sp_election_providers::onchain; impl_outer_origin! { pub enum Origin for Test {} @@ -194,6 +195,13 @@ parameter_types! { pub const StakingUnsignedPriority: u64 = u64::max_value() / 2; } +impl onchain::Config for Test { + type AccountId = ::AccountId; + type BlockNumber = ::BlockNumber; + type Accuracy = Perbill; + type DataProvider = Staking; +} + impl pallet_staking::Config for Test { type RewardRemainder = (); type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; @@ -216,6 +224,7 @@ impl pallet_staking::Config for Test { type MaxIterations = (); type MinSolutionScoreBump = (); type OffchainSolutionWeightLimit = (); + type ElectionProvider = onchain::OnChainSequentialPhragmen; type WeightInfo = (); } diff --git a/frame/offences/benchmarking/Cargo.toml b/frame/offences/benchmarking/Cargo.toml index 80492288d74bf..ede129ce77228 100644 --- a/frame/offences/benchmarking/Cargo.toml +++ b/frame/offences/benchmarking/Cargo.toml @@ -26,6 +26,7 @@ pallet-session = { version = "2.0.0", default-features = false, path = "../../se pallet-staking = { version = "2.0.0", default-features = false, features = ["runtime-benchmarks"], path = "../../staking" } sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" } sp-staking = { version = "2.0.0", default-features = false, path = "../../../primitives/staking" } +sp-election-providers = { version = "2.0.0", default-features = false, path = "../../../primitives/election-providers" } sp-std = { version = "2.0.0", default-features = false, path = "../../../primitives/std" } [dev-dependencies] @@ -50,6 +51,7 @@ std = [ "pallet-staking/std", "sp-runtime/std", "sp-staking/std", + "sp-election-providers/std", "sp-std/std", "codec/std", ] diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs index 8e0bb361e15ce..3ea1bbb8ebd1d 100644 --- a/frame/offences/benchmarking/src/mock.rs +++ b/frame/offences/benchmarking/src/mock.rs @@ -29,7 +29,7 @@ use sp_runtime::{ traits::{IdentityLookup, Block as BlockT}, testing::{Header, UintAuthorityId}, }; - +use sp_election_providers::onchain; type AccountId = u64; type AccountIndex = u32; @@ -147,6 +147,13 @@ parameter_types! { pub type Extrinsic = sp_runtime::testing::TestXt; +impl onchain::Config for Test { + type AccountId = AccountId; + type BlockNumber = BlockNumber; + type Accuracy = Perbill; + type DataProvider = Staking; +} + impl pallet_staking::Config for Test { type Currency = Balances; type UnixTime = pallet_timestamp::Module; @@ -169,6 +176,7 @@ impl pallet_staking::Config for Test { type MaxIterations = (); type MinSolutionScoreBump = (); type OffchainSolutionWeightLimit = (); + type ElectionProvider = onchain::OnChainSequentialPhragmen; type WeightInfo = (); } diff --git a/frame/session/benchmarking/Cargo.toml b/frame/session/benchmarking/Cargo.toml index fc3099e1b95cb..061af4d28e47b 100644 --- a/frame/session/benchmarking/Cargo.toml +++ b/frame/session/benchmarking/Cargo.toml @@ -16,6 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] sp-std = { version = "2.0.0", default-features = false, path = "../../../primitives/std" } sp-session = { version = "2.0.0", default-features = false, path = "../../../primitives/session" } sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" } +sp-election-providers = { version = "2.0.0", default-features = false, path = "../../../primitives/election-providers" } frame-system = { version = "2.0.0", default-features = false, path = "../../system" } frame-benchmarking = { version = "2.0.0", default-features = false, path = "../../benchmarking" } frame-support = { version = "2.0.0", default-features = false, path = "../../support" } @@ -37,6 +38,7 @@ default = ["std"] std = [ "sp-std/std", "sp-session/std", + "sp-election-providers/std", "sp-runtime/std", "frame-system/std", "frame-benchmarking/std", diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs index 31593b3da54b3..9519b0bc79459 100644 --- a/frame/session/benchmarking/src/mock.rs +++ b/frame/session/benchmarking/src/mock.rs @@ -21,6 +21,7 @@ use sp_runtime::traits::IdentityLookup; use frame_support::{impl_outer_origin, impl_outer_dispatch, parameter_types}; +use sp_election_providers::onchain; type AccountId = u64; type AccountIndex = u32; @@ -147,13 +148,21 @@ parameter_types! { pub type Extrinsic = sp_runtime::testing::TestXt; -impl frame_system::offchain::SendTransactionTypes for Test where +impl frame_system::offchain::SendTransactionTypes for Test +where Call: From, { type OverarchingCall = Call; type Extrinsic = Extrinsic; } +impl onchain::Config for Test { + type AccountId = AccountId; + type BlockNumber = BlockNumber; + type Accuracy = sp_runtime::Perbill; + type DataProvider = Staking; +} + impl pallet_staking::Config for Test { type Currency = Balances; type UnixTime = pallet_timestamp::Module; @@ -176,6 +185,7 @@ impl pallet_staking::Config for Test { type MaxIterations = (); type MinSolutionScoreBump = (); type OffchainSolutionWeightLimit = (); + type ElectionProvider = onchain::OnChainSequentialPhragmen; type WeightInfo = (); } diff --git a/frame/session/src/lib.rs b/frame/session/src/lib.rs index 90eba3815a7a5..23ff157b9e813 100644 --- a/frame/session/src/lib.rs +++ b/frame/session/src/lib.rs @@ -158,21 +158,33 @@ impl< } impl< - BlockNumber: Rem + Sub + Zero + PartialOrd + Saturating + Clone, - Period: Get, - Offset: Get, -> EstimateNextSessionRotation for PeriodicSessions { + BlockNumber: Rem + + Sub + + Zero + + PartialOrd + + Saturating + + Clone, + Period: Get, + Offset: Get, + > EstimateNextSessionRotation for PeriodicSessions +{ + fn average_session_length() -> BlockNumber { + Period::get() + } + fn estimate_next_session_rotation(now: BlockNumber) -> Option { let offset = Offset::get(); let period = Period::get(); Some(if now > offset { - let block_after_last_session = (now.clone() - offset) % period.clone(); + let block_after_last_session = (now.clone() - offset.clone()) % period.clone(); if block_after_last_session > Zero::zero() { - now.saturating_add( - period.saturating_sub(block_after_last_session) - ) + now.saturating_add(period.saturating_sub(block_after_last_session)) } else { - now + // this branch happens when the session is already rotated or will rotate in this + // block (depending on being called before or after `session::on_initialize`). Here, + // we assume the latter, namely that this is called after `session::on_initialize`, + // and thus we add period to it as well. + now + period } } else { offset @@ -851,6 +863,10 @@ impl EstimateNextNewSession for Module { T::NextSessionRotation::estimate_next_session_rotation(now) } + fn average_session_length() -> T::BlockNumber { + T::NextSessionRotation::average_session_length() + } + fn weight(now: T::BlockNumber) -> Weight { T::NextSessionRotation::weight(now) } diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 7c2fc21fde54e..93ec34025bd1e 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -17,6 +17,7 @@ static_assertions = "1.1.0" serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.6", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } +# TWO_PHASE_NOTE:: ideally we should be able to get rid of this. sp-npos-elections = { version = "2.0.0", default-features = false, path = "../../primitives/npos-elections" } sp-io ={ version = "2.0.0", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } @@ -25,6 +26,7 @@ frame-support = { version = "2.0.0", default-features = false, path = "../suppor frame-system = { version = "2.0.0", default-features = false, path = "../system" } pallet-session = { version = "2.0.0", default-features = false, features = ["historical"], path = "../session" } pallet-authorship = { version = "2.0.0", default-features = false, path = "../authorship" } +sp-election-providers = { version = "2.0.0", default-features = false, path = "../../primitives/election-providers" } sp-application-crypto = { version = "2.0.0", default-features = false, path = "../../primitives/application-crypto" } # Optional imports for benchmarking @@ -40,6 +42,7 @@ pallet-timestamp = { version = "2.0.0", path = "../timestamp" } pallet-staking-reward-curve = { version = "2.0.0", path = "../staking/reward-curve" } substrate-test-utils = { version = "2.0.0", path = "../../test-utils" } frame-benchmarking = { version = "2.0.0", path = "../benchmarking" } +sp-election-providers = { version = "2.0.0", features = ["runtime-benchmarks"], path = "../../primitives/election-providers" } rand_chacha = { version = "0.2" } parking_lot = "0.11.1" hex = "0.4" @@ -59,8 +62,10 @@ std = [ "frame-system/std", "pallet-authorship/std", "sp-application-crypto/std", + "sp-election-providers/std", ] runtime-benchmarks = [ "frame-benchmarking", + "sp-election-providers/runtime-benchmarks", "rand_chacha", ] diff --git a/frame/staking/fuzzer/Cargo.toml b/frame/staking/fuzzer/Cargo.toml index db65e347d8e2a..dd28df6180173 100644 --- a/frame/staking/fuzzer/Cargo.toml +++ b/frame/staking/fuzzer/Cargo.toml @@ -27,6 +27,7 @@ sp-std = { version = "2.0.0", path = "../../../primitives/std" } sp-io ={ version = "2.0.0", path = "../../../primitives/io" } sp-core = { version = "2.0.0", path = "../../../primitives/core" } sp-npos-elections = { version = "2.0.0", path = "../../../primitives/npos-elections" } +sp-election-providers = { version = "2.0.0", path = "../../../primitives/election-providers" } sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } [[bin]] diff --git a/frame/staking/fuzzer/src/mock.rs b/frame/staking/fuzzer/src/mock.rs index b3c9dd9f57b60..da6617d0519b3 100644 --- a/frame/staking/fuzzer/src/mock.rs +++ b/frame/staking/fuzzer/src/mock.rs @@ -151,13 +151,28 @@ parameter_types! { pub type Extrinsic = sp_runtime::testing::TestXt; -impl frame_system::offchain::SendTransactionTypes for Test where +impl frame_system::offchain::SendTransactionTypes for Test +where Call: From, { type OverarchingCall = Call; type Extrinsic = Extrinsic; } +pub struct MockElectionProvider; +impl sp_election_providers::ElectionProvider for MockElectionProvider { + type Error = (); + type DataProvider = pallet_staking::Module; + + fn elect() -> Result, Self::Error> { + Err(()) + } + + fn ongoing() -> bool { + false + } +} + impl pallet_staking::Config for Test { type Currency = Balances; type UnixTime = pallet_timestamp::Module; @@ -181,4 +196,5 @@ impl pallet_staking::Config for Test { type UnsignedPriority = (); type OffchainSolutionWeightLimit = (); type WeightInfo = (); + type ElectionProvider = MockElectionProvider; } diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 795f222158e05..109145ede5e31 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -232,10 +232,11 @@ //! //! The controller account can free a portion (or all) of the funds using the //! [`unbond`](enum.Call.html#variant.unbond) call. Note that the funds are not immediately -//! accessible. Instead, a duration denoted by [`BondingDuration`](./trait.Config.html#associatedtype.BondingDuration) -//! (in number of eras) must pass until the funds can actually be removed. Once the -//! `BondingDuration` is over, the [`withdraw_unbonded`](./enum.Call.html#variant.withdraw_unbonded) -//! call can be used to actually withdraw the funds. +//! accessible. Instead, a duration denoted by +//! [`BondingDuration`](./trait.Config.html#associatedtype.BondingDuration) (in number of eras) must +//! pass until the funds can actually be removed. Once the `BondingDuration` is over, the +//! [`withdraw_unbonded`](./enum.Call.html#variant.withdraw_unbonded) call can be used to actually +//! withdraw the funds. //! //! Note that there is a limitation to the number of fund-chunks that can be scheduled to be //! unlocked in the future via [`unbond`](enum.Call.html#variant.unbond). In case this maximum @@ -304,7 +305,7 @@ use frame_support::{ }; use pallet_session::historical; use sp_runtime::{ - Percent, Perbill, PerU16, PerThing, InnerOf, RuntimeDebug, DispatchError, + Percent, Perbill, PerU16, InnerOf, RuntimeDebug, DispatchError, curve::PiecewiseLinear, traits::{ Convert, Zero, StaticLookup, CheckedSub, Saturating, SaturatedConversion, @@ -327,15 +328,14 @@ use frame_system::{ }; use sp_npos_elections::{ ExtendedBalance, Assignment, ElectionScore, ElectionResult as PrimitiveElectionResult, - build_support_map, evaluate_support, seq_phragmen, generate_solution_type, - is_score_better, VotingLimit, SupportMap, VoteWeight, + to_supports, EvaluateSupport, seq_phragmen, generate_solution_type, is_score_better, Supports, + VoteWeight, CompactSolution, PerThing128, }; +use sp_election_providers::ElectionProvider; pub use weights::WeightInfo; const STAKING_ID: LockIdentifier = *b"staking "; pub const MAX_UNLOCKING_CHUNKS: usize = 32; -pub const MAX_NOMINATIONS: usize = ::LIMIT; - pub(crate) const LOG_TARGET: &'static str = "staking"; // syntactic sugar for logging. @@ -344,7 +344,7 @@ macro_rules! log { ($level:tt, $patter:expr $(, $values:expr)* $(,)?) => { frame_support::debug::$level!( target: crate::LOG_TARGET, - $patter $(, $values)* + concat!("💸 ", $patter) $(, $values)* ) }; } @@ -364,6 +364,8 @@ static_assertions::const_assert!(size_of::() <= size_of::() /// Maximum number of stakers that can be stored in a snapshot. pub(crate) const MAX_VALIDATORS: usize = ValidatorIndex::max_value() as usize; pub(crate) const MAX_NOMINATORS: usize = NominatorIndex::max_value() as usize; +pub const MAX_NOMINATIONS: usize = + ::LIMIT; /// Counter for the number of eras that have passed. pub type EraIndex = u32; @@ -387,10 +389,12 @@ pub type OffchainAccuracy = PerU16; pub type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; -type PositiveImbalanceOf = - <::Currency as Currency<::AccountId>>::PositiveImbalance; -type NegativeImbalanceOf = - <::Currency as Currency<::AccountId>>::NegativeImbalance; +type PositiveImbalanceOf = <::Currency as Currency< + ::AccountId, +>>::PositiveImbalance; +type NegativeImbalanceOf = <::Currency as Currency< + ::AccountId, +>>::NegativeImbalance; /// Information regarding the active era (era in used in session). #[derive(Encode, Decode, RuntimeDebug)] @@ -772,7 +776,7 @@ impl SessionInterface<::AccountId> for T w pub trait Config: frame_system::Config + SendTransactionTypes> { /// The staking balance. - type Currency: LockableCurrency; + type Currency: LockableCurrency; /// Time used for computing era duration. /// @@ -787,6 +791,14 @@ pub trait Config: frame_system::Config + SendTransactionTypes> { /// [`BalanceOf`]. type CurrencyToVote: CurrencyToVote>; + /// Something that provides the election functionality. + type ElectionProvider: sp_election_providers::ElectionProvider< + Self::AccountId, + Self::BlockNumber, + // we only accept an election provider that has staking as data provider. + DataProvider = Module, + >; + /// Tokens have been minted and are unused for validator-reward. /// See [Era payout](./index.html#era-payout). type RewardRemainder: OnUnbalanced>; @@ -883,7 +895,9 @@ pub enum Forcing { } impl Default for Forcing { - fn default() -> Self { Forcing::NotForcing } + fn default() -> Self { + Forcing::NotForcing + } } // A value placed in storage that represents the current version of the Staking storage. This value @@ -1059,28 +1073,45 @@ decl_storage! { /// The earliest era for which we have a pending, unapplied slash. EarliestUnappliedSlash: Option; + /// The last planned session scheduled by the session pallet. + /// + /// This is basically in sync with the call to [`SessionManager::new_session`]. + pub CurrentPlannedSession get(fn current_planned_session): SessionIndex; + /// Snapshot of validators at the beginning of the current election window. This should only /// have a value when [`EraElectionStatus`] == `ElectionStatus::Open(_)`. + /// + /// TWO_PHASE_NOTE: should be removed once we switch to multi-phase. pub SnapshotValidators get(fn snapshot_validators): Option>; /// Snapshot of nominators at the beginning of the current election window. This should only /// have a value when [`EraElectionStatus`] == `ElectionStatus::Open(_)`. + /// + /// TWO_PHASE_NOTE: should be removed once we switch to multi-phase. pub SnapshotNominators get(fn snapshot_nominators): Option>; /// The next validator set. At the end of an era, if this is available (potentially from the /// result of an offchain worker), it is immediately used. Otherwise, the on-chain election /// is executed. + /// + /// TWO_PHASE_NOTE: should be removed once we switch to multi-phase. pub QueuedElected get(fn queued_elected): Option>>; /// The score of the current [`QueuedElected`]. + /// + /// TWO_PHASE_NOTE: should be removed once we switch to multi-phase. pub QueuedScore get(fn queued_score): Option; /// Flag to control the execution of the offchain election. When `Open(_)`, we accept /// solutions to be submitted. + /// + /// TWO_PHASE_NOTE: should be removed once we switch to multi-phase. pub EraElectionStatus get(fn era_election_status): ElectionStatus; /// True if the current **planned** session is final. Note that this does not take era /// forcing into account. + /// + /// TWO_PHASE_NOTE: should be removed once we switch to multi-phase. pub IsCurrentSessionFinal get(fn is_current_session_final): bool = false; /// True if network has been upgraded to this version. @@ -1298,14 +1329,14 @@ decl_module! { ElectionStatus::::Open(now) ); add_weight(0, 1, 0); - log!(info, "💸 Election window is Open({:?}). Snapshot created", now); + log!(info, "Election window is Open({:?}). Snapshot created", now); } else { - log!(warn, "💸 Failed to create snapshot at {:?}.", now); + log!(warn, "Failed to create snapshot at {:?}.", now); } } } } else { - log!(warn, "💸 Estimating next session change failed."); + log!(warn, "Estimating next session change failed."); } add_weight(0, 0, T::NextNewSession::weight(now)) } @@ -1320,16 +1351,15 @@ decl_module! { /// to open. If so, it runs the offchain worker code. fn offchain_worker(now: T::BlockNumber) { use offchain_election::{set_check_offchain_execution_status, compute_offchain_election}; - if Self::era_election_status().is_open_at(now) { let offchain_status = set_check_offchain_execution_status::(now); if let Err(why) = offchain_status { - log!(warn, "💸 skipping offchain worker in open election window due to [{}]", why); + log!(warn, "skipping offchain worker in open election window due to [{}]", why); } else { if let Err(e) = compute_offchain_election::() { - log!(error, "💸 Error in election offchain worker: {:?}", e); + log!(error, "Error in election offchain worker: {:?}", e); } else { - log!(debug, "💸 Executed offchain worker thread without errors."); + log!(debug, "Executed offchain worker thread without errors."); } } } @@ -2102,7 +2132,7 @@ decl_module! { #[weight = T::WeightInfo::submit_solution_better( size.validators.into(), size.nominators.into(), - compact.len() as u32, + compact.voter_count() as u32, winners.len() as u32, )] pub fn submit_election_solution( @@ -2136,7 +2166,7 @@ decl_module! { #[weight = T::WeightInfo::submit_solution_better( size.validators.into(), size.nominators.into(), - compact.len() as u32, + compact.voter_count() as u32, winners.len() as u32, )] pub fn submit_election_solution_unsigned( @@ -2175,7 +2205,10 @@ impl Module { } /// Internal impl of [`Self::slashable_balance_of`] that returns [`VoteWeight`]. - pub fn slashable_balance_of_vote_weight(stash: &T::AccountId, issuance: BalanceOf) -> VoteWeight { + pub fn slashable_balance_of_vote_weight( + stash: &T::AccountId, + issuance: BalanceOf, + ) -> VoteWeight { T::CurrencyToVote::to_vote(Self::slashable_balance_of(stash), issuance) } @@ -2214,7 +2247,7 @@ impl Module { { log!( warn, - "💸 Snapshot size too big [{} <> {}][{} <> {}].", + "Snapshot size too big [{} <> {}][{} <> {}].", num_validators, MAX_VALIDATORS, num_nominators, @@ -2238,10 +2271,7 @@ impl Module { >::kill(); } - fn do_payout_stakers( - validator_stash: T::AccountId, - era: EraIndex, - ) -> DispatchResult { + fn do_payout_stakers(validator_stash: T::AccountId, era: EraIndex) -> DispatchResult { // Validate input data let current_era = CurrentEra::get().ok_or(Error::::InvalidEraToReward)?; ensure!(era <= current_era, Error::::InvalidEraToReward); @@ -2534,7 +2564,7 @@ impl Module { validator_at, ).map_err(|e| { // log the error since it is not propagated into the runtime error. - log!(warn, "💸 un-compacting solution failed due to {:?}", e); + log!(warn, "un-compacting solution failed due to {:?}", e); Error::::OffchainElectionBogusCompact })?; @@ -2549,7 +2579,7 @@ impl Module { // all of the indices must map to either a validator or a nominator. If this is ever // not the case, then the locking system of staking is most likely faulty, or we // have bigger problems. - log!(error, "💸 detected an error in the staking locking and snapshot."); + log!(error, "detected an error in the staking locking and snapshot."); // abort. return Err(Error::::OffchainElectionBogusNominator.into()); } @@ -2598,20 +2628,19 @@ impl Module { ); // build the support map thereof in order to evaluate. - let supports = build_support_map::( - &winners, - &staked_assignments, - ).map_err(|_| Error::::OffchainElectionBogusEdge)?; + let supports = to_supports(&winners, &staked_assignments) + .map_err(|_| Error::::OffchainElectionBogusEdge)?; // Check if the score is the same as the claimed one. - let submitted_score = evaluate_support(&supports); + let submitted_score = (&supports).evaluate(); ensure!(submitted_score == claimed_score, Error::::OffchainElectionBogusScore); // At last, alles Ok. Exposures and store the result. - let exposures = Self::collect_exposure(supports); + let exposures = Self::collect_exposures(supports); log!( info, - "💸 A better solution (with compute {:?} and score {:?}) has been validated and stored on chain.", + "A better solution (with compute {:?} and score {:?}) has been validated and stored \ + on chain.", compute, submitted_score, ); @@ -2744,6 +2773,7 @@ impl Module { // Set staking information for new era. let maybe_new_validators = Self::select_and_update_validators(current_era); + let _unused_new_validators = Self::enact_election(current_era); maybe_new_validators } @@ -2811,7 +2841,7 @@ impl Module { log!( info, - "💸 new validator set of size {:?} has been elected via {:?} for era {:?}", + "new validator set of size {:?} has been elected via {:?} for staring era {:?}", elected_stashes.len(), compute, current_era, @@ -2860,20 +2890,20 @@ impl Module { Self::slashable_balance_of_fn(), ); - let supports = build_support_map::( + let supports = to_supports( &elected_stashes, &staked_assignments, ) .map_err(|_| log!( error, - "💸 on-chain phragmen is failing due to a problem in the result. This must be a bug." + "on-chain phragmen is failing due to a problem in the result. This must be a bug." ) ) .ok()?; // collect exposures - let exposures = Self::collect_exposure(supports); + let exposures = Self::collect_exposures(supports); // In order to keep the property required by `on_session_ending` that we must return the // new validator set even if it's the same as the old, as long as any underlying @@ -2899,7 +2929,7 @@ impl Module { /// Self votes are added and nominations before the most recent slashing span are ignored. /// /// No storage item is updated. - pub fn do_phragmen( + pub fn do_phragmen( iterations: usize, ) -> Option> where @@ -2938,7 +2968,7 @@ impl Module { // If we don't have enough candidates, nothing to do. log!( warn, - "💸 Chain does not have enough staking candidates to operate. Era {:?}.", + "chain does not have enough staking candidates to operate. Era {:?}.", Self::current_era() ); None @@ -2949,14 +2979,15 @@ impl Module { all_nominators, Some((iterations, 0)), // exactly run `iterations` rounds. ) - .map_err(|err| log!(error, "Call to seq-phragmen failed due to {}", err)) + .map_err(|err| log!(error, "Call to seq-phragmen failed due to {:?}", err)) .ok() } } - /// Consume a set of [`Supports`] from [`sp_npos_elections`] and collect them into a [`Exposure`] - fn collect_exposure( - supports: SupportMap, + /// Consume a set of [`Supports`] from [`sp_npos_elections`] and collect them into a + /// [`Exposure`]. + fn collect_exposures( + supports: Supports, ) -> Vec<(T::AccountId, Exposure>)> { let total_issuance = T::Currency::total_issuance(); let to_currency = |e: ExtendedBalance| T::CurrencyToVote::to_currency(e, total_issuance); @@ -2988,6 +3019,80 @@ impl Module { }).collect::)>>() } + /// Process the output of the election. + /// + /// This ensures enough validators have been elected, converts all supports to exposures and + /// writes them to the associated storage. + /// + /// Returns `Err(())` if less than [`MinimumValidatorCount`] validators have been elected, `Ok` + /// otherwise. + // TWO_PHASE_NOTE: the deadcode + #[allow(dead_code)] + pub fn process_election( + flat_supports: sp_npos_elections::Supports, + current_era: EraIndex, + ) -> Result, ()> { + let exposures = Self::collect_exposures(flat_supports); + let elected_stashes = exposures.iter().cloned().map(|(x, _)| x).collect::>(); + + if (elected_stashes.len() as u32) <= Self::minimum_validator_count() { + log!( + warn, + "chain does not have enough staking candidates to operate for era {:?}", + current_era, + ); + return Err(()); + } + + // Populate Stakers and write slot stake. + let mut total_stake: BalanceOf = Zero::zero(); + exposures.into_iter().for_each(|(stash, exposure)| { + total_stake = total_stake.saturating_add(exposure.total); + >::insert(current_era, &stash, &exposure); + + let mut exposure_clipped = exposure; + let clipped_max_len = T::MaxNominatorRewardedPerValidator::get() as usize; + if exposure_clipped.others.len() > clipped_max_len { + exposure_clipped.others.sort_by(|a, b| a.value.cmp(&b.value).reverse()); + exposure_clipped.others.truncate(clipped_max_len); + } + >::insert(¤t_era, &stash, exposure_clipped); + }); + + // Insert current era staking information + >::insert(¤t_era, total_stake); + + // collect the pref of all winners + for stash in &elected_stashes { + let pref = Self::validators(stash); + >::insert(¤t_era, stash, pref); + } + + // emit event + // TWO_PHASE_NOTE: remove the inner value. + Self::deposit_event(RawEvent::StakingElection(ElectionCompute::Signed)); + + log!( + info, + "new validator set of size {:?} has been processed for era {:?}", + elected_stashes.len(), + current_era, + ); + + Ok(elected_stashes) + } + + /// Enact and process the election using the `ElectionProvider` type. + /// + /// This will also process the election, as noted in [`process_election`]. + fn enact_election(_current_era: EraIndex) -> Option> { + let outcome = T::ElectionProvider::elect().map(|_| ()); + log!(debug, "Experimental election provider outputted {:?}", outcome); + // TWO_PHASE_NOTE: This code path shall not return anything for now. Later on, redirect the + // results to `process_election`. + None + } + /// Remove all associated data of a stash account from the staking system. /// /// Assumes storage is upgraded before calling. @@ -3080,7 +3185,11 @@ impl Module { } #[cfg(feature = "runtime-benchmarks")] - pub fn add_era_stakers(current_era: EraIndex, controller: T::AccountId, exposure: Exposure>) { + pub fn add_era_stakers( + current_era: EraIndex, + controller: T::AccountId, + exposure: Exposure>, + ) { >::insert(¤t_era, &controller, &exposure); } @@ -3093,6 +3202,106 @@ impl Module { pub fn set_slash_reward_fraction(fraction: Perbill) { SlashRewardFraction::put(fraction); } + + /// Get all of the voters that are eligible for the npos election. + /// + /// This will use all on-chain nominators, and all the validators will inject a self vote. + /// + /// ### Slashing + /// + /// All nominations that have been submitted before the last non-zero slash of the validator are + /// auto-chilled. + /// + /// Note that this is VERY expensive. Use with care. + pub fn get_npos_voters() -> Vec<(T::AccountId, VoteWeight, Vec)> { + let weight_of = Self::slashable_balance_of_fn(); + let mut all_voters = Vec::new(); + + for (validator, _) in >::iter() { + // append self vote + let self_vote = (validator.clone(), weight_of(&validator), vec![validator.clone()]); + all_voters.push(self_vote); + } + + for (nominator, nominations) in >::iter() { + let Nominations { submitted_in, mut targets, suppressed: _ } = nominations; + + // Filter out nomination targets which were nominated before the most recent + // slashing span. + targets.retain(|stash| { + Self::slashing_spans(&stash) + .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) + }); + + let vote_weight = weight_of(&nominator); + all_voters.push((nominator, vote_weight, targets)) + } + + all_voters + } + + pub fn get_npos_targets() -> Vec { + >::iter().map(|(v, _)| v).collect::>() + } +} + +impl sp_election_providers::ElectionDataProvider + for Module +{ + fn desired_targets() -> u32 { + Self::validator_count() + } + + fn voters() -> Vec<(T::AccountId, VoteWeight, Vec)> { + Self::get_npos_voters() + } + + fn targets() -> Vec { + Self::get_npos_targets() + } + + fn next_election_prediction(now: T::BlockNumber) -> T::BlockNumber { + let current_era = Self::current_era().unwrap_or(0); + let current_session = Self::current_planned_session(); + let current_era_start_session_index = + Self::eras_start_session_index(current_era).unwrap_or(0); + let era_length = current_session + .saturating_sub(current_era_start_session_index) + .min(T::SessionsPerEra::get()); + + let session_length = T::NextNewSession::average_session_length(); + + let until_this_session_end = T::NextNewSession::estimate_next_new_session(now) + .unwrap_or_default() + .saturating_sub(now); + + let sessions_left: T::BlockNumber = T::SessionsPerEra::get() + .saturating_sub(era_length) + // one session is computed in this_session_end. + .saturating_sub(1) + .into(); + + now.saturating_add( + until_this_session_end.saturating_add(sessions_left.saturating_mul(session_length)), + ) + } + + #[cfg(any(feature = "runtime-benchmarks", test))] + fn put_snapshot( + voters: Vec<(T::AccountId, VoteWeight, Vec)>, + targets: Vec, + ) { + targets.into_iter().for_each(|v| { + >::insert(v, ValidatorPrefs { commission: Perbill::zero() }); + }); + + voters.into_iter().for_each(|(v, _s, t)| { + >::insert( + v, + Nominations { targets: t, submitted_in: 0, suppressed: false }, + ); + }); + } } /// In this implementation `new_session(session)` must be called before `end_session(session-1)` @@ -3108,6 +3317,7 @@ impl pallet_session::SessionManager for Module { >::block_number(), new_index ); + CurrentPlannedSession::put(new_index); Self::new_session(new_index) } fn start_session(start_index: SessionIndex) { @@ -3130,10 +3340,12 @@ impl pallet_session::SessionManager for Module { } } -impl historical::SessionManager>> for Module { - fn new_session(new_index: SessionIndex) - -> Option>)>> - { +impl historical::SessionManager>> + for Module +{ + fn new_session( + new_index: SessionIndex, + ) -> Option>)>> { >::new_session(new_index).map(|validators| { let current_era = Self::current_era() // Must be some as a new era has been created. @@ -3158,8 +3370,8 @@ impl historical::SessionManager pallet_authorship::EventHandler for Module - where - T: Config + pallet_authorship::Config + pallet_session::Config +where + T: Config + pallet_authorship::Config + pallet_session::Config, { fn note_author(author: T::AccountId) { Self::reward_by_ids(vec![(author, 20)]) @@ -3202,9 +3414,10 @@ impl Convert } /// This is intended to be used with `FilterHistoricalOffences`. -impl +impl OnOffenceHandler, Weight> -for Module where + for Module +where T: pallet_session::Config::AccountId>, T: pallet_session::historical::Config< FullIdentification = Exposure<::AccountId, BalanceOf>, @@ -3218,12 +3431,15 @@ for Module where >, { fn on_offence( - offenders: &[OffenceDetails>], + offenders: &[OffenceDetails< + T::AccountId, + pallet_session::historical::IdentificationTuple, + >], slash_fraction: &[Perbill], slash_session: SessionIndex, ) -> Result { if !Self::can_report() { - return Err(()) + return Err(()); } let reward_proportion = SlashRewardFraction::get(); @@ -3334,6 +3550,7 @@ for Module where } fn can_report() -> bool { + // TWO_PHASE_NOTE: we can get rid of this API Self::era_election_status().is_closed() } } @@ -3344,7 +3561,8 @@ pub struct FilterHistoricalOffences { } impl ReportOffence - for FilterHistoricalOffences, R> where + for FilterHistoricalOffences, R> +where T: Config, R: ReportOffence, O: Offence, @@ -3401,7 +3619,7 @@ impl frame_support::unsigned::ValidateUnsigned for Module { return invalid.into(); } - log!(debug, "💸 validateUnsigned succeeded for a solution at era {}.", era); + log!(debug, "validateUnsigned succeeded for a solution at era {}.", era); ValidTransaction::with_tag_prefix("StakingOffchain") // The higher the score[0], the better a solution is. diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 048806b062395..d01f8b59a681f 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -27,7 +27,7 @@ use frame_support::{ use sp_core::H256; use sp_io; use sp_npos_elections::{ - build_support_map, evaluate_support, reduce, ExtendedBalance, StakedAssignment, ElectionScore, + to_supports, reduce, ExtendedBalance, StakedAssignment, ElectionScore, EvaluateSupport, }; use sp_runtime::{ curve::PiecewiseLinear, @@ -36,6 +36,7 @@ use sp_runtime::{ }; use sp_staking::offence::{OffenceDetails, OnOffenceHandler}; use std::{cell::RefCell, collections::HashSet}; +use sp_election_providers::onchain; pub const INIT_TIMESTAMP: u64 = 30_000; pub const BLOCK_TIME: u64 = 1000; @@ -252,6 +253,12 @@ impl OnUnbalanced> for RewardRemainderMock { } } +impl onchain::Config for Test { + type AccountId = AccountId; + type BlockNumber = BlockNumber; + type Accuracy = Perbill; + type DataProvider = Staking; +} impl Config for Test { type Currency = Balances; type UnixTime = Timestamp; @@ -274,6 +281,7 @@ impl Config for Test { type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type UnsignedPriority = UnsignedPriority; type OffchainSolutionWeightLimit = OffchainSolutionWeightLimit; + type ElectionProvider = onchain::OnChainSequentialPhragmen; type WeightInfo = (); } @@ -769,7 +777,7 @@ pub(crate) fn add_slash(who: &AccountId) { on_offence_now( &[ OffenceDetails { - offender: (who.clone(), Staking::eras_stakers(Staking::active_era().unwrap().index, who.clone())), + offender: (who.clone(), Staking::eras_stakers(active_era(), who.clone())), reporters: vec![], }, ], @@ -850,8 +858,8 @@ pub(crate) fn horrible_npos_solution( let score = { let (_, _, better_score) = prepare_submission_with(true, true, 0, |_| {}); - let support = build_support_map::(&winners, &staked_assignment).unwrap(); - let score = evaluate_support(&support); + let support = to_supports::(&winners, &staked_assignment).unwrap(); + let score = support.evaluate(); assert!(sp_npos_elections::is_score_better::( better_score, @@ -950,11 +958,11 @@ pub(crate) fn prepare_submission_with( Staking::slashable_balance_of_fn(), ); - let support_map = build_support_map::( + let support_map = to_supports( winners.as_slice(), staked.as_slice(), ).unwrap(); - evaluate_support::(&support_map) + support_map.evaluate() } else { Default::default() }; @@ -971,9 +979,8 @@ pub(crate) fn prepare_submission_with( /// Make all validator and nominator request their payment pub(crate) fn make_all_reward_payment(era: EraIndex) { - let validators_with_reward = ErasRewardPoints::::get(era).individual.keys() - .cloned() - .collect::>(); + let validators_with_reward = + ErasRewardPoints::::get(era).individual.keys().cloned().collect::>(); // reward validators for validator_controller in validators_with_reward.iter().filter_map(Staking::bonded) { @@ -990,19 +997,19 @@ pub(crate) fn make_all_reward_payment(era: EraIndex) { macro_rules! assert_session_era { ($session:expr, $era:expr) => { assert_eq!( - Session::current_index(), - $session, - "wrong session {} != {}", - Session::current_index(), - $session, - ); - assert_eq!( - Staking::active_era().unwrap().index, - $era, - "wrong active era {} != {}", - Staking::active_era().unwrap().index, - $era, - ); + Session::current_index(), + $session, + "wrong session {} != {}", + Session::current_index(), + $session, + ); + assert_eq!( + Staking::current_era().unwrap(), + $era, + "wrong current era {} != {}", + Staking::current_era().unwrap(), + $era, + ); }; } diff --git a/frame/staking/src/offchain_election.rs b/frame/staking/src/offchain_election.rs index 433e02261cc58..bec4174ad42d3 100644 --- a/frame/staking/src/offchain_election.rs +++ b/frame/staking/src/offchain_election.rs @@ -25,8 +25,8 @@ use codec::Decode; use frame_support::{traits::Get, weights::Weight, IterableStorageMap}; use frame_system::offchain::SubmitTransaction; use sp_npos_elections::{ - build_support_map, evaluate_support, reduce, Assignment, ElectionResult, ElectionScore, - ExtendedBalance, + to_supports, EvaluateSupport, reduce, Assignment, ElectionResult, ElectionScore, + ExtendedBalance, CompactSolution, }; use sp_runtime::{ offchain::storage::StorageValueRef, traits::TrailingZeroInput, PerThing, RuntimeDebug, @@ -127,7 +127,7 @@ pub(crate) fn compute_offchain_election() -> Result<(), OffchainElect crate::log!( info, - "💸 prepared a seq-phragmen solution with {} balancing iterations and score {:?}", + "prepared a seq-phragmen solution with {} balancing iterations and score {:?}", iters, score, ); @@ -265,7 +265,7 @@ pub fn trim_to_weight( where for<'r> FN: Fn(&'r T::AccountId) -> Option, { - match compact.len().checked_sub(maximum_allowed_voters as usize) { + match compact.voter_count().checked_sub(maximum_allowed_voters as usize) { Some(to_remove) if to_remove > 0 => { // grab all voters and sort them by least stake. let balance_of = >::slashable_balance_of_fn(); @@ -284,7 +284,7 @@ where if compact.remove_voter(index) { crate::log!( trace, - "💸 removed a voter at index {} with stake {:?} from compact to reduce the size", + "removed a voter at index {} with stake {:?} from compact to reduce the size", index, _stake, ); @@ -297,19 +297,17 @@ where } crate::log!( - warn, - "💸 {} nominators out of {} had to be removed from compact solution due to size limits.", - removed, - compact.len() + removed, - ); + warn, + "{} nominators out of {} had to be removed from compact solution due to size \ + limits.", + removed, + compact.voter_count() + removed, + ); Ok(compact) } _ => { // nada, return as-is - crate::log!( - info, - "💸 Compact solution did not get trimmed due to block weight limits.", - ); + crate::log!(info, "Compact solution did not get trimmed due to block weight limits.",); Ok(compact) } } @@ -324,12 +322,7 @@ pub fn prepare_submission( do_reduce: bool, maximum_weight: Weight, ) -> Result< - ( - Vec, - CompactAssignments, - ElectionScore, - ElectionSize, - ), + (Vec, CompactAssignments, ElectionScore, ElectionSize), OffchainElectionError, > where @@ -398,16 +391,19 @@ where let maximum_allowed_voters = maximum_compact_len::(winners.len() as u32, size, maximum_weight); - crate::log!(debug, "💸 Maximum weight = {:?} // current weight = {:?} // maximum voters = {:?} // current votes = {:?}", + crate::log!( + debug, + "Maximum weight = {:?} // current weight = {:?} // maximum voters = {:?} // current votes \ + = {:?}", maximum_weight, T::WeightInfo::submit_solution_better( - size.validators.into(), - size.nominators.into(), - compact.len() as u32, - winners.len() as u32, + size.validators.into(), + size.nominators.into(), + compact.voter_count() as u32, + winners.len() as u32, ), maximum_allowed_voters, - compact.len(), + compact.voter_count(), ); let compact = trim_to_weight::(maximum_allowed_voters, compact, &nominator_index)?; @@ -423,9 +419,9 @@ where >::slashable_balance_of_fn(), ); - let support_map = build_support_map::(&winners, &staked) + let support_map = to_supports::(&winners, &staked) .map_err(|_| OffchainElectionError::ElectionFailed)?; - evaluate_support::(&support_map) + support_map.evaluate() }; // winners to index. Use a simple for loop for a more expressive early exit in case of error. diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index d3139b53e6f97..f96d23ffb5c3b 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -244,11 +244,9 @@ pub fn get_weak_solution( >::slashable_balance_of_fn(), ); - let support_map = build_support_map::( - winners.as_slice(), - staked.as_slice(), - ).unwrap(); - evaluate_support::(&support_map) + let support_map = + to_supports::(winners.as_slice(), staked.as_slice()).unwrap(); + support_map.evaluate() }; // compact encode the assignment. diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index bf0b2bf0da484..56a7e4388927a 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -1788,6 +1788,7 @@ fn bond_with_duplicate_vote_should_be_ignored_by_npos_election() { .minimum_validator_count(1) .build() .execute_with(|| { + // disable the nominator assert_ok!(Staking::chill(Origin::signed(100))); // make stakes equal. @@ -1808,6 +1809,7 @@ fn bond_with_duplicate_vote_should_be_ignored_by_npos_election() { } assert_ok!(Staking::bond(Origin::signed(1), 2, 1000, RewardDestination::Controller)); + // 11 should not be elected. All of these count as ONE vote. assert_ok!(Staking::nominate(Origin::signed(2), vec![11, 11, 11, 21, 31,])); assert_ok!(Staking::bond(Origin::signed(3), 4, 1000, RewardDestination::Controller)); @@ -1861,7 +1863,6 @@ fn bond_with_duplicate_vote_should_be_ignored_by_npos_election_elected() { assert_ok!(Staking::nominate(Origin::signed(4), vec![21, 31])); // winners should be 21 and 31. Otherwise this election is taking duplicates into account. - let sp_npos_elections::ElectionResult { winners, assignments, @@ -2004,7 +2005,7 @@ fn reward_from_authorship_event_handler_works() { fn add_reward_points_fns_works() { ExtBuilder::default().build_and_execute(|| { // Not mandatory but must be coherent with rewards - assert_eq!(Session::validators(), vec![21, 11]); + assert_eq_uvec!(Session::validators(), vec![21, 11]); >::reward_by_ids(vec![ (21, 1), @@ -3023,7 +3024,7 @@ mod offchain_election { assert_eq!(Staking::era_election_status(), ElectionStatus::Open(37)); run_to_block(40); - assert_session_era!(4, 0); + assert_session_era!(4, 1); assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); assert!(Staking::snapshot_nominators().is_none()); assert!(Staking::snapshot_validators().is_none()); @@ -3041,7 +3042,7 @@ mod offchain_election { assert!(Staking::snapshot_validators().is_some()); run_to_block(90); - assert_session_era!(9, 1); + assert_session_era!(9, 2); assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); assert!(Staking::snapshot_nominators().is_none()); assert!(Staking::snapshot_validators().is_none()); @@ -4950,3 +4951,92 @@ fn cannot_bond_extra_to_lower_than_ed() { ); }) } + +mod election_data_provider { + use super::*; + use sp_election_providers::ElectionDataProvider; + + #[test] + fn voters_include_self_vote() { + ExtBuilder::default().nominate(false).build().execute_with(|| { + assert!(>::iter().map(|(x, _)| x).all(|v| Staking::voters() + .into_iter() + .find(|(w, _, t)| { v == *w && t[0] == *w }) + .is_some())) + }) + } + + #[test] + fn voters_exclude_slashed() { + ExtBuilder::default().build().execute_with(|| { + assert_eq!(Staking::nominators(101).unwrap().targets, vec![11, 21]); + assert_eq!( + >::voters() + .iter() + .find(|x| x.0 == 101) + .unwrap() + .2, + vec![11, 21] + ); + + start_active_era(1); + add_slash(&11); + + // 11 is gone. + start_active_era(2); + assert_eq!( + >::voters() + .iter() + .find(|x| x.0 == 101) + .unwrap() + .2, + vec![21] + ); + + // resubmit and it is back + assert_ok!(Staking::nominate(Origin::signed(100), vec![11, 21])); + assert_eq!( + >::voters() + .iter() + .find(|x| x.0 == 101) + .unwrap() + .2, + vec![11, 21] + ); + }) + } + + #[test] + fn estimate_next_election_works() { + ExtBuilder::default().session_per_era(5).period(5).build().execute_with(|| { + // first session is always length 0. + for b in 1..20 { + run_to_block(b); + assert_eq!(Staking::next_election_prediction(System::block_number()), 20); + } + + // election + run_to_block(20); + assert_eq!(Staking::next_election_prediction(System::block_number()), 45); + assert_eq!(staking_events().len(), 1); + assert_eq!( + *staking_events().last().unwrap(), + RawEvent::StakingElection(ElectionCompute::OnChain) + ); + + for b in 21..45 { + run_to_block(b); + assert_eq!(Staking::next_election_prediction(System::block_number()), 45); + } + + // election + run_to_block(45); + assert_eq!(Staking::next_election_prediction(System::block_number()), 70); + assert_eq!(staking_events().len(), 3); + assert_eq!( + *staking_events().last().unwrap(), + RawEvent::StakingElection(ElectionCompute::OnChain) + ); + }) + } +} diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index 0b2d3bceea5ec..eabe93d2ae008 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -347,22 +347,23 @@ impl Happened for () { /// be the default value), or where the account is being removed or reset back to the default value /// where previously it did exist (though may have been in a default state). This works well with /// system module's `CallOnCreatedAccount` and `CallKillAccount`. -pub struct StorageMapShim< - S, - Created, - Removed, - K, - T ->(sp_std::marker::PhantomData<(S, Created, Removed, K, T)>); +pub struct StorageMapShim( + sp_std::marker::PhantomData<(S, Created, Removed, K, T)>, +); impl< - S: StorageMap, - Created: Happened, - Removed: Happened, - K: FullCodec, - T: FullCodec, -> StoredMap for StorageMapShim { - fn get(k: &K) -> T { S::get(k) } - fn is_explicit(k: &K) -> bool { S::contains_key(k) } + S: StorageMap, + Created: Happened, + Removed: Happened, + K: FullCodec, + T: FullCodec, + > StoredMap for StorageMapShim +{ + fn get(k: &K) -> T { + S::get(k) + } + fn is_explicit(k: &K) -> bool { + S::contains_key(k) + } fn insert(k: &K, t: T) { let existed = S::contains_key(&k); S::insert(k, t); @@ -413,10 +414,16 @@ impl< } } -/// Something that can estimate at which block the next session rotation will happen. This should -/// be the same logical unit that dictates `ShouldEndSession` to the session module. No Assumptions -/// are made about the scheduling of the sessions. +/// Something that can estimate at which block the next session rotation will happen. +/// +/// This should be the same logical unit that dictates `ShouldEndSession` to the session module. No +/// Assumptions are made about the scheduling of the sessions. pub trait EstimateNextSessionRotation { + /// Return the average length of a session. + /// + /// This may or may not be accurate. + fn average_session_length() -> BlockNumber; + /// Return the block number at which the next session rotation is estimated to happen. /// /// None should be returned if the estimation fails to come to an answer @@ -426,7 +433,11 @@ pub trait EstimateNextSessionRotation { fn weight(now: BlockNumber) -> Weight; } -impl EstimateNextSessionRotation for () { +impl EstimateNextSessionRotation for () { + fn average_session_length() -> BlockNumber { + Default::default() + } + fn estimate_next_session_rotation(_: BlockNumber) -> Option { Default::default() } @@ -436,9 +447,15 @@ impl EstimateNextSessionRotation for () { } } -/// Something that can estimate at which block the next `new_session` will be triggered. This must -/// always be implemented by the session module. +/// Something that can estimate at which block the next `new_session` will be triggered. +/// +/// This must always be implemented by the session module. pub trait EstimateNextNewSession { + /// Return the average length of a session. + /// + /// This may or may not be accurate. + fn average_session_length() -> BlockNumber; + /// Return the block number at which the next new session is estimated to happen. fn estimate_next_new_session(now: BlockNumber) -> Option; @@ -446,7 +463,11 @@ pub trait EstimateNextNewSession { fn weight(now: BlockNumber) -> Weight; } -impl EstimateNextNewSession for () { +impl EstimateNextNewSession for () { + fn average_session_length() -> BlockNumber { + Default::default() + } + fn estimate_next_new_session(_: BlockNumber) -> Option { Default::default() } From 8607d3dac236faac5d93ae49203046dbc1fa3bc3 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 15 Jan 2021 15:54:54 +0000 Subject: [PATCH 03/86] add signed phase. --- bin/node/runtime/src/lib.rs | 16 + .../src/benchmarking.rs | 76 ++ .../election-provider-multi-phase/src/lib.rs | 122 ++- .../election-provider-multi-phase/src/mock.rs | 61 +- .../src/signed.rs | 717 ++++++++++++++++++ .../src/unsigned.rs | 100 +-- .../src/weights.rs | 37 + 7 files changed, 1053 insertions(+), 76 deletions(-) create mode 100644 frame/election-provider-multi-phase/src/signed.rs diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index bc3526af195f7..603d87b1f977f 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -508,6 +508,12 @@ parameter_types! { pub const SignedPhase: u32 = 100; pub const UnsignedPhase: u32 = 100; + // signed config + pub const SignedMaxSubmissions: u32 = 10; + pub const SignedRewardBase: Balance = 1 * DOLLARS; + pub const SignedDepositBase: Balance = 1 * DOLLARS; + pub const SignedDepositByte: Balance = 1 * CENTS; + // fallback: no need to do on-chain phragmen initially. pub const Fallback: FallbackStrategy = FallbackStrategy::Nothing; @@ -531,6 +537,16 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type MinerMaxIterations = MinerMaxIterations; type MinerMaxWeight = MinerMaxWeight; type UnsignedPriority = TwoPhaseUnsignedPriority; + type SignedMaxSubmissions = SignedMaxSubmissions; + type SignedRewardBase = SignedRewardBase; + type SignedRewardFactor = (); // no score-based reward + type SignedRewardMax = SignedRewardBase; + type SignedDepositBase = SignedDepositBase; + type SignedDepositByte = SignedDepositByte; + type SignedDepositWeight = (); + type SignedMaxWeight = MinerMaxWeight; + type SlashHandler = (); // burn slashes + type RewardHandler = (); // nothing todo upon rewards/ type DataProvider = Staking; type OnChainAccuracy = Perbill; type CompactSolution = pallet_staking::CompactAssignments; diff --git a/frame/election-provider-multi-phase/src/benchmarking.rs b/frame/election-provider-multi-phase/src/benchmarking.rs index a7a4eed852850..ccd37f447671d 100644 --- a/frame/election-provider-multi-phase/src/benchmarking.rs +++ b/frame/election-provider-multi-phase/src/benchmarking.rs @@ -175,6 +175,39 @@ benchmarks! { assert!(>::current_phase().is_signed()); } + finalize_signed_phase_accept_solution { + let receiver = account("receiver", 0, SEED); + let initial_balance = T::Currency::minimum_balance() * 10u32.into(); + T::Currency::make_free_balance_be(&receiver, initial_balance); + let ready: ReadySolution = Default::default(); + let deposit: BalanceOf = 10u32.into(); + let reward: BalanceOf = 20u32.into(); + + assert_ok!(T::Currency::reserve(&receiver, deposit)); + assert_eq!(T::Currency::free_balance(&receiver), initial_balance - 10u32.into()); + }: { + >::finalize_signed_phase_accept_solution(ready, &receiver, deposit, reward) + } verify { + assert_eq!(T::Currency::free_balance(&receiver), initial_balance + 20u32.into()); + assert_eq!(T::Currency::reserved_balance(&receiver), 0u32.into()); + } + + finalize_signed_phase_reject_solution { + let receiver = account("receiver", 0, SEED); + let initial_balance = T::Currency::minimum_balance().max(One::one()) * 10u32.into(); + let deposit: BalanceOf = 10u32.into(); + T::Currency::make_free_balance_be(&receiver, initial_balance); + assert_ok!(T::Currency::reserve(&receiver, deposit)); + + assert_eq!(T::Currency::free_balance(&receiver), initial_balance - 10u32.into()); + assert_eq!(T::Currency::reserved_balance(&receiver), 10u32.into()); + }: { + >::finalize_signed_phase_reject_solution(&receiver, deposit) + } verify { + assert_eq!(T::Currency::free_balance(&receiver), initial_balance - 10u32.into()); + assert_eq!(T::Currency::reserved_balance(&receiver), 0u32.into()); + } + on_initialize_open_unsigned_with_snapshot { assert!(>::snapshot().is_none()); assert!(>::current_phase().is_off()); @@ -206,6 +239,33 @@ benchmarks! { assert!(>::snapshot().is_some()); } + submit { + let c in 1 .. (T::SignedMaxSubmissions::get() - 1); + + // the solution will be worse than all of them meaning the score need to be checked against all. + let solution = RawSolution { score: [(10_000_000u128 - 1).into(), 0, 0], ..Default::default() }; + + >::put(Phase::Signed); + >::put(1); + + for i in 0..c { + >::mutate(|queue| { + let solution = RawSolution { score: [(10_000_000 + i).into(), 0, 0], ..Default::default() }; + let signed_submission = SignedSubmission { solution, ..Default::default() }; + // note: this is quite tricky: we know that the queue will stay sorted here. The + // last will be best. + queue.push(signed_submission); + }) + } + + let caller = frame_benchmarking::whitelisted_caller(); + T::Currency::make_free_balance_be(&caller, T::Currency::minimum_balance() * 10u32.into()); + + }: _(RawOrigin::Signed(caller), solution, c) + verify { + assert!(>::signed_submissions().len() as u32 == c + 1); + } + submit_unsigned { // number of votes in snapshot. let v in (T::BenchmarkingConfig::VOTERS[0]) .. T::BenchmarkingConfig::VOTERS[1]; @@ -260,6 +320,22 @@ mod test { assert_ok!(test_benchmark_feasibility_check::()); }); + ExtBuilder::default().build_and_execute(|| { + assert_ok!(test_benchmark_finalize_signed_phase_accept_solution::()); + }); + + ExtBuilder::default().build_and_execute(|| { + assert_ok!(test_benchmark_finalize_signed_phase_reject_solution::()); + }); + + ExtBuilder::default().build_and_execute(|| { + assert_ok!(test_benchmark_on_initialize_open_signed::()); + }); + + ExtBuilder::default().build_and_execute(|| { + assert_ok!(test_benchmark_submit::()); + }); + ExtBuilder::default().build_and_execute(|| { assert_ok!(test_benchmark_submit_unsigned::()); }); diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 79112297f869f..664030315dc73 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -48,7 +48,7 @@ //! In the signed phase, solutions (of type [`RawSolution`]) are submitted and queued on chain. A //! deposit is reserved, based on the size of the solution, for the cost of keeping this solution //! on-chain for a number of blocks, and the potential weight of the solution upon being checked. A -//! maximum of [`pallet::Config::MaxSignedSubmissions`] solutions are stored. The queue is always +//! maximum of [`pallet::Config::SignedMaxSubmissions`] solutions are stored. The queue is always //! sorted based on score (worse to best). //! //! Upon arrival of a new solution: @@ -174,7 +174,7 @@ use codec::{Decode, Encode, HasCompact}; use frame_support::{ dispatch::DispatchResultWithPostInfo, ensure, - traits::{Currency, Get, ReservableCurrency}, + traits::{Currency, Get, ReservableCurrency, OnUnbalanced}, weights::Weight, }; use frame_system::{ensure_none, ensure_signed, offchain::SendTransactionTypes}; @@ -216,8 +216,8 @@ pub mod weights; use weights::WeightInfo; -// pub mod signed; -// use signed::SignedSubmission; +pub mod signed; +use signed::{SignedSubmission, BalanceOf, NegativeImbalanceOf, PositiveImbalanceOf}; /// The compact solution type used by this crate. pub type CompactOf = ::CompactSolution; @@ -535,6 +535,40 @@ pub mod pallet { /// this values, based on [`WeightInfo::submit_unsigned`]. type MinerMaxWeight: Get; + /// Maximum number of singed submissions that can be queued. + #[pallet::constant] + type SignedMaxSubmissions: Get; + /// Maximum weight of a signed solution. + /// + /// This should probably be similar to [`Config::MinerMaxWeight`]. + #[pallet::constant] + type SignedMaxWeight: Get; + + /// Base reward for a signed solution + #[pallet::constant] + type SignedRewardBase: Get>; + /// Per-score reward for a signed solution. + #[pallet::constant] + type SignedRewardFactor: Get; + /// Maximum cap for a signed solution. + #[pallet::constant] + type SignedRewardMax: Get>>; + + /// Base deposit for a signed solution. + #[pallet::constant] + type SignedDepositBase: Get>; + /// Per-byte deposit for a signed solution. + #[pallet::constant] + type SignedDepositByte: Get>; + /// Per-weight deposit for a signed solution. + #[pallet::constant] + type SignedDepositWeight: Get>; + + /// Handler for the slashed deposits. + type SlashHandler: OnUnbalanced>; + /// Handler for the rewards. + type RewardHandler: OnUnbalanced>; + /// Something that will provide the election data. type DataProvider: ElectionDataProvider; @@ -654,6 +688,77 @@ pub mod pallet { ExtendedBalance: From>>, ExtendedBalance: From>>, { + /// Submit a solution for the signed phase. + /// + /// The dispatch origin fo this call must be __signed__. + /// + /// The solution is potentially queued, based on the claimed score and processed at the end + /// of the signed phase. + /// + /// A deposit is reserved and recorded for the solution. Based on the outcome, the solution + /// might be rewarded, slashed, or get all or a part of the deposit back. + /// + /// # + /// Queue size must be provided as witness data. + /// # + #[pallet::weight(T::WeightInfo::submit(*witness_data))] + pub fn submit( + origin: OriginFor, + solution: RawSolution>, + witness_data: u32, + ) -> DispatchResultWithPostInfo { + let who = ensure_signed(origin)?; + + // ensure witness data is correct. + ensure!( + witness_data >= >::decode_len().unwrap_or_default() as u32, + Error::::InvalidWitness, + ); + + // ensure solution is timely. + ensure!(Self::current_phase().is_signed(), Error::::EarlySubmission,); + + // NOTE: this is the only case where having separate snapshot would have been better + // because could do just decode_len. But we can create abstractions to do this. + + // build size. Note: this is not needed for weight calc, thus not input. + // defensive-only: if phase is signed, snapshot will exist. + let size = Self::build_solution_size().unwrap_or_default(); + + // NOTE: we compute this function once in `insert_submission` as well, could optimize. + ensure!( + Self::feasibility_weight_of(&solution, size) < T::SignedMaxWeight::get(), + Error::::TooMuchWeight, + ); + + // ensure solution claims is better. + let mut signed_submissions = Self::signed_submissions(); + let index = Self::insert_submission(&who, &mut signed_submissions, solution, size) + .ok_or(Error::::QueueFull)?; + + // collect deposit. Thereafter, the function cannot fail. + // Defensive -- index is valid. + let deposit = signed_submissions.get(index).map(|s| s.deposit).unwrap_or_default(); + T::Currency::reserve(&who, deposit).map_err(|_| Error::::CannotPayDeposit)?; + + // Remove the weakest, if needed. + if signed_submissions.len() as u32 > T::SignedMaxSubmissions::get() { + Self::remove_weakest(&mut signed_submissions); + } + debug_assert!(signed_submissions.len() as u32 <= T::SignedMaxSubmissions::get()); + + log!( + info, + "queued signed solution with (claimed) score {:?}", + signed_submissions.get(index).map(|s| s.solution.score).unwrap_or_default() + ); + + // store the new signed submission. + >::put(signed_submissions); + Self::deposit_event(Event::SolutionStored(ElectionCompute::Signed)); + Ok(None.into()) + } + /// Submit a solution for the unsigned phase. /// /// The dispatch origin fo this call must be __none__. @@ -854,6 +959,15 @@ pub mod pallet { #[pallet::getter(fn snapshot_metadata)] pub type SnapshotMetadata = StorageValue<_, RoundSnapshotMetadata>; + /// Sorted (worse -> best) list of unchecked, signed solutions. + #[pallet::storage] + #[pallet::getter(fn signed_submissions)] + pub type SignedSubmissions = StorageValue< + _, + Vec, CompactOf>>, + ValueQuery, + >; + #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] pub struct Pallet(PhantomData); diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index 71102f3e6e526..fc899aebc436d 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -201,8 +201,14 @@ parameter_types! { pub static DesiredTargets: u32 = 2; pub static SignedPhase: u64 = 10; pub static UnsignedPhase: u64 = 5; - pub static MaxSignedSubmissions: u32 = 5; - + pub static SignedMaxSubmissions: u32 = 5; + pub static SignedDepositBase: Balance = 5; + pub static SignedDepositByte: Balance = 0; + pub static SignedDepositWeight: Balance = 0; + pub static SignedRewardBase: Balance = 7; + pub static SignedRewardFactor: Perbill = Perbill::zero(); + pub static SignedRewardMax: Balance = 10; + pub static SignedMaxWeight: Weight = BlockWeights::get().max_block; pub static MinerMaxIterations: u32 = 5; pub static UnsignedPriority: u64 = 100; pub static SolutionImprovementThreshold: Perbill = Perbill::zero(); @@ -244,6 +250,27 @@ impl two_phase::weights::WeightInfo for DualMockWeightInfo { <() as two_phase::weights::WeightInfo>::on_initialize_open_unsigned_without_snapshot() } } + fn finalize_signed_phase_accept_solution() -> Weight { + if MockWeightInfo::get() { + Zero::zero() + } else { + <() as two_phase::weights::WeightInfo>::finalize_signed_phase_accept_solution() + } + } + fn finalize_signed_phase_reject_solution() -> Weight { + if MockWeightInfo::get() { + Zero::zero() + } else { + <() as two_phase::weights::WeightInfo>::finalize_signed_phase_reject_solution() + } + } + fn submit(c: u32) -> Weight { + if MockWeightInfo::get() { + Zero::zero() + } else { + <() as two_phase::weights::WeightInfo>::submit(c) + } + } fn submit_unsigned(v: u32, t: u32, a: u32, d: u32) -> Weight { if MockWeightInfo::get() { // 10 base @@ -273,6 +300,16 @@ impl crate::Config for Runtime { type MinerMaxIterations = MinerMaxIterations; type MinerMaxWeight = MinerMaxWeight; type UnsignedPriority = UnsignedPriority; + type SignedRewardBase = SignedRewardBase; + type SignedRewardFactor = SignedRewardFactor; + type SignedRewardMax = SignedRewardMax; + type SignedDepositBase = SignedDepositBase; + type SignedDepositByte = (); + type SignedDepositWeight = (); + type SignedMaxWeight = SignedMaxWeight; + type SignedMaxSubmissions = SignedMaxSubmissions; + type SlashHandler = (); + type RewardHandler = (); type DataProvider = StakingMock; type WeightInfo = DualMockWeightInfo; type BenchmarkingConfig = (); @@ -344,6 +381,26 @@ impl ExtBuilder { VOTERS.with(|v| v.borrow_mut().push((who, stake, targets))); self } + pub fn signed_max_submission(self, count: u32) -> Self { + ::set(count); + self + } + pub fn signed_deposit(self, base: u64, byte: u64, weight: u64) -> Self { + ::set(base); + ::set(byte); + ::set(weight); + self + } + pub fn reward(self, base: u64, factor: Perbill, max: u64) -> Self { + ::set(base); + ::set(factor); + ::set(max); + self + } + pub fn signed_weight(self, weight: Weight) -> Self { + ::set(weight); + self + } pub fn build(self) -> sp_io::TestExternalities { sp_tracing::try_init_simple(); let mut storage = diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs new file mode 100644 index 0000000000000..593a0da26573a --- /dev/null +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -0,0 +1,717 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The signed phase implementation. + +use super::*; +use codec::Encode; +use sp_arithmetic::traits::SaturatedConversion; +use sp_npos_elections::{is_score_better, CompactSolution}; +use sp_runtime::Perbill; + +/// A raw, unchecked signed submission. +/// +/// This is just a wrapper around [`RawSolution`] and some additional info. +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, Default)] +pub struct SignedSubmission { + /// Who submitted this solution. + pub(crate) who: A, + /// The deposit reserved for storing this solution. + pub(crate) deposit: B, + /// The reward that should be given to this solution, if chosen the as the final one. + pub(crate) reward: B, + /// The raw solution itself. + pub(crate) solution: RawSolution, +} + +pub(crate) type BalanceOf = + <::Currency as Currency<::AccountId>>::Balance; +pub(crate) type PositiveImbalanceOf = <::Currency as Currency< + ::AccountId, +>>::PositiveImbalance; +pub(crate) type NegativeImbalanceOf = <::Currency as Currency< + ::AccountId, +>>::NegativeImbalance; + +/* + + + -------------------------------------------- NODE + + // signed configs + pub const SignedMaxSubmissions: u32 = 10; + pub const SignedRewardBase: Balance = 1 * DOLLARS; + pub const SignedDepositBase: Balance = 1 * DOLLARS; + pub const SignedDepositByte: Balance = 1 * CENTS; +*/ + +impl Pallet +where + ExtendedBalance: From>>, + ExtendedBalance: From>>, +{ + /// Finish the singed phase. Process the signed submissions from best to worse until a valid one + /// is found, rewarding the best one and slashing the invalid ones along the way. + /// + /// Returns true if we have a good solution in the signed phase. + /// + /// This drains the [`SignedSubmissions`], potentially storing the best valid one in + /// [`QueuedSolution`]. + pub fn finalize_signed_phase() -> (bool, Weight) { + let mut all_submission: Vec> = >::take(); + let mut found_solution = false; + let mut weight = T::DbWeight::get().reads(1); + + while let Some(best) = all_submission.pop() { + let SignedSubmission { solution, who, deposit, reward } = best; + let active_voters = solution.compact.voter_count() as u32; + let feasibility_weight = { + // defensive only: at the end of signed phase, snapshot will exits. + let RoundSnapshotMetadata { voters_len, targets_len } = + Self::snapshot_metadata().unwrap_or_default(); + let desired_targets = Self::desired_targets().unwrap_or_default(); + let v = voters_len as u32; + let t = targets_len as u32; + let a = active_voters; + let w = desired_targets; + T::WeightInfo::feasibility_check(v, t, a, w) + }; + match Self::feasibility_check(solution, ElectionCompute::Signed) { + Ok(ready_solution) => { + Self::finalize_signed_phase_accept_solution( + ready_solution, + &who, + deposit, + reward, + ); + found_solution = true; + + weight = weight.saturating_add(feasibility_weight); + weight = weight + .saturating_add(T::WeightInfo::finalize_signed_phase_accept_solution()); + break; + } + Err(_) => { + // we assume a worse case feasibility check happened anyhow. + weight = weight.saturating_add(feasibility_weight); + Self::finalize_signed_phase_reject_solution(&who, deposit); + weight = weight + .saturating_add(T::WeightInfo::finalize_signed_phase_reject_solution()); + } + } + } + + // Any unprocessed solution is not pointless to even ponder upon. Feasible or malicious, + // they didn't end up being used. Unreserve the bonds. + all_submission.into_iter().for_each(|not_processed| { + let SignedSubmission { who, deposit, .. } = not_processed; + let _remaining = T::Currency::unreserve(&who, deposit); + weight = weight.saturating_add(T::DbWeight::get().writes(1)); + debug_assert!(_remaining.is_zero()); + }); + + (found_solution, weight) + } + + /// Helper function for the case where a solution is accepted in the signed phase. + /// + /// Extracted to facilitate with weight calculation. + /// + /// Infallible + pub fn finalize_signed_phase_accept_solution( + ready_solution: ReadySolution, + who: &T::AccountId, + deposit: BalanceOf, + reward: BalanceOf, + ) { + // write this ready solution. + >::put(ready_solution); + + // unreserve deposit. + let _remaining = T::Currency::unreserve(who, deposit); + debug_assert!(_remaining.is_zero()); + + // Reward. + let positive_imbalance = T::Currency::deposit_creating(who, reward); + T::RewardHandler::on_unbalanced(positive_imbalance); + } + + /// Helper function for the case where a solution is accepted in the rejected phase. + /// + /// Extracted to facilitate with weight calculation. + /// + /// Infallible + pub fn finalize_signed_phase_reject_solution(who: &T::AccountId, deposit: BalanceOf) { + let (negative_imbalance, _remaining) = T::Currency::slash_reserved(who, deposit); + debug_assert!(_remaining.is_zero()); + T::SlashHandler::on_unbalanced(negative_imbalance); + } + + /// Find a proper position in the queue for the signed queue, whilst maintaining the order of + /// solution quality. If insertion was successful, `Some(index)` is returned where index is the + /// index of the newly inserted item. + /// + /// Invariant: The returned index is always a valid index in `queue` and can safely be used to + /// inspect the newly inserted element. + pub fn insert_submission( + who: &T::AccountId, + queue: &mut Vec, CompactOf>>, + solution: RawSolution>, + size: SolutionSize, + ) -> Option { + // from the last score, compare and see if the current one is better. If none, then the + // awarded index is 0. + let outcome = queue + .iter() + .enumerate() + .rev() + .find_map(|(i, s)| { + if is_score_better::( + solution.score, + s.solution.score, + T::SolutionImprovementThreshold::get(), + ) { + Some(i + 1) + } else { + None + } + }) + .or(Some(0)) + .and_then(|at| { + if at == 0 && queue.len() as u32 >= T::SignedMaxSubmissions::get() { + // if this is worse than all, and the queue is full, don't bother. + None + } else { + // add to the designated spot. If the length is too much, remove one. + let reward = Self::reward_for(&solution); + let deposit = Self::deposit_for(&solution, size); + let submission = + SignedSubmission { who: who.clone(), deposit, reward, solution }; + // Proof: `at` must always less than or equal queue.len() for this not to panic. + // It is either 0 (in which case `0 <= queue.len()`) or one of the queue indices + // + 1. The biggest queue index is `queue.len() - 1`, thus `at <= queue.len()`. + queue.insert(at, submission); + Some(at) + } + }); + + // If the call site is sane and removes the weakest, then this must always be correct. + debug_assert!(queue.len() as u32 <= T::SignedMaxSubmissions::get() + 1); + outcome + } + + /// Removes the weakest element of the queue, namely the first one, should the length of the + /// queue be enough. + /// + /// noop if the queue is empty. Bond of the removed solution is returned. + pub fn remove_weakest( + queue: &mut Vec, CompactOf>>, + ) { + if queue.len() > 0 { + let SignedSubmission { who, deposit, .. } = queue.remove(0); + let _remainder = T::Currency::unreserve(&who, deposit); + debug_assert!(_remainder.is_zero()); + } + } + + /// The feasibility weight of the given raw solution. + pub fn feasibility_weight_of( + solution: &RawSolution>, + size: SolutionSize, + ) -> Weight { + T::WeightInfo::feasibility_check( + size.voters, + size.targets, + solution.compact.voter_count() as u32, + solution.compact.unique_targets().len() as u32, + ) + } + + /// Collect sufficient deposit to store this solution this chain. + /// + /// The deposit is composed of 3 main elements: + /// + /// 1. base deposit, fixed for all submissions. + /// 2. a per-byte deposit, for renting the state usage. + /// 3. a per-weight deposit, for the potential weight usage in an upcoming on_initialize + pub fn deposit_for(solution: &RawSolution>, size: SolutionSize) -> BalanceOf { + let encoded_len: BalanceOf = solution.using_encoded(|e| e.len() as u32).into(); + let feasibility_weight = Self::feasibility_weight_of(solution, size); + + let len_deposit = T::SignedDepositByte::get() * encoded_len; + let weight_deposit = T::SignedDepositWeight::get() * feasibility_weight.saturated_into(); + + T::SignedDepositBase::get() + len_deposit + weight_deposit + } + + /// The reward for this solution, if successfully chosen as the best one at the end of the + /// signed phase. + pub fn reward_for(solution: &RawSolution>) -> BalanceOf { + let raw_reward = T::SignedRewardBase::get() + + T::SignedRewardFactor::get() * solution.score[0].saturated_into::>(); + + match T::SignedRewardMax::get() { + Some(cap) => raw_reward.min(cap), + None => raw_reward, + } + } + + /// Build the solution size from the snapshot metadata, if it exists. Else, returns `None`. + pub(crate) fn build_solution_size() -> Option { + let metadata = Self::snapshot_metadata()?; + Some(SolutionSize { + voters: metadata.voters_len as u32, + targets: metadata.targets_len as u32, + }) + } +} + +#[cfg(test)] +mod tests { + use super::{ + mock::{Origin, *}, + Error, Phase, *, + }; + use frame_support::dispatch::DispatchResultWithPostInfo; + + fn submit_with_witness( + origin: Origin, + solution: RawSolution>, + ) -> DispatchResultWithPostInfo { + TwoPhase::submit(origin, solution, TwoPhase::signed_submissions().len() as u32) + } + + #[test] + fn cannot_submit_too_early() { + ExtBuilder::default().build_and_execute(|| { + roll_to(2); + assert_eq!(TwoPhase::current_phase(), Phase::Off); + + // create a temp snapshot only for this test. + TwoPhase::create_snapshot(); + let solution = raw_solution(); + + assert_noop!( + submit_with_witness(Origin::signed(10), solution), + Error::::EarlySubmission, + ); + }) + } + + #[test] + fn wrong_witness_fails() { + ExtBuilder::default().build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + let solution = raw_solution(); + // submit this once correctly + assert_ok!(submit_with_witness(Origin::signed(99), solution.clone())); + assert_eq!(TwoPhase::signed_submissions().len(), 1); + + // now try and cheat by passing a lower queue length + assert_noop!( + TwoPhase::submit(Origin::signed(99), solution, 0,), + Error::::InvalidWitness, + ); + }) + } + + #[test] + fn should_pay_deposit() { + ExtBuilder::default().build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + let solution = raw_solution(); + assert_eq!(balances(&99), (100, 0)); + + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + + assert_eq!(balances(&99), (95, 5)); + assert_eq!(TwoPhase::signed_submissions().first().unwrap().deposit, 5); + }) + } + + #[test] + fn good_solution_is_rewarded() { + ExtBuilder::default().build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + let solution = raw_solution(); + assert_eq!(balances(&99), (100, 0)); + + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_eq!(balances(&99), (95, 5)); + + assert!(TwoPhase::finalize_signed_phase().0); + assert_eq!(balances(&99), (100 + 7, 0)); + }) + } + + #[test] + fn reward_is_capped() { + ExtBuilder::default().reward(5, Perbill::from_percent(25), 10).build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + let solution = raw_solution(); + assert_eq!(solution.score[0], 40); + assert_eq!(balances(&99), (100, 0)); + + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_eq!(balances(&99), (95, 5)); + + assert!(TwoPhase::finalize_signed_phase().0); + // expected reward is 5 + 10 + assert_eq!(balances(&99), (100 + 10, 0)); + }); + + ExtBuilder::default().reward(5, Perbill::from_percent(25), 20).build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + let solution = raw_solution(); + assert_eq!(solution.score[0], 40); + assert_eq!(balances(&99), (100, 0)); + + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_eq!(balances(&99), (95, 5)); + + assert!(TwoPhase::finalize_signed_phase().0); + // expected reward is 5 + 10 + assert_eq!(balances(&99), (100 + 15, 0)); + }); + } + + #[test] + fn bad_solution_is_slashed() { + ExtBuilder::default().build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + let mut solution = raw_solution(); + assert_eq!(balances(&99), (100, 0)); + + // make the solution invalid. + solution.score[0] += 1; + + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_eq!(balances(&99), (95, 5)); + + // no good solution was stored. + assert!(!TwoPhase::finalize_signed_phase().0); + // and the bond is gone. + assert_eq!(balances(&99), (95, 0)); + }) + } + + #[test] + fn suppressed_solution_gets_bond_back() { + ExtBuilder::default().build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + let mut solution = raw_solution(); + assert_eq!(balances(&99), (100, 0)); + assert_eq!(balances(&999), (100, 0)); + + // submit as correct. + assert_ok!(submit_with_witness(Origin::signed(99), solution.clone())); + + // make the solution invalid and weaker. + solution.score[0] -= 1; + assert_ok!(submit_with_witness(Origin::signed(999), solution)); + assert_eq!(balances(&99), (95, 5)); + assert_eq!(balances(&999), (95, 5)); + + // _some_ good solution was stored. + assert!(TwoPhase::finalize_signed_phase().0); + + // 99 is rewarded. + assert_eq!(balances(&99), (100 + 7, 0)); + // 999 gets everything back. + assert_eq!(balances(&999), (100, 0)); + }) + } + + #[test] + fn cannot_submit_worse_with_full_queue() { + ExtBuilder::default().build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + for s in 0..SignedMaxSubmissions::get() { + // score is always getting better + let solution = RawSolution { score: [(5 + s).into(), 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + } + + // weaker. + let solution = RawSolution { score: [4, 0, 0], ..Default::default() }; + + assert_noop!( + submit_with_witness(Origin::signed(99), solution), + Error::::QueueFull, + ); + }) + } + + #[test] + fn weakest_is_removed_if_better_provided() { + ExtBuilder::default().build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + for s in 0..SignedMaxSubmissions::get() { + // score is always getting better + let solution = RawSolution { score: [(5 + s).into(), 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + } + + assert_eq!( + TwoPhase::signed_submissions() + .into_iter() + .map(|s| s.solution.score[0]) + .collect::>(), + vec![5, 6, 7, 8, 9] + ); + + // better. + let solution = RawSolution { score: [20, 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + + // the one with score 5 was rejected, the new one inserted. + assert_eq!( + TwoPhase::signed_submissions() + .into_iter() + .map(|s| s.solution.score[0]) + .collect::>(), + vec![6, 7, 8, 9, 20] + ); + }) + } + + #[test] + fn weakest_is_removed_if_better_provided_wont_remove_self() { + ExtBuilder::default().build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + for s in 1..SignedMaxSubmissions::get() { + // score is always getting better + let solution = RawSolution { score: [(5 + s).into(), 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + } + + let solution = RawSolution { score: [4, 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + + assert_eq!( + TwoPhase::signed_submissions() + .into_iter() + .map(|s| s.solution.score[0]) + .collect::>(), + vec![4, 6, 7, 8, 9] + ); + + // better. + let solution = RawSolution { score: [5, 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + + // the one with score 5 was rejected, the new one inserted. + assert_eq!( + TwoPhase::signed_submissions() + .into_iter() + .map(|s| s.solution.score[0]) + .collect::>(), + vec![5, 6, 7, 8, 9] + ); + }) + } + + #[test] + fn early_ejected_solution_gets_bond_back() { + ExtBuilder::default().signed_deposit(2, 0, 0).build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + for s in 0..SignedMaxSubmissions::get() { + // score is always getting better + let solution = RawSolution { score: [(5 + s).into(), 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + } + + assert_eq!(balances(&99).1, 2 * 5); + assert_eq!(balances(&999).1, 0); + + // better. + let solution = RawSolution { score: [20, 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(999), solution)); + + // got one bond back. + assert_eq!(balances(&99).1, 2 * 4); + assert_eq!(balances(&999).1, 2); + }) + } + + #[test] + fn equally_good_solution_is_not_accepted() { + ExtBuilder::default().signed_max_submission(3).build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + for i in 0..SignedMaxSubmissions::get() { + let solution = RawSolution { score: [(5 + i).into(), 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + } + assert_eq!( + TwoPhase::signed_submissions() + .into_iter() + .map(|s| s.solution.score[0]) + .collect::>(), + vec![5, 6, 7] + ); + + // 5 is not accepted. This will only cause processing with no benefit. + let solution = RawSolution { score: [5, 0, 0], ..Default::default() }; + assert_noop!( + submit_with_witness(Origin::signed(99), solution), + Error::::QueueFull, + ); + }) + } + + #[test] + fn solutions_are_always_sorted() { + ExtBuilder::default().signed_max_submission(3).build_and_execute(|| { + let scores = || { + TwoPhase::signed_submissions() + .into_iter() + .map(|s| s.solution.score[0]) + .collect::>() + }; + + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + let solution = RawSolution { score: [5, 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_eq!(scores(), vec![5]); + + let solution = RawSolution { score: [8, 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_eq!(scores(), vec![5, 8]); + + let solution = RawSolution { score: [3, 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_eq!(scores(), vec![3, 5, 8]); + + let solution = RawSolution { score: [6, 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_eq!(scores(), vec![5, 6, 8]); + + let solution = RawSolution { score: [6, 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_eq!(scores(), vec![6, 6, 8]); + + let solution = RawSolution { score: [10, 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_eq!(scores(), vec![6, 8, 10]); + + let solution = RawSolution { score: [12, 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_eq!(scores(), vec![8, 10, 12]); + }) + } + + #[test] + fn all_in_one_singed_submission_scenario() { + // a combination of: + // - good_solution_is_rewarded + // - bad_solution_is_slashed + // - suppressed_solution_gets_bond_back + ExtBuilder::default().build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + assert_eq!(balances(&99), (100, 0)); + assert_eq!(balances(&999), (100, 0)); + assert_eq!(balances(&9999), (100, 0)); + let mut solution = raw_solution(); + + // submit a correct one. + assert_ok!(submit_with_witness(Origin::signed(99), solution.clone())); + + // make the solution invalidly better and submit. This ought to be slashed. + solution.score[0] += 1; + assert_ok!(submit_with_witness(Origin::signed(999), solution.clone())); + + // make the solution invalidly worse and submit. This ought to be suppressed and + // returned. + solution.score[0] -= 1; + assert_ok!(submit_with_witness(Origin::signed(9999), solution)); + + assert_eq!( + TwoPhase::signed_submissions().iter().map(|x| x.who).collect::>(), + vec![9999, 99, 999] + ); + + // _some_ good solution was stored. + assert!(TwoPhase::finalize_signed_phase().0); + + // 99 is rewarded. + assert_eq!(balances(&99), (100 + 7, 0)); + // 999 is slashed. + assert_eq!(balances(&999), (95, 0)); + // 9999 gets everything back. + assert_eq!(balances(&9999), (100, 0)); + }) + } + + #[test] + fn cannot_consume_too_much_future_weight() { + ExtBuilder::default().signed_weight(40).mock_weight_info(true).build_and_execute(|| { + roll_to(15); + assert!(TwoPhase::current_phase().is_signed()); + + let (solution, witness) = TwoPhase::mine_solution(2).unwrap(); + let solution_weight = ::WeightInfo::feasibility_check( + witness.voters, + witness.targets, + solution.compact.voter_count() as u32, + solution.compact.unique_targets().len() as u32, + ); + // default solution will have 5 edges (5 * 5 + 10) + assert_eq!(solution_weight, 35); + assert_eq!(solution.compact.voter_count(), 5); + assert_eq!(::SignedMaxWeight::get(), 40); + + assert_ok!(submit_with_witness(Origin::signed(99), solution.clone())); + + ::set(30); + + // note: resubmitting the same solution is technically okay as long as the queue has + // space. + assert_noop!( + submit_with_witness(Origin::signed(99), solution), + Error::::TooMuchWeight, + ); + }) + } +} diff --git a/frame/election-provider-multi-phase/src/unsigned.rs b/frame/election-provider-multi-phase/src/unsigned.rs index e922dc32dfa34..3cc0639455d15 100644 --- a/frame/election-provider-multi-phase/src/unsigned.rs +++ b/frame/election-provider-multi-phase/src/unsigned.rs @@ -254,7 +254,9 @@ where } debug_assert!( - weight_with(voters.min(size.voters)) <= max_weight, + weight_with(voters.min(size.voters)) <= max_weight || weight_with(0) > max_weight, + // --------------------------------------------------^^ Base weight is more than max, + // not much we can do here. "weight_with({}) <= {}", voters.min(size.voters), max_weight, @@ -343,84 +345,42 @@ where } #[cfg(test)] -mod max_weight { - #![allow(unused_variables)] +mod weight_trim { use super::{mock::*, *}; - struct TestWeight; - impl crate::weights::WeightInfo for TestWeight { - fn on_initialize_nothing() -> Weight { - unreachable!() - } - fn on_initialize_open_signed() -> Weight { - unreachable!() - } - fn on_initialize_open_unsigned_with_snapshot() -> Weight { - unreachable!() - } - fn on_initialize_open_unsigned_without_snapshot() -> Weight { - unreachable!() - } - fn submit_unsigned(v: u32, t: u32, a: u32, d: u32) -> Weight { - (0 * v + 0 * t + 1000 * a + 0 * d) as Weight - } - fn feasibility_check(v: u32, _t: u32, a: u32, d: u32) -> Weight { - unreachable!() - } - } - #[test] fn find_max_voter_binary_search_works() { - let w = SolutionSize { voters: 10, targets: 0 }; + ::set(true); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 0), 0); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1), 0); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 999), 0); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1000), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1001), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1990), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1999), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2000), 2); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2001), 2); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2010), 2); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2990), 2); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2999), 2); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 3000), 3); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 3333), 3); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 5500), 5); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 7777), 7); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 9999), 9); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 10_000), 10); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 10_999), 10); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 11_000), 10); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 22_000), 10); + let w = SolutionSize { voters: 10, targets: 0 }; + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 0), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 9), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 14), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 15), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 19), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 20), 2); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 25), 3); let w = SolutionSize { voters: 1, targets: 0 }; - - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 0), 0); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1), 0); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 999), 0); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1000), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1001), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1990), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1999), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2000), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2001), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2010), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 3333), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 0), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 9), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 14), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 15), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 19), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 20), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 25), 1); let w = SolutionSize { voters: 2, targets: 0 }; - - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 0), 0); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1), 0); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 999), 0); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1000), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1001), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1999), 1); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2000), 2); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2001), 2); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2010), 2); - assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 3333), 2); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 0), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 9), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 14), 0); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 15), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 19), 1); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 20), 2); + assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 25), 2); } } diff --git a/frame/election-provider-multi-phase/src/weights.rs b/frame/election-provider-multi-phase/src/weights.rs index 6070b771593ce..da549dbdc6ebe 100644 --- a/frame/election-provider-multi-phase/src/weights.rs +++ b/frame/election-provider-multi-phase/src/weights.rs @@ -50,6 +50,9 @@ pub trait WeightInfo { fn on_initialize_open_signed() -> Weight; fn on_initialize_open_unsigned_without_snapshot() -> Weight; fn on_initialize_open_unsigned_with_snapshot() -> Weight; + fn finalize_signed_phase_accept_solution() -> Weight; + fn finalize_signed_phase_reject_solution() -> Weight; + fn submit(c: u32) -> Weight; fn submit_unsigned(v: u32, t: u32, a: u32, d: u32) -> Weight; fn feasibility_check(v: u32, t: u32, a: u32, d: u32) -> Weight; } @@ -76,6 +79,23 @@ impl WeightInfo for SubstrateWeight { .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } + fn finalize_signed_phase_accept_solution() -> Weight { + (38_088_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + fn finalize_signed_phase_reject_solution() -> Weight { + (17_124_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn submit(c: u32) -> Weight { + (52_370_000 as Weight) + // Standard Error: 17_000 + .saturating_add((3_395_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32) -> Weight { (0 as Weight) // Standard Error: 21_000 @@ -121,6 +141,23 @@ impl WeightInfo for () { .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } + fn finalize_signed_phase_accept_solution() -> Weight { + (38_088_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + fn finalize_signed_phase_reject_solution() -> Weight { + (17_124_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn submit(c: u32) -> Weight { + (52_370_000 as Weight) + // Standard Error: 17_000 + .saturating_add((3_395_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32) -> Weight { (0 as Weight) // Standard Error: 21_000 From 6523f5bbb0ca55bd4d3e4be782d30aabb2e147d9 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 15 Jan 2021 16:08:52 +0000 Subject: [PATCH 04/86] remove comments --- frame/election-provider-multi-phase/src/signed.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 593a0da26573a..e7ec459a503e0 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -47,18 +47,6 @@ pub(crate) type NegativeImbalanceOf = <::Currency as Currency< ::AccountId, >>::NegativeImbalance; -/* - - - -------------------------------------------- NODE - - // signed configs - pub const SignedMaxSubmissions: u32 = 10; - pub const SignedRewardBase: Balance = 1 * DOLLARS; - pub const SignedDepositBase: Balance = 1 * DOLLARS; - pub const SignedDepositByte: Balance = 1 * CENTS; -*/ - impl Pallet where ExtendedBalance: From>>, From 632e107d46f9493831e3beb4c5036e702cd4e03a Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 15 Jan 2021 20:57:38 +0000 Subject: [PATCH 05/86] Undo bad formattings. --- frame/staking/src/mock.rs | 26 +++++++++++++------------- frame/support/src/traits.rs | 31 +++++++++++++++---------------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index d01f8b59a681f..46f1c16c971e0 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -997,19 +997,19 @@ pub(crate) fn make_all_reward_payment(era: EraIndex) { macro_rules! assert_session_era { ($session:expr, $era:expr) => { assert_eq!( - Session::current_index(), - $session, - "wrong session {} != {}", - Session::current_index(), - $session, - ); - assert_eq!( - Staking::current_era().unwrap(), - $era, - "wrong current era {} != {}", - Staking::current_era().unwrap(), - $era, - ); + Session::current_index(), + $session, + "wrong session {} != {}", + Session::current_index(), + $session, + ); + assert_eq!( + Staking::current_era().unwrap(), + $era, + "wrong current era {} != {}", + Staking::current_era().unwrap(), + $era, + ); }; } diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index eabe93d2ae008..d4dbdbf5ef115 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -347,23 +347,22 @@ impl Happened for () { /// be the default value), or where the account is being removed or reset back to the default value /// where previously it did exist (though may have been in a default state). This works well with /// system module's `CallOnCreatedAccount` and `CallKillAccount`. -pub struct StorageMapShim( - sp_std::marker::PhantomData<(S, Created, Removed, K, T)>, -); +pub struct StorageMapShim< + S, + Created, + Removed, + K, + T +>(sp_std::marker::PhantomData<(S, Created, Removed, K, T)>); impl< - S: StorageMap, - Created: Happened, - Removed: Happened, - K: FullCodec, - T: FullCodec, - > StoredMap for StorageMapShim -{ - fn get(k: &K) -> T { - S::get(k) - } - fn is_explicit(k: &K) -> bool { - S::contains_key(k) - } + S: StorageMap, + Created: Happened, + Removed: Happened, + K: FullCodec, + T: FullCodec, +> StoredMap for StorageMapShim { + fn get(k: &K) -> T { S::get(k) } + fn is_explicit(k: &K) -> bool { S::contains_key(k) } fn insert(k: &K, t: T) { let existed = S::contains_key(&k); S::insert(k, t); From b4fc5e1701ad9960fc90fe0063fb76136c98392f Mon Sep 17 00:00:00 2001 From: kianenigma Date: Mon, 18 Jan 2021 13:21:18 +0000 Subject: [PATCH 06/86] some formatting cleanup. --- bin/node/runtime/src/lib.rs | 6 ++--- frame/session/src/lib.rs | 24 ++++++++------------ primitives/npos-elections/compact/src/lib.rs | 1 + primitives/npos-elections/src/phragmen.rs | 5 +--- primitives/npos-elections/src/tests.rs | 8 +------ 5 files changed, 15 insertions(+), 29 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index bc3526af195f7..d5f0d663e0081 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -504,9 +504,9 @@ impl pallet_staking::Config for Runtime { use pallet_election_provider_multi_phase::FallbackStrategy; parameter_types! { - // phase durations - pub const SignedPhase: u32 = 100; - pub const UnsignedPhase: u32 = 100; + // phase durations. 1/4 of the last session for each. + pub const SignedPhase: u32 = EPOCH_DURATION_IN_BLOCKS / 4; + pub const UnsignedPhase: u32 = EPOCH_DURATION_IN_BLOCKS / 4; // fallback: no need to do on-chain phragmen initially. pub const Fallback: FallbackStrategy = FallbackStrategy::Nothing; diff --git a/frame/session/src/lib.rs b/frame/session/src/lib.rs index 6f619129d8eaa..1ee4d123aa79b 100644 --- a/frame/session/src/lib.rs +++ b/frame/session/src/lib.rs @@ -158,25 +158,15 @@ impl< } impl< - BlockNumber: Rem - + Sub - + Zero - + PartialOrd - + Saturating - + Clone, - Period: Get, - Offset: Get, - > EstimateNextSessionRotation for PeriodicSessions -{ - fn average_session_length() -> BlockNumber { - Period::get() - } - + BlockNumber: Rem + Sub + Zero + PartialOrd + Saturating + Clone, + Period: Get, + Offset: Get, +> EstimateNextSessionRotation for PeriodicSessions { fn estimate_next_session_rotation(now: BlockNumber) -> Option { let offset = Offset::get(); let period = Period::get(); Some(if now > offset { - let block_after_last_session = (now.clone() - offset.clone()) % period.clone(); + let block_after_last_session = (now.clone() - offset) % period.clone(); if block_after_last_session > Zero::zero() { now.saturating_add(period.saturating_sub(block_after_last_session)) } else { @@ -198,6 +188,10 @@ impl< // reasonable to come back here and properly calculate the weight of this function. 0 } + + fn average_session_length() -> BlockNumber { + Period::get() + } } /// A trait for managing creation of new validator set. diff --git a/primitives/npos-elections/compact/src/lib.rs b/primitives/npos-elections/compact/src/lib.rs index 01b59c1a43549..191998a341924 100644 --- a/primitives/npos-elections/compact/src/lib.rs +++ b/primitives/npos-elections/compact/src/lib.rs @@ -362,6 +362,7 @@ fn imports() -> Result { } } } + struct SolutionDef { vis: syn::Visibility, ident: syn::Ident, diff --git a/primitives/npos-elections/src/phragmen.rs b/primitives/npos-elections/src/phragmen.rs index bdd64503a56d7..24a6b81af31a7 100644 --- a/primitives/npos-elections/src/phragmen.rs +++ b/primitives/npos-elections/src/phragmen.rs @@ -107,10 +107,7 @@ where .map(|w_ptr| (w_ptr.borrow().who.clone(), w_ptr.borrow().backed_stake)) .collect(); - Ok(ElectionResult { - winners, - assignments, - }) + Ok(ElectionResult { winners, assignments }) } /// Core implementation of seq-phragmen. diff --git a/primitives/npos-elections/src/tests.rs b/primitives/npos-elections/src/tests.rs index a6c50b2fcba30..bc148f118ce42 100644 --- a/primitives/npos-elections/src/tests.rs +++ b/primitives/npos-elections/src/tests.rs @@ -59,13 +59,7 @@ fn float_phragmen_poc_works() { &_Support { own: 0.0, total: 35.0, others: vec![(20u64, 20.0), (30u64, 15.0)] } ); - equalize_float( - phragmen_result.assignments, - &mut support_map, - 0.0, - 2, - stake_of, - ); + equalize_float(phragmen_result.assignments, &mut support_map, 0.0, 2, stake_of); assert_eq!( support_map.get(&2).unwrap(), From cc26881f03f799fe49f5f96f657073cd9d597ff8 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Mon, 18 Jan 2021 15:44:08 +0000 Subject: [PATCH 07/86] Small self-cleanup. --- .../election-provider-multi-phase/src/lib.rs | 130 ++++++++---------- .../election-provider-multi-phase/src/mock.rs | 13 +- .../src/unsigned.rs | 66 ++++++--- 3 files changed, 112 insertions(+), 97 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 79112297f869f..3028ee9ee9b06 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -15,10 +15,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! # Two phase, offchain election provider pallet. +//! # Multi phase, offchain election provider pallet. //! -//! As the name suggests, this election-provider has two distinct phases (see [`Phase`]), signed and -//! unsigned. +//! Currently, this election-provider has two distinct phases (see [`Phase`]), **signed** and +//! **unsigned**. //! //! ## Phases //! @@ -35,13 +35,15 @@ //! ``` //! //! Note that the unsigned phase starts [`pallet::Config::UnsignedPhase`] blocks before the -//! `next_election_prediction`, but only ends when a call to [`ElectionProvider::elect`] happens. +//! `next_election_prediction`, but only ends when a call to [`ElectionProvider::elect`] happens. If +//! no `elect` happens, the signed phase is extended. //! //! > Given this, it is rather important for the user of this pallet to ensure it always terminates //! election via `elect` before requesting a new one. //! //! Each of the phases can be disabled by essentially setting their length to zero. If both phases -//! have length zero, then the pallet essentially runs only the on-chain backup. +//! have length zero, then the pallet essentially runs only the fallback strategy, denoted by +//! [`Config::FallbackStrategy`]. //! //! ### Signed Phase //! @@ -102,6 +104,10 @@ //! than the best queued one (see [`pallet::Config::SolutionImprovementThreshold`]) and will limit //! the weigh of the solution to [`pallet::Config::MinerMaxWeight`]. //! +//! The unsigned phase can be made passive depending on how the previous signed phase went, by +//! setting the first inner value of [`Phase`] to `false`. For now, the signed phase is always +//! active. +//! //! ### Fallback //! //! If we reach the end of both phases (i.e. call to [`ElectionProvider::elect`] happens) and no @@ -165,19 +171,19 @@ //! on-chain election provider as fallback, or special _noop_ fallback that simply returns an error, //! thus replicating [`FallbackStrategy::Nothing`]. //! -//! **Score based on size**: We should always prioritize small solutions over bigger ones, if there -//! is a tie. Even more harsh should be to enforce the bound of the `reduce` algorithm. +//! **Score based on (byte) size**: We should always prioritize small solutions over bigger ones, if +//! there is a tie. Even more harsh should be to enforce the bound of the `reduce` algorithm. #![cfg_attr(not(feature = "std"), no_std)] -use codec::{Decode, Encode, HasCompact}; +use codec::{Decode, Encode}; use frame_support::{ dispatch::DispatchResultWithPostInfo, ensure, traits::{Currency, Get, ReservableCurrency}, weights::Weight, }; -use frame_system::{ensure_none, ensure_signed, offchain::SendTransactionTypes}; +use frame_system::{ensure_none, offchain::SendTransactionTypes}; use sp_election_providers::{ElectionDataProvider, ElectionProvider, onchain}; use sp_npos_elections::{ assignment_ratio_to_staked_normalized, is_score_better, CompactSolution, ElectionScore, @@ -205,20 +211,11 @@ pub mod helpers; const LOG_TARGET: &'static str = "election-provider"; -// for the helper macros -#[doc(hidden)] -pub use sp_runtime::traits::UniqueSaturatedInto; -#[doc(hidden)] -pub use sp_std; - pub mod unsigned; pub mod weights; use weights::WeightInfo; -// pub mod signed; -// use signed::SignedSubmission; - /// The compact solution type used by this crate. pub type CompactOf = ::CompactSolution; @@ -231,6 +228,7 @@ pub type CompactAccuracyOf = as CompactSolution>::Accuracy; /// The accuracy of the election, when computed on-chain. Equal to [`Config::OnChainAccuracy`]. pub type OnChainAccuracyOf = ::OnChainAccuracy; +/// Wrapper type that implements the configurations needed for the on-chain backup. struct OnChainConfig(sp_std::marker::PhantomData) where ExtendedBalance: From>>, @@ -310,7 +308,7 @@ impl Phase { } } -/// A configuration for the module to indicate what should happen in the case of a fallback i.e. +/// A configuration for the pallet to indicate what should happen in the case of a fallback i.e. /// reaching a call to `elect` with no good solution. #[cfg_attr(test, derive(Clone))] pub enum FallbackStrategy { @@ -379,7 +377,7 @@ pub struct ReadySolution { compute: ElectionCompute, } -/// Solution size of the election. +/// Size of the snapshot from which the solution was derived. /// /// This is needed for proper weight calculation. #[derive(PartialEq, Eq, Clone, Copy, Encode, Decode, RuntimeDebug, Default)] @@ -397,10 +395,9 @@ pub struct SolutionSize { } /// A snapshot of all the data that is needed for en entire round. They are provided by -/// [`ElectionDataProvider`] at the beginning of the signed phase (or the unsigned phase, if signed -/// phase is non-existent) and are kept around until the round is finished. +/// [`ElectionDataProvider`] and are kept around until the round is finished. /// -/// These are stored together because they are often times accessed together. +/// These are stored together because they are often accessed together. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, Default)] pub struct RoundSnapshot { /// All of the voters. @@ -433,16 +430,16 @@ pub enum ElectionError { Feasibility(FeasibilityError), /// An error in the on-chain fallback. OnChainFallback(onchain::Error), - /// No fallback is configured + /// No fallback is configured. NoFallbackConfigured, /// An internal error in the NPoS elections crate. NposElections(sp_npos_elections::Error), /// Snapshot data was unavailable unexpectedly. SnapshotUnAvailable, /// Submitting a transaction to the pool failed. - /// - /// This can only happen in the unsigned phase. PoolSubmissionFailed, + /// The pre-dispatch checks failed for the mined solution. + PreDispatchChecksFailed, } impl From for ElectionError { @@ -470,7 +467,8 @@ pub enum FeasibilityError { WrongWinnerCount, /// The snapshot is not available. /// - /// This must be an internal error of the chain. + /// Kinda defensive: The pallet should technically never attempt to do a feasibility check when + /// no snapshot is present. SnapshotUnavailable, /// Internal error from the election crate. NposElection(sp_npos_elections::Error), @@ -520,12 +518,12 @@ pub mod pallet { type SignedPhase: Get; /// The minimum amount of improvement to the solution score that defines a solution as - /// "better". + /// "better" (in any phase). #[pallet::constant] type SolutionImprovementThreshold: Get; /// The priority of the unsigned transaction submitted in the unsigned-phase - type UnsignedPriority: Get; + type MinerTxPriority: Get; /// Maximum number of iteration of balancing that will be executed in the embedded miner of /// the pallet. type MinerMaxIterations: Get; @@ -586,15 +584,17 @@ pub mod pallet { { let (need_snapshot, enabled, additional) = if current_phase == Phase::Signed { // followed by a signed phase: close the signed phase, no need for snapshot. - // TODO + // NOTE: SIGNED_PHASE (false, true, Weight::zero()) } else { - // no signed phase + // no signed phase: create a new snapshot, definitely `enable` the unsigned + // phase. (true, true, Weight::zero()) }; Self::on_initialize_open_unsigned(need_snapshot, enabled, now); log!(info, "Starting unsigned phase({}) at #{:?}.", enabled, now); + let base_weight = if need_snapshot { T::WeightInfo::on_initialize_open_unsigned_with_snapshot() } else { @@ -616,7 +616,7 @@ pub mod pallet { } Err(e) => log!(error, "error while submitting transaction in OCW: {:?}", e), }, - Err(why) => log!(error, "Error in unsigned offchain worker: {:?}", why), + Err(why) => log!(warn, "denied offchain worker: {:?}", why), } } } @@ -658,16 +658,16 @@ pub mod pallet { /// /// The dispatch origin fo this call must be __none__. /// - /// This submission is checked on the fly, thus it is likely yo be more limited and smaller. - /// Moreover, this unsigned solution is only validated when submitted to the pool from the - /// local process. Effectively, this means that only active validators can submit this - /// transaction when authoring a block. + /// This submission is checked on the fly. Moreover, this unsigned solution is only + /// validated when submitted to the pool from the **local** node. Effectively, this means + /// that only active validators can submit this transaction when authoring a block (similar + /// to an inherent). /// /// To prevent any incorrect solution (and thus wasted time/weight), this transaction will - /// panic if the solution submitted by the validator is invalid, effectively putting their - /// authoring reward at risk. + /// panic if the solution submitted by the validator is invalid in any way, effectively + /// putting their authoring reward at risk. /// - /// No deposit or reward is associated with this. + /// No deposit or reward is associated with this submission. #[pallet::weight(T::WeightInfo::submit_unsigned( witness.voters, witness.targets, @@ -683,8 +683,7 @@ pub mod pallet { let error_message = "Invalid unsigned submission must produce invalid block and \ deprive validator from their authoring reward."; - // check phase and score. - // NOTE: since we do this in pre-dispatch, we can just ignore it here. + // NOTE: since we do this in pre-dispatch, we _could_ just ignore it here. Self::unsigned_pre_dispatch_checks(&solution).expect(error_message); // ensure witness was correct. @@ -733,6 +732,7 @@ pub mod pallet { UnsignedPhaseStarted(u32), } + /// Error of the pallet that can be returned in response to dispatches. #[pallet::error] pub enum Error { /// Submission was too early. @@ -781,15 +781,14 @@ pub mod pallet { ValidTransaction::with_tag_prefix("OffchainElection") // The higher the score[0], the better a solution is. .priority( - T::UnsignedPriority::get() - .saturating_add(solution.score[0].saturated_into()), + T::MinerTxPriority::get().saturating_add(solution.score[0].saturated_into()), ) // used to deduplicate unsigned solutions: each validator should produce one // solution per round at most, and solutions are not propagate. .and_provides(solution.round) // transaction should stay in the pool for the duration of the unsigned phase. .longevity(T::UnsignedPhase::get().saturated_into::()) - // We don't propagate this. This can never the validated at a remote node. + // We don't propagate this. This can never be validated at a remote node. .propagate(false) .build() } else { @@ -816,7 +815,7 @@ pub mod pallet { /// Internal counter for the number of rounds. /// /// This is useful for de-duplication of transactions submitted to the pool, and general - /// diagnostics of the module. + /// diagnostics of the pallet. /// /// This is merely incremented once per every time that an upstream `elect` is called. #[pallet::storage] @@ -828,7 +827,7 @@ pub mod pallet { #[pallet::getter(fn current_phase)] pub type CurrentPhase = StorageValue<_, Phase, ValueQuery>; - /// Current best solution, signed or unsigned. + /// Current best solution, signed or unsigned, queued to be returned upon `elect`. #[pallet::storage] #[pallet::getter(fn queued_solution)] pub type QueuedSolution = StorageValue<_, ReadySolution>; @@ -867,7 +866,7 @@ where /// Logic for `::on_initialize` when signed phase is being opened. /// /// This is decoupled for easy weight calculation. - pub fn on_initialize_open_signed() { + pub(crate) fn on_initialize_open_signed() { >::put(Phase::Signed); Self::create_snapshot(); Self::deposit_event(Event::SignedPhaseStarted(Self::round())); @@ -877,7 +876,7 @@ where /// /// This is decoupled for easy weight calculation. Note that the default weight benchmark of /// this function will assume an empty signed queue for `finalize_signed_phase`. - pub fn on_initialize_open_unsigned( + pub(crate) fn on_initialize_open_unsigned( need_snapshot: bool, enabled: bool, now: T::BlockNumber, @@ -888,7 +887,6 @@ where Self::create_snapshot(); } - // for now always start the unsigned phase. >::put(Phase::Unsigned((enabled, now))); Self::deposit_event(Event::UnsignedPhaseStarted(Self::round())); } @@ -898,7 +896,7 @@ where /// 1. [`SnapshotMetadata`] /// 2. [`RoundSnapshot`] /// 3. [`DesiredTargets`] - pub fn create_snapshot() { + pub(crate) fn create_snapshot() { // if any of them don't exist, create all of them. This is a bit conservative. let targets = T::DataProvider::targets(); let voters = T::DataProvider::voters(); @@ -912,15 +910,14 @@ where >::put(RoundSnapshot { voters, targets }); } + /// Kill everything created by [`Pallet::create_snapshot`]. + pub(crate) fn kill_snapshot() { + >::kill(); + >::kill(); + >::kill(); + } + /// Checks the feasibility of a solution. - /// - /// This checks the solution for the following: - /// - /// 0. **all** of the used indices must be correct. - /// 1. present correct number of winners. - /// 2. any assignment is checked to match with [Snapshot::voters]. - /// 3. for each assignment, the check of `ElectionDataProvider` is also examined. - /// 4. the claimed score is valid. fn feasibility_check( solution: RawSolution>, compute: ElectionCompute, @@ -938,7 +935,7 @@ where // NOTE: this is a bit of duplicate, but we keep it around for veracity. The unsigned path // already checked this in `unsigned_per_dispatch_checks`. The signed path *could* check it - // upon arrival. + // upon arrival, thus we would then remove it here. Given overlay it is cheap anyhow ensure!(winners.len() as u32 == desired_targets, FeasibilityError::WrongWinnerCount,); // read the entire snapshot. @@ -957,8 +954,7 @@ where .map(|i| target_at(i).ok_or(FeasibilityError::InvalidWinner)) .collect::, FeasibilityError>>()?; - // Then convert compact -> Assignment. This will fail if any of the indices are gibberish. - // that winner indices are already checked. + // Then convert compact -> assignment. This will fail if any of the indices are gibberish. let assignments = compact .into_assignment(voter_at, target_at) .map_err::(Into::into)?; @@ -993,6 +989,7 @@ where // This might fail if the normalization fails. Very unlikely. See `integrity_test`. let staked_assignments = assignment_ratio_to_staked_normalized(assignments, stake_of) .map_err::(Into::into)?; + // This might fail if one of the voter edges is pointing to a non-winner, which is not // really possible anymore because all the winners come from the same `compact`. let supports = sp_npos_elections::to_supports(&winners, &staked_assignments) @@ -1018,9 +1015,7 @@ where >::put(Phase::Off); // kill snapshots - >::kill(); - >::kill(); - >::kill(); + Self::kill_snapshot(); } /// On-chain fallback of election. @@ -1036,10 +1031,6 @@ where } fn do_elect() -> Result, ElectionError> { - // NOTE: SignedSubmission is guaranteed to be drained by the end of the signed phase too, - // thus no need for a manual cleanup: - // TODO - // debug_assert!(Self::signed_submissions().is_empty()); >::take() .map_or_else( || match T::Fallback::get() { @@ -1073,7 +1064,6 @@ where fn elect() -> Result, Self::Error> { let outcome = Self::do_elect(); - // cleanup. Self::post_elect(); outcome } @@ -1144,7 +1134,7 @@ mod feasibility_check { assert_noop!( TwoPhase::feasibility_check(solution, COMPUTE), - FeasibilityError::WrongWinnerCount + FeasibilityError::WrongWinnerCount, ); }) } diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index 71102f3e6e526..6fcebe7f08eaf 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -93,11 +93,6 @@ pub fn roll_to_with_ocw(n: u64) { } } -/// Get the free and reserved balance of some account. -pub fn balances(who: &AccountId) -> (Balance, Balance) { - (Balances::free_balance(who), Balances::reserved_balance(who)) -} - /// Spit out a verifiable raw solution. /// /// This is a good example of what an offchain miner would do. @@ -204,7 +199,7 @@ parameter_types! { pub static MaxSignedSubmissions: u32 = 5; pub static MinerMaxIterations: u32 = 5; - pub static UnsignedPriority: u64 = 100; + pub static MinerTxPriority: u64 = 100; pub static SolutionImprovementThreshold: Perbill = Perbill::zero(); pub static MinerMaxWeight: Weight = BlockWeights::get().max_block; pub static MockWeightInfo: bool = false; @@ -272,7 +267,7 @@ impl crate::Config for Runtime { type SolutionImprovementThreshold = SolutionImprovementThreshold; type MinerMaxIterations = MinerMaxIterations; type MinerMaxWeight = MinerMaxWeight; - type UnsignedPriority = UnsignedPriority; + type MinerTxPriority = MinerTxPriority; type DataProvider = StakingMock; type WeightInfo = DualMockWeightInfo; type BenchmarkingConfig = (); @@ -311,8 +306,8 @@ impl ElectionDataProvider for StakingMock { } impl ExtBuilder { - pub fn unsigned_priority(self, p: u64) -> Self { - ::set(p); + pub fn miner_tx_priority(self, p: u64) -> Self { + ::set(p); self } pub fn solution_improvement_threshold(self, p: Perbill) -> Self { diff --git a/frame/election-provider-multi-phase/src/unsigned.rs b/frame/election-provider-multi-phase/src/unsigned.rs index e922dc32dfa34..4907d3f0e761a 100644 --- a/frame/election-provider-multi-phase/src/unsigned.rs +++ b/frame/election-provider-multi-phase/src/unsigned.rs @@ -20,14 +20,20 @@ use crate::*; use frame_support::dispatch::DispatchResult; use frame_system::offchain::SubmitTransaction; -use sp_npos_elections::{seq_phragmen, CompactSolution, ElectionResult}; +use sp_npos_elections::{ + seq_phragmen, CompactSolution, ElectionResult, assignment_ratio_to_staked_normalized, reduce, + assignment_staked_to_ratio_normalized, +}; use sp_runtime::{offchain::storage::StorageValueRef, traits::TrailingZeroInput}; use sp_std::cmp::Ordering; /// Storage key used to store the persistent offchain worker status. pub(crate) const OFFCHAIN_HEAD_DB: &[u8] = b"parity/multi-phase-unsigned-election/"; + /// The repeat threshold of the offchain worker. This means we won't run the offchain worker twice /// within a window of 5 blocks. +// TODO: this should go into config, and we should store the solution an repeat with this threshold +// until we can submit it, or if the election happened. Okay for now though pub(crate) const OFFCHAIN_REPEAT: u32 = 5; impl Pallet @@ -35,7 +41,7 @@ where ExtendedBalance: From>>, ExtendedBalance: From>>, { - /// Min a new npos solution. + /// Mine a new npos solution. pub fn mine_solution( iters: usize, ) -> Result<(RawSolution>, SolutionSize), ElectionError> { @@ -50,14 +56,15 @@ where Some((iters, 0)), ) .map_err(Into::into) - .and_then(|election_result| { - if election_result.winners.len() as u32 == desired_targets { - Ok(election_result) - } else { - Err(ElectionError::Feasibility(FeasibilityError::WrongWinnerCount)) - } - }) .and_then(Self::prepare_election_result) + .and_then(|(raw_solution, size)| { + Self::unsigned_pre_dispatch_checks(&raw_solution) + .map_err(|e| { + log!(warn, "pre-disaptch-checks failed for mined solution: {:?}", e); + ElectionError::PreDispatchChecksFailed + })?; + Ok((raw_solution, size)) + }) } /// Convert a raw solution from [`sp_npos_elections::ElectionResult`] to [`RawSolution`], which @@ -67,6 +74,9 @@ where pub fn prepare_election_result( election_result: ElectionResult>, ) -> Result<(RawSolution>, SolutionSize), ElectionError> { + // NOTE: This code path is generally not optimized as it is run offchain. Could use some at + // some point though. + // storage items. Note: we have already read this from storage, they must be in cache. let RoundSnapshot { voters, targets } = Self::snapshot().ok_or(ElectionError::SnapshotUnAvailable)?; @@ -83,13 +93,12 @@ where let ElectionResult { assignments, winners } = election_result; // convert to staked and reduce. - let mut staked = - sp_npos_elections::assignment_ratio_to_staked_normalized(assignments, &stake_of) - .map_err::(Into::into)?; + let mut staked = assignment_ratio_to_staked_normalized(assignments, &stake_of) + .map_err::(Into::into)?; sp_npos_elections::reduce(&mut staked); // convert back to ration and make compact. - let ratio = sp_npos_elections::assignment_staked_to_ratio_normalized(staked)?; + let ratio = assignment_staked_to_ratio_normalized(staked)?; let compact = >::from_assignment(ratio, &voter_index, &target_index)?; let size = SolutionSize { voters: voters.len() as u32, targets: targets.len() as u32 }; @@ -116,7 +125,8 @@ where /// Get a random number of iterations to run the balancing in the OCW. /// - /// Uses the offchain seed to generate a random number, maxed with `T::MinerMaxIterations`. + /// Uses the offchain seed to generate a random number, maxed with + /// [`Config::MinerMaxIterations`]. pub fn get_balancing_iters() -> usize { match T::MinerMaxIterations::get() { 0 => 0, @@ -315,11 +325,18 @@ where .map_err(|_| ElectionError::PoolSubmissionFailed) } + /// Do the basics checks that MUST happen during the validation and pre-dispatch of an unsigned + /// transaction. + /// + /// Can optionally also be called during dispatch, if needed. + /// + /// NOTE: Ideally, these tests should move more and more outside of this and more to the miner's + /// code, so that we do less and less storage reads here. pub(crate) fn unsigned_pre_dispatch_checks( solution: &RawSolution>, ) -> DispatchResult { // ensure solution is timely. Don't panic yet. This is a cheap check. - ensure!(Self::current_phase().is_unsigned_open(), Error::::EarlySubmission); + ensure!(Self::current_phase().is_unsigned_open(), Error::::EarlySubmission,); // ensure correct number of winners. ensure!( @@ -335,7 +352,7 @@ where q.score, T::SolutionImprovementThreshold::get() )), - Error::::WeakSubmission + Error::::WeakSubmission, ); Ok(()) @@ -476,6 +493,19 @@ mod tests { ) .is_ok()); assert!(::pre_dispatch(&call).is_ok()); + + // unsigned -- but not enabled. + >::put(Phase::Unsigned((false, 25))); + assert!(TwoPhase::current_phase().is_unsigned()); + assert!(matches!( + ::validate_unsigned(TransactionSource::Local, &call) + .unwrap_err(), + TransactionValidityError::Invalid(InvalidTransaction::Custom(0)) + )); + assert!(matches!( + ::pre_dispatch(&call).unwrap_err(), + TransactionValidityError::Invalid(InvalidTransaction::Custom(0)) + )); }) } @@ -534,7 +564,7 @@ mod tests { #[test] fn priority_is_set() { - ExtBuilder::default().unsigned_priority(20).desired_targets(0).build_and_execute(|| { + ExtBuilder::default().miner_tx_priority(20).desired_targets(0).build_and_execute(|| { roll_to(25); assert!(TwoPhase::current_phase().is_unsigned()); @@ -650,7 +680,7 @@ mod tests { // mine seq_phragmen solution with 2 iters. assert_eq!( TwoPhase::mine_solution(2).unwrap_err(), - ElectionError::Feasibility(FeasibilityError::WrongWinnerCount), + ElectionError::PreDispatchChecksFailed, ); }) } From 15cdb28c2335b1498a2b82f72bf7125fb7ac27db Mon Sep 17 00:00:00 2001 From: kianenigma Date: Tue, 19 Jan 2021 11:39:13 +0000 Subject: [PATCH 08/86] Add todo --- frame/election-provider-multi-phase/src/lib.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 664030315dc73..c1dd24f3387c9 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -1131,6 +1131,8 @@ where // change phase >::put(Phase::Off); + // TODO: clean signed submissions: test case: call `elect` in the middle of the signed phase. + // kill snapshots >::kill(); >::kill(); @@ -1150,10 +1152,6 @@ where } fn do_elect() -> Result, ElectionError> { - // NOTE: SignedSubmission is guaranteed to be drained by the end of the signed phase too, - // thus no need for a manual cleanup: - // TODO - // debug_assert!(Self::signed_submissions().is_empty()); >::take() .map_or_else( || match T::Fallback::get() { From 75eca3f450076eee1e022945ce928a57e496412f Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 27 Jan 2021 09:33:39 +0000 Subject: [PATCH 09/86] Make it all build --- bin/node/runtime/src/lib.rs | 2 +- .../src/benchmarking.rs | 18 +--- .../src/helpers.rs | 60 +++---------- .../election-provider-multi-phase/src/lib.rs | 90 +++++++------------ .../election-provider-multi-phase/src/mock.rs | 2 +- .../src/unsigned.rs | 26 ++---- frame/staking/src/lib.rs | 5 +- 7 files changed, 62 insertions(+), 141 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 636e0b4b784cc..0636817e456d4 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -530,7 +530,7 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type SolutionImprovementThreshold = MinSolutionScoreBump; type MinerMaxIterations = MinerMaxIterations; type MinerMaxWeight = MinerMaxWeight; - type UnsignedPriority = TwoPhaseUnsignedPriority; + type MinerTxPriority = TwoPhaseUnsignedPriority; type DataProvider = Staking; type OnChainAccuracy = Perbill; type CompactSolution = pallet_staking::CompactAssignments; diff --git a/frame/election-provider-multi-phase/src/benchmarking.rs b/frame/election-provider-multi-phase/src/benchmarking.rs index a7a4eed852850..4cd6bdf60a4ef 100644 --- a/frame/election-provider-multi-phase/src/benchmarking.rs +++ b/frame/election-provider-multi-phase/src/benchmarking.rs @@ -25,9 +25,8 @@ use frame_support::{assert_ok, traits::OnInitialize}; use frame_system::RawOrigin; use rand::{prelude::SliceRandom, rngs::SmallRng, SeedableRng}; use sp_election_providers::Assignment; -use sp_npos_elections::ExtendedBalance; -use sp_runtime::InnerOf; use sp_arithmetic::traits::One; +use sp_runtime::InnerOf; use sp_std::convert::TryInto; const SEED: u32 = 0; @@ -39,12 +38,7 @@ fn solution_with_size( size: SolutionSize, active_voters_count: u32, desired_targets: u32, -) -> RawSolution> -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, - > as sp_std::convert::TryFrom>::Error: sp_std::fmt::Debug, -{ +) -> RawSolution> { assert!(size.targets >= desired_targets, "must have enough targets"); assert!( size.targets >= (>::LIMIT * 2) as u32, @@ -129,7 +123,7 @@ where .iter() .map(|(voter, _stake, votes)| { let percent_per_edge: InnerOf> = - (100 / votes.len()).try_into().unwrap(); + (100 / votes.len()).try_into().unwrap_or_else(|_| panic!("failed to convert")); Assignment { who: voter.clone(), distribution: votes @@ -148,12 +142,6 @@ where } benchmarks! { - where_clause { - where ExtendedBalance: From>>, - > as sp_std::convert::TryFrom>::Error: sp_std::fmt::Debug, - ExtendedBalance: From>>, - } - on_initialize_nothing { assert!(>::current_phase().is_off()); }: { diff --git a/frame/election-provider-multi-phase/src/helpers.rs b/frame/election-provider-multi-phase/src/helpers.rs index da4a092653ae7..9b2f817f1ca09 100644 --- a/frame/election-provider-multi-phase/src/helpers.rs +++ b/frame/election-provider-multi-phase/src/helpers.rs @@ -17,11 +17,7 @@ //! Some helper functions/macros for this crate. -use super::{ - Config, VoteWeight, CompactVoterIndexOf, CompactTargetIndexOf, CompactAccuracyOf, - OnChainAccuracyOf, ExtendedBalance, -}; -use sp_runtime::InnerOf; +use super::{Config, VoteWeight, CompactVoterIndexOf, CompactTargetIndexOf}; use sp_std::{collections::btree_map::BTreeMap, convert::TryInto, boxed::Box, prelude::*}; #[macro_export] @@ -39,11 +35,7 @@ macro_rules! log { /// This can be used to efficiently build index getter closures. pub fn generate_voter_cache( snapshot: &Vec<(T::AccountId, VoteWeight, Vec)>, -) -> BTreeMap -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +) -> BTreeMap { let mut cache: BTreeMap = BTreeMap::new(); snapshot.iter().enumerate().for_each(|(i, (x, _, _))| { let _existed = cache.insert(x.clone(), i); @@ -64,11 +56,7 @@ where /// The snapshot must be the same is the one used to create `cache`. pub fn voter_index_fn( cache: &BTreeMap, -) -> Box Option> + '_> -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +) -> Box Option> + '_> { Box::new(move |who| { cache.get(who).and_then(|i| >>::try_into(*i).ok()) }) @@ -81,11 +69,7 @@ where /// The snapshot must be the same is the one used to create `cache`. pub fn voter_index_fn_usize( cache: &BTreeMap, -) -> Box Option + '_> -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +) -> Box Option + '_> { Box::new(move |who| cache.get(who).cloned()) } @@ -97,11 +81,7 @@ where /// Not meant to be used in production. pub fn voter_index_fn_linear( snapshot: &Vec<(T::AccountId, VoteWeight, Vec)>, -) -> Box Option> + '_> -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +) -> Box Option> + '_> { Box::new(move |who| { snapshot .iter() @@ -115,11 +95,7 @@ where /// The returning index type is the same as the one defined in [`T::CompactSolution::Target`]. pub fn target_index_fn_linear( snapshot: &Vec, -) -> Box Option> + '_> -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +) -> Box Option> + '_> { Box::new(move |who| { snapshot .iter() @@ -132,11 +108,7 @@ where /// account using a linearly indexible snapshot. pub fn voter_at_fn( snapshot: &Vec<(T::AccountId, VoteWeight, Vec)>, -) -> Box) -> Option + '_> -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +) -> Box) -> Option + '_> { Box::new(move |i| { as TryInto>::try_into(i) .ok() @@ -148,11 +120,7 @@ where /// account using a linearly indexible snapshot. pub fn target_at_fn( snapshot: &Vec, -) -> Box) -> Option + '_> -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +) -> Box) -> Option + '_> { Box::new(move |i| { as TryInto>::try_into(i) .ok() @@ -165,11 +133,7 @@ where /// This is not optimized and uses a linear search. pub fn stake_of_fn_linear( snapshot: &Vec<(T::AccountId, VoteWeight, Vec)>, -) -> Box VoteWeight + '_> -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +) -> Box VoteWeight + '_> { Box::new(move |who| { snapshot.iter().find(|(x, _, _)| x == who).map(|(_, x, _)| *x).unwrap_or_default() }) @@ -184,11 +148,7 @@ where pub fn stake_of_fn<'a, T: Config>( snapshot: &'a Vec<(T::AccountId, VoteWeight, Vec)>, cache: &'a BTreeMap, -) -> Box VoteWeight + 'a> -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +) -> Box VoteWeight + 'a> { Box::new(move |who| { if let Some(index) = cache.get(who) { snapshot.get(*index).map(|(_, x, _)| x).cloned().unwrap_or_default() diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 3028ee9ee9b06..f528f37a82fbf 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -173,6 +173,10 @@ //! //! **Score based on (byte) size**: We should always prioritize small solutions over bigger ones, if //! there is a tie. Even more harsh should be to enforce the bound of the `reduce` algorithm. +//! +//! **Offchain resubmit**: Essentially port https://github.com/paritytech/substrate/pull/7976 to +//! this pallet as well. The `OFFCHAIN_REPEAT` also needs to become an adjustable parameter of the +//! pallet. #![cfg_attr(not(feature = "std"), no_std)] @@ -187,14 +191,14 @@ use frame_system::{ensure_none, offchain::SendTransactionTypes}; use sp_election_providers::{ElectionDataProvider, ElectionProvider, onchain}; use sp_npos_elections::{ assignment_ratio_to_staked_normalized, is_score_better, CompactSolution, ElectionScore, - EvaluateSupport, ExtendedBalance, PerThing128, Supports, VoteWeight, + EvaluateSupport, PerThing128, Supports, VoteWeight, }; use sp_runtime::{ transaction_validity::{ InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity, TransactionValidityError, ValidTransaction, }, - DispatchError, InnerOf, PerThing, Perbill, RuntimeDebug, SaturatedConversion, + DispatchError, PerThing, Perbill, RuntimeDebug, SaturatedConversion, }; use sp_std::prelude::*; use sp_arithmetic::{ @@ -229,15 +233,8 @@ pub type CompactAccuracyOf = as CompactSolution>::Accuracy; pub type OnChainAccuracyOf = ::OnChainAccuracy; /// Wrapper type that implements the configurations needed for the on-chain backup. -struct OnChainConfig(sp_std::marker::PhantomData) -where - ExtendedBalance: From>>, - ExtendedBalance: From>>; -impl onchain::Config for OnChainConfig -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +struct OnChainConfig(sp_std::marker::PhantomData); +impl onchain::Config for OnChainConfig { type AccountId = T::AccountId; type BlockNumber = T::BlockNumber; type Accuracy = T::OnChainAccuracy; @@ -498,11 +495,7 @@ pub mod pallet { use frame_system::pallet_prelude::*; #[pallet::config] - pub trait Config: frame_system::Config + SendTransactionTypes> - where - ExtendedBalance: From>>, - ExtendedBalance: From>>, - { + pub trait Config: frame_system::Config + SendTransactionTypes> { type Event: From> + Into<::Event> + IsType<::Event>; @@ -559,11 +552,7 @@ pub mod pallet { } #[pallet::hooks] - impl Hooks> for Pallet - where - ExtendedBalance: From>>, - ExtendedBalance: From>>, - { + impl Hooks> for Pallet { fn on_initialize(now: T::BlockNumber) -> Weight { let next_election = T::DataProvider::next_election_prediction(now).max(now); @@ -609,7 +598,7 @@ pub mod pallet { fn offchain_worker(n: T::BlockNumber) { // We only run the OCW in the fist block of the unsigned phase. if Self::current_phase().is_unsigned_open_at(n) { - match Self::set_check_offchain_execution_status(n) { + match Self::try_acquire_offchain_lock(n) { Ok(_) => match Self::mine_and_submit() { Ok(_) => { log!(info, "successfully submitted a solution via OCW at block {:?}", n) @@ -633,15 +622,25 @@ pub mod pallet { let max_vote: usize = as CompactSolution>::LIMIT; // 1. Maximum sum of [ChainAccuracy; 16] must fit into `UpperOf`.. - let maximum_chain_accuracy: Vec>> = - (0..max_vote).map(|_| >::one().deconstruct().into()).collect(); + let maximum_chain_accuracy: Vec>> = (0..max_vote) + .map(|_| { + >>::from( + >::one().deconstruct(), + ) + }) + .collect(); let _: UpperOf> = maximum_chain_accuracy .iter() .fold(Zero::zero(), |acc, x| acc.checked_add(x).unwrap()); // 2. Maximum sum of [CompactAccuracy; 16] must fit into `UpperOf`. - let maximum_chain_accuracy: Vec>> = - (0..max_vote).map(|_| >::one().deconstruct().into()).collect(); + let maximum_chain_accuracy: Vec>> = (0..max_vote) + .map(|_| { + >>::from( + >::one().deconstruct(), + ) + }) + .collect(); let _: UpperOf> = maximum_chain_accuracy .iter() .fold(Zero::zero(), |acc, x| acc.checked_add(x).unwrap()); @@ -649,11 +648,7 @@ pub mod pallet { } #[pallet::call] - impl Pallet - where - ExtendedBalance: From>>, - ExtendedBalance: From>>, - { + impl Pallet { /// Submit a solution for the unsigned phase. /// /// The dispatch origin fo this call must be __none__. @@ -709,11 +704,7 @@ pub mod pallet { #[pallet::event] #[pallet::metadata(::AccountId = "AccountId")] #[pallet::generate_deposit(pub(super) fn deposit_event)] - pub enum Event - where - ExtendedBalance: From>>, - ExtendedBalance: From>>, - { + pub enum Event { /// A solution was stored with the given compute. /// /// If the solution is signed, this means that it hasn't yet been processed. If the @@ -755,11 +746,7 @@ pub mod pallet { pub struct Origin(PhantomData); #[pallet::validate_unsigned] - impl ValidateUnsigned for Pallet - where - ExtendedBalance: From>>, - ExtendedBalance: From>>, - { + impl ValidateUnsigned for Pallet { type Call = Call; fn validate_unsigned(source: TransactionSource, call: &Self::Call) -> TransactionValidity { if let Call::submit_unsigned(solution, _) = call { @@ -781,7 +768,9 @@ pub mod pallet { ValidTransaction::with_tag_prefix("OffchainElection") // The higher the score[0], the better a solution is. .priority( - T::MinerTxPriority::get().saturating_add(solution.score[0].saturated_into()), + T::MinerTxPriority::get().saturating_add( + solution.score[0].saturated_into() + ), ) // used to deduplicate unsigned solutions: each validator should produce one // solution per round at most, and solutions are not propagate. @@ -858,11 +847,7 @@ pub mod pallet { pub struct Pallet(PhantomData); } -impl Pallet -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +impl Pallet { /// Logic for `::on_initialize` when signed phase is being opened. /// /// This is decoupled for easy weight calculation. @@ -1019,10 +1004,7 @@ where } /// On-chain fallback of election. - fn onchain_fallback() -> Result, ElectionError> - where - ExtendedBalance: From<::Inner>, - { + fn onchain_fallback() -> Result, ElectionError> { > as ElectionProvider< T::AccountId, T::BlockNumber, @@ -1054,11 +1036,7 @@ where } } -impl ElectionProvider for Pallet -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +impl ElectionProvider for Pallet { type Error = ElectionError; type DataProvider = T::DataProvider; diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index 6fcebe7f08eaf..8f9c6b58d4268 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -38,7 +38,7 @@ use sp_npos_elections::{ }; use sp_runtime::{ testing::Header, - traits::{BlakeTwo256, Block as BlockT, IdentityLookup}, + traits::{BlakeTwo256, IdentityLookup}, PerU16, }; use std::sync::Arc; diff --git a/frame/election-provider-multi-phase/src/unsigned.rs b/frame/election-provider-multi-phase/src/unsigned.rs index 4907d3f0e761a..01a9fe3d6e4f0 100644 --- a/frame/election-provider-multi-phase/src/unsigned.rs +++ b/frame/election-provider-multi-phase/src/unsigned.rs @@ -21,7 +21,7 @@ use crate::*; use frame_support::dispatch::DispatchResult; use frame_system::offchain::SubmitTransaction; use sp_npos_elections::{ - seq_phragmen, CompactSolution, ElectionResult, assignment_ratio_to_staked_normalized, reduce, + seq_phragmen, CompactSolution, ElectionResult, assignment_ratio_to_staked_normalized, assignment_staked_to_ratio_normalized, }; use sp_runtime::{offchain::storage::StorageValueRef, traits::TrailingZeroInput}; @@ -32,15 +32,9 @@ pub(crate) const OFFCHAIN_HEAD_DB: &[u8] = b"parity/multi-phase-unsigned-electio /// The repeat threshold of the offchain worker. This means we won't run the offchain worker twice /// within a window of 5 blocks. -// TODO: this should go into config, and we should store the solution an repeat with this threshold -// until we can submit it, or if the election happened. Okay for now though pub(crate) const OFFCHAIN_REPEAT: u32 = 5; -impl Pallet -where - ExtendedBalance: From>>, - ExtendedBalance: From>>, -{ +impl Pallet { /// Mine a new npos solution. pub fn mine_solution( iters: usize, @@ -279,9 +273,7 @@ where /// don't run twice within a window of length [`OFFCHAIN_REPEAT`]. /// /// Returns `Ok(())` if offchain worker should happen, `Err(reason)` otherwise. - pub(crate) fn set_check_offchain_execution_status( - now: T::BlockNumber, - ) -> Result<(), &'static str> { + pub(crate) fn try_acquire_offchain_lock(now: T::BlockNumber) -> Result<(), &'static str> { let storage = StorageValueRef::persistent(&OFFCHAIN_HEAD_DB); let threshold = T::BlockNumber::from(OFFCHAIN_REPEAT); @@ -762,25 +754,25 @@ mod tests { assert!(TwoPhase::current_phase().is_unsigned()); // first execution -- okay. - assert!(TwoPhase::set_check_offchain_execution_status(25).is_ok()); + assert!(TwoPhase::try_acquire_offchain_lock(25).is_ok()); // next block: rejected. - assert!(TwoPhase::set_check_offchain_execution_status(26).is_err()); + assert!(TwoPhase::try_acquire_offchain_lock(26).is_err()); // allowed after `OFFCHAIN_REPEAT` - assert!(TwoPhase::set_check_offchain_execution_status((26 + OFFCHAIN_REPEAT).into()) + assert!(TwoPhase::try_acquire_offchain_lock((26 + OFFCHAIN_REPEAT).into()) .is_ok()); // a fork like situation: re-execute last 3. - assert!(TwoPhase::set_check_offchain_execution_status( + assert!(TwoPhase::try_acquire_offchain_lock( (26 + OFFCHAIN_REPEAT - 3).into() ) .is_err()); - assert!(TwoPhase::set_check_offchain_execution_status( + assert!(TwoPhase::try_acquire_offchain_lock( (26 + OFFCHAIN_REPEAT - 2).into() ) .is_err()); - assert!(TwoPhase::set_check_offchain_execution_status( + assert!(TwoPhase::try_acquire_offchain_lock( (26 + OFFCHAIN_REPEAT - 1).into() ) .is_err()); diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 685b9fd380293..f7ea7d5f98b58 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -3381,7 +3381,10 @@ impl sp_election_providers::ElectionDataProvider, ) { targets.into_iter().for_each(|v| { - >::insert(v, ValidatorPrefs { commission: Perbill::zero() }); + >::insert( + v, + ValidatorPrefs { commission: Perbill::zero(), blocked: false }, + ); }); voters.into_iter().for_each(|(v, _s, t)| { From 8daec3a9e38a5ebe717a5c71f673fb35afd1ae9e Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 27 Jan 2021 10:14:56 +0000 Subject: [PATCH 10/86] self-review --- .../src/benchmarking.rs | 12 +- .../election-provider-multi-phase/src/lib.rs | 127 ++++++++---------- .../election-provider-multi-phase/src/mock.rs | 4 +- .../src/unsigned.rs | 67 +++++---- 4 files changed, 106 insertions(+), 104 deletions(-) diff --git a/frame/election-provider-multi-phase/src/benchmarking.rs b/frame/election-provider-multi-phase/src/benchmarking.rs index 4cd6bdf60a4ef..98a874967fc03 100644 --- a/frame/election-provider-multi-phase/src/benchmarking.rs +++ b/frame/election-provider-multi-phase/src/benchmarking.rs @@ -35,7 +35,7 @@ const SEED: u32 = 0; /// /// The snapshot is also created internally. fn solution_with_size( - size: SolutionSize, + size: SolutionOrSnapshotSize, active_voters_count: u32, desired_targets: u32, ) -> RawSolution> { @@ -102,9 +102,9 @@ fn solution_with_size( assert_eq!(all_voters.len() as u32, size.voters); assert_eq!(winners.len() as u32, desired_targets); - >::put(RoundSnapshotMetadata { - voters_len: all_voters.len() as u32, - targets_len: targets.len() as u32, + >::put(SolutionOrSnapshotSize { + voters: all_voters.len() as u32, + targets: targets.len() as u32, }); >::put(desired_targets); >::put(RoundSnapshot { voters: all_voters.clone(), targets: targets.clone() }); @@ -205,7 +205,7 @@ benchmarks! { // number of desired targets. Must be a subset of `t` component. let d in (T::BenchmarkingConfig::DESIRED_TARGETS[0]) .. T::BenchmarkingConfig::DESIRED_TARGETS[1]; - let witness = SolutionSize { voters: v, targets: t }; + let witness = SolutionOrSnapshotSize { voters: v, targets: t }; let raw_solution = solution_with_size::(witness, a, d); assert!(>::queued_solution().is_none()); @@ -227,7 +227,7 @@ benchmarks! { // number of desired targets. Must be a subset of `t` component. let d in (T::BenchmarkingConfig::DESIRED_TARGETS[0]) .. T::BenchmarkingConfig::DESIRED_TARGETS[1]; - let size = SolutionSize { voters: v, targets: t }; + let size = SolutionOrSnapshotSize { voters: v, targets: t }; let raw_solution = solution_with_size::(size, a, d); assert_eq!(raw_solution.compact.voter_count() as u32, a); diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index f528f37a82fbf..567b2aadb1de8 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -44,7 +44,6 @@ //! Each of the phases can be disabled by essentially setting their length to zero. If both phases //! have length zero, then the pallet essentially runs only the fallback strategy, denoted by //! [`Config::FallbackStrategy`]. -//! //! ### Signed Phase //! //! In the signed phase, solutions (of type [`RawSolution`]) are submitted and queued on chain. A @@ -137,6 +136,24 @@ //! possible, reducing solutions size/weight. The on-chain solution can use more space for accuracy, //! but should still be fast to prevent massively large blocks in case of a fallback. //! +//! ## Error types +//! +//! This pallet provides a verbose error system to ease future debugging and debugging. The +//! overall hierarchy of errors is as follows: +//! +//! 1. [`pallet::Error`]: These are the errors that can be returned in the dispatchables of the +//! pallet, either signed or unsigned. Since decomposition with nested enums is not possible +//! here, they are prefixed with the logical sub-system to which they belong. +//! 2. [`ElectionError`]: These are the errors that can be generated while the pallet is doing +//! something in automatic scenarios, such as `offchain_worker` or `on_initialize`. These errors +//! are helpful for logging and are thus nested as: +//! - [`ElectionError::Miner`]: wraps a [`unsigned::MinerError`]. +//! - [`ElectionError::Feasibility`]: wraps a [`FeasibilityError`]. +//! - [`ElectionError::OnChainFallback`]: wraps a [`sp_election_providers::onchain::Error`]. +//! +//! Note that there could be an overlap between these sub-errors. For example, A +//! `SnapshotUnavailable` can happen in both miner and feasibility check phase. +//! //! ## Future Plans //! //! **Challenge Phase**. We plan adding a third phase to the pallet, called the challenge phase. @@ -374,23 +391,6 @@ pub struct ReadySolution { compute: ElectionCompute, } -/// Size of the snapshot from which the solution was derived. -/// -/// This is needed for proper weight calculation. -#[derive(PartialEq, Eq, Clone, Copy, Encode, Decode, RuntimeDebug, Default)] -pub struct SolutionSize { - /// Number of all voters. - /// - /// This must match the on-chain snapshot. - #[codec(compact)] - voters: u32, - /// Number of all targets. - /// - /// This must match the on-chain snapshot. - #[codec(compact)] - targets: u32, -} - /// A snapshot of all the data that is needed for en entire round. They are provided by /// [`ElectionDataProvider`] and are kept around until the round is finished. /// @@ -403,40 +403,34 @@ pub struct RoundSnapshot { pub targets: Vec, } -/// Some metadata related to snapshot. +/// Encodes the length of a solution or a snapshot. /// -/// In this pallet, there are cases where we want to read the whole snapshot (voters, targets, -/// desired), and cases that we are interested in just the length of these values. The former favors -/// the snapshot to be stored in one struct (as it is now) while the latter prefers them to be -/// separate to enable the use of `decode_len`. This approach is a middle ground, storing the -/// snapshot as one struct, whilst storing the lengths separately. -#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, Default)] -pub struct RoundSnapshotMetadata { +/// This is stored automatically on-chain, and it contains the **size of the entire snapshot**. +/// This is also used in dispatchables as weight witness data and should **only contain the size of +/// the presented solution**, not the entire snapshot. +#[derive(PartialEq, Eq, Clone, Copy, Encode, Decode, RuntimeDebug, Default)] +pub struct SolutionOrSnapshotSize { /// The length of voters. - voters_len: u32, + #[codec(compact)] + voters: u32, /// The length of targets. - targets_len: u32, + #[codec(compact)] + targets: u32, } /// Internal errors of the pallet. /// /// Note that this is different from [`pallet::Error`]. -#[derive(RuntimeDebug, Eq, PartialEq)] +#[derive(Debug, Eq, PartialEq)] pub enum ElectionError { - /// A feasibility error. + /// An error happened in the feasibility check sub-system. Feasibility(FeasibilityError), + /// An error in the miner (offchain) sub-system. + Miner(unsigned::MinerError), /// An error in the on-chain fallback. OnChainFallback(onchain::Error), - /// No fallback is configured. + /// No fallback is configured. This is a special case. NoFallbackConfigured, - /// An internal error in the NPoS elections crate. - NposElections(sp_npos_elections::Error), - /// Snapshot data was unavailable unexpectedly. - SnapshotUnAvailable, - /// Submitting a transaction to the pool failed. - PoolSubmissionFailed, - /// The pre-dispatch checks failed for the mined solution. - PreDispatchChecksFailed, } impl From for ElectionError { @@ -445,20 +439,20 @@ impl From for ElectionError { } } -impl From for ElectionError { - fn from(e: sp_npos_elections::Error) -> Self { - ElectionError::NposElections(e) - } -} - impl From for ElectionError { fn from(e: FeasibilityError) -> Self { ElectionError::Feasibility(e) } } +impl From for ElectionError { + fn from(e: unsigned::MinerError) -> Self { + ElectionError::Miner(e) + } +} + /// Errors that can happen in the feasibility check. -#[derive(RuntimeDebug, Eq, PartialEq)] +#[derive(Debug, Eq, PartialEq)] pub enum FeasibilityError { /// Wrong number of winners presented. WrongWinnerCount, @@ -599,12 +593,10 @@ pub mod pallet { // We only run the OCW in the fist block of the unsigned phase. if Self::current_phase().is_unsigned_open_at(n) { match Self::try_acquire_offchain_lock(n) { - Ok(_) => match Self::mine_and_submit() { - Ok(_) => { - log!(info, "successfully submitted a solution via OCW at block {:?}", n) - } - Err(e) => log!(error, "error while submitting transaction in OCW: {:?}", e), - }, + Ok(_) => { + let outcome = Self::mine_and_submit().map_err(ElectionError::from); + log!(info, "miner exeuction done: {:?}", outcome); + } Err(why) => log!(warn, "denied offchain worker: {:?}", why), } } @@ -672,7 +664,7 @@ pub mod pallet { pub fn submit_unsigned( origin: OriginFor, solution: RawSolution>, - witness: SolutionSize, + witness: SolutionOrSnapshotSize, ) -> DispatchResultWithPostInfo { ensure_none(origin)?; let error_message = "Invalid unsigned submission must produce invalid block and \ @@ -682,12 +674,12 @@ pub mod pallet { Self::unsigned_pre_dispatch_checks(&solution).expect(error_message); // ensure witness was correct. - let RoundSnapshotMetadata { voters_len, targets_len } = + let SolutionOrSnapshotSize { voters, targets } = Self::snapshot_metadata().expect(error_message); // NOTE: we are asserting, not `ensure`ing -- we want to panic here. - assert!(voters_len as u32 == witness.voters, error_message); - assert!(targets_len as u32 == witness.targets, error_message); + assert!(voters as u32 == witness.voters, error_message); + assert!(targets as u32 == witness.targets, error_message); let ready = Self::feasibility_check(solution, ElectionCompute::Unsigned).expect(error_message); @@ -727,24 +719,15 @@ pub mod pallet { #[pallet::error] pub enum Error { /// Submission was too early. - EarlySubmission, + PreDispatchEarlySubmission, /// Wrong number of winners presented. - WrongWinnerCount, + PreDispatchWrongWinnerCount, /// Submission was too weak, score-wise. - WeakSubmission, - /// The queue was full, and the solution was not better than any of the existing ones. - QueueFull, - /// The origin failed to pay the deposit. - CannotPayDeposit, - /// witness data to dispatchable is invalid. - InvalidWitness, - /// The signed submission consumes too much weight - TooMuchWeight, + PreDispatchWeakSubmission, } #[pallet::origin] pub struct Origin(PhantomData); - #[pallet::validate_unsigned] impl ValidateUnsigned for Pallet { type Call = Call; @@ -840,7 +823,7 @@ pub mod pallet { /// Only exists when [`Snapshot`] is present. #[pallet::storage] #[pallet::getter(fn snapshot_metadata)] - pub type SnapshotMetadata = StorageValue<_, RoundSnapshotMetadata>; + pub type SnapshotMetadata = StorageValue<_, SolutionOrSnapshotSize>; #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] @@ -887,9 +870,9 @@ impl Pallet { let voters = T::DataProvider::voters(); let desired_targets = T::DataProvider::desired_targets(); - >::put(RoundSnapshotMetadata { - voters_len: voters.len() as u32, - targets_len: targets.len() as u32, + >::put(SolutionOrSnapshotSize { + voters: voters.len() as u32, + targets: targets.len() as u32, }); >::put(desired_targets); >::put(RoundSnapshot { voters, targets }); diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index 8f9c6b58d4268..402886703a1e7 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -127,9 +127,9 @@ pub fn raw_solution() -> RawSolution> { RawSolution { compact, score, round } } -pub fn witness() -> SolutionSize { +pub fn witness() -> SolutionOrSnapshotSize { TwoPhase::snapshot() - .map(|snap| SolutionSize { + .map(|snap| SolutionOrSnapshotSize { voters: snap.voters.len() as u32, targets: snap.targets.len() as u32, }) diff --git a/frame/election-provider-multi-phase/src/unsigned.rs b/frame/election-provider-multi-phase/src/unsigned.rs index 01a9fe3d6e4f0..a796986bd12c9 100644 --- a/frame/election-provider-multi-phase/src/unsigned.rs +++ b/frame/election-provider-multi-phase/src/unsigned.rs @@ -34,14 +34,32 @@ pub(crate) const OFFCHAIN_HEAD_DB: &[u8] = b"parity/multi-phase-unsigned-electio /// within a window of 5 blocks. pub(crate) const OFFCHAIN_REPEAT: u32 = 5; +#[derive(Debug, Eq, PartialEq)] +pub enum MinerError { + /// An internal error in the NPoS elections crate. + NposElections(sp_npos_elections::Error), + /// Snapshot data was unavailable unexpectedly. + SnapshotUnAvailable, + /// Submitting a transaction to the pool failed. + PoolSubmissionFailed, + /// The pre-dispatch checks failed for the mined solution. + PreDispatchChecksFailed, +} + +impl From for MinerError { + fn from(e: sp_npos_elections::Error) -> Self { + MinerError::NposElections(e) + } +} + impl Pallet { /// Mine a new npos solution. pub fn mine_solution( iters: usize, - ) -> Result<(RawSolution>, SolutionSize), ElectionError> { + ) -> Result<(RawSolution>, SolutionOrSnapshotSize), MinerError> { let RoundSnapshot { voters, targets } = - Self::snapshot().ok_or(ElectionError::SnapshotUnAvailable)?; - let desired_targets = Self::desired_targets().ok_or(ElectionError::SnapshotUnAvailable)?; + Self::snapshot().ok_or(MinerError::SnapshotUnAvailable)?; + let desired_targets = Self::desired_targets().ok_or(MinerError::SnapshotUnAvailable)?; seq_phragmen::<_, CompactAccuracyOf>( desired_targets as usize, @@ -55,7 +73,7 @@ impl Pallet { Self::unsigned_pre_dispatch_checks(&raw_solution) .map_err(|e| { log!(warn, "pre-disaptch-checks failed for mined solution: {:?}", e); - ElectionError::PreDispatchChecksFailed + MinerError::PreDispatchChecksFailed })?; Ok((raw_solution, size)) }) @@ -67,14 +85,14 @@ impl Pallet { /// Will always reduce the solution as well. pub fn prepare_election_result( election_result: ElectionResult>, - ) -> Result<(RawSolution>, SolutionSize), ElectionError> { + ) -> Result<(RawSolution>, SolutionOrSnapshotSize), MinerError> { // NOTE: This code path is generally not optimized as it is run offchain. Could use some at // some point though. // storage items. Note: we have already read this from storage, they must be in cache. let RoundSnapshot { voters, targets } = - Self::snapshot().ok_or(ElectionError::SnapshotUnAvailable)?; - let desired_targets = Self::desired_targets().ok_or(ElectionError::SnapshotUnAvailable)?; + Self::snapshot().ok_or(MinerError::SnapshotUnAvailable)?; + let desired_targets = Self::desired_targets().ok_or(MinerError::SnapshotUnAvailable)?; // closures. let cache = helpers::generate_voter_cache::(&voters); @@ -88,14 +106,15 @@ impl Pallet { // convert to staked and reduce. let mut staked = assignment_ratio_to_staked_normalized(assignments, &stake_of) - .map_err::(Into::into)?; + .map_err::(Into::into)?; sp_npos_elections::reduce(&mut staked); // convert back to ration and make compact. let ratio = assignment_staked_to_ratio_normalized(staked)?; let compact = >::from_assignment(ratio, &voter_index, &target_index)?; - let size = SolutionSize { voters: voters.len() as u32, targets: targets.len() as u32 }; + let size = + SolutionOrSnapshotSize { voters: voters.len() as u32, targets: targets.len() as u32 }; let maximum_allowed_voters = Self::maximum_voter_for_weight::( desired_targets, size, @@ -154,7 +173,7 @@ impl Pallet { maximum_allowed_voters: u32, mut compact: CompactOf, nominator_index: FN, - ) -> Result, ElectionError> + ) -> Result, MinerError> where for<'r> FN: Fn(&'r T::AccountId) -> Option>, { @@ -162,7 +181,7 @@ impl Pallet { Some(to_remove) if to_remove > 0 => { // grab all voters and sort them by least stake. let RoundSnapshot { voters, .. } = - Self::snapshot().ok_or(ElectionError::SnapshotUnAvailable)?; + Self::snapshot().ok_or(MinerError::SnapshotUnAvailable)?; let mut voters_sorted = voters .into_iter() .map(|(who, stake, _)| (who.clone(), stake)) @@ -175,7 +194,7 @@ impl Pallet { for (maybe_index, _stake) in voters_sorted.iter().map(|(who, stake)| (nominator_index(&who), stake)) { - let index = maybe_index.ok_or(ElectionError::SnapshotUnAvailable)?; + let index = maybe_index.ok_or(MinerError::SnapshotUnAvailable)?; if compact.remove_voter(index) { removed += 1 } @@ -199,7 +218,7 @@ impl Pallet { /// This only returns a value between zero and `size.nominators`. pub fn maximum_voter_for_weight( desired_winners: u32, - size: SolutionSize, + size: SolutionOrSnapshotSize, max_weight: Weight, ) -> u32 { if size.voters < 1 { @@ -306,7 +325,7 @@ impl Pallet { } /// Mine a new solution, and submit it back to the chain as an unsigned transaction. - pub(crate) fn mine_and_submit() -> Result<(), ElectionError> { + pub(crate) fn mine_and_submit() -> Result<(), MinerError> { let balancing = Self::get_balancing_iters(); let (raw_solution, witness) = Self::mine_solution(balancing)?; @@ -314,7 +333,7 @@ impl Pallet { let call = Call::submit_unsigned(raw_solution, witness).into(); SubmitTransaction::>::submit_unsigned_transaction(call) - .map_err(|_| ElectionError::PoolSubmissionFailed) + .map_err(|_| MinerError::PoolSubmissionFailed) } /// Do the basics checks that MUST happen during the validation and pre-dispatch of an unsigned @@ -328,13 +347,13 @@ impl Pallet { solution: &RawSolution>, ) -> DispatchResult { // ensure solution is timely. Don't panic yet. This is a cheap check. - ensure!(Self::current_phase().is_unsigned_open(), Error::::EarlySubmission,); + ensure!(Self::current_phase().is_unsigned_open(), Error::::PreDispatchEarlySubmission); // ensure correct number of winners. ensure!( Self::desired_targets().unwrap_or_default() == solution.compact.unique_targets().len() as u32, - Error::::WrongWinnerCount, + Error::::PreDispatchWrongWinnerCount, ); // ensure score is being improved. Panic henceforth. @@ -344,7 +363,7 @@ impl Pallet { q.score, T::SolutionImprovementThreshold::get() )), - Error::::WeakSubmission, + Error::::PreDispatchWeakSubmission, ); Ok(()) @@ -380,7 +399,7 @@ mod max_weight { #[test] fn find_max_voter_binary_search_works() { - let w = SolutionSize { voters: 10, targets: 0 }; + let w = SolutionOrSnapshotSize { voters: 10, targets: 0 }; assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 0), 0); assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1), 0); @@ -404,7 +423,7 @@ mod max_weight { assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 11_000), 10); assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 22_000), 10); - let w = SolutionSize { voters: 1, targets: 0 }; + let w = SolutionOrSnapshotSize { voters: 1, targets: 0 }; assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 0), 0); assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1), 0); @@ -418,7 +437,7 @@ mod max_weight { assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 2010), 1); assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 3333), 1); - let w = SolutionSize { voters: 2, targets: 0 }; + let w = SolutionOrSnapshotSize { voters: 2, targets: 0 }; assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 0), 0); assert_eq!(TwoPhase::maximum_voter_for_weight::(0, w, 1), 0); @@ -576,7 +595,7 @@ mod tests { #[should_panic(expected = "Invalid unsigned submission must produce invalid block and \ deprive validator from their authoring reward.: \ DispatchError::Module { index: 0, error: 1, message: \ - Some(\"WrongWinnerCount\") }")] + Some(\"PreDispatchWrongWinnerCount\") }")] fn unfeasible_solution_panics() { ExtBuilder::default().build_and_execute(|| { roll_to(25); @@ -672,7 +691,7 @@ mod tests { // mine seq_phragmen solution with 2 iters. assert_eq!( TwoPhase::mine_solution(2).unwrap_err(), - ElectionError::PreDispatchChecksFailed, + MinerError::PreDispatchChecksFailed, ); }) } @@ -720,7 +739,7 @@ mod tests { assert_eq!(solution.score[0], 12); assert_noop!( TwoPhase::unsigned_pre_dispatch_checks(&solution), - Error::::WeakSubmission, + Error::::PreDispatchWeakSubmission, ); // submitting this will actually panic. From 49613edd1df346f6a82184d7075ecc228828cdf0 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 27 Jan 2021 10:42:35 +0000 Subject: [PATCH 11/86] Some doc tests. --- Cargo.lock | 1 - bin/node/runtime/Cargo.toml | 2 -- frame/election-provider-multi-phase/src/lib.rs | 8 +++++++- frame/staking/src/lib.rs | 2 +- frame/staking/src/tests.rs | 1 - primitives/npos-elections/src/tests.rs | 1 + 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a50ab923b0f04..91af7e22e2c19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3986,7 +3986,6 @@ dependencies = [ "sp-inherents", "sp-io", "sp-keyring", - "sp-npos-elections", "sp-offchain", "sp-runtime", "sp-session", diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index e1f87cd3cc6ca..96e110ea1ec50 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -35,7 +35,6 @@ sp-keyring = { version = "2.0.0", optional = true, path = "../../../primitives/k sp-session = { version = "2.0.0", default-features = false, path = "../../../primitives/session" } sp-transaction-pool = { version = "2.0.0", default-features = false, path = "../../../primitives/transaction-pool" } sp-version = { version = "2.0.0", default-features = false, path = "../../../primitives/version" } -sp-npos-elections = { version = "2.0.0", default-features = false, path = "../../../primitives/npos-elections" } # frame dependencies frame-executive = { version = "2.0.0", default-features = false, path = "../../../frame/executive" } @@ -116,7 +115,6 @@ std = [ "pallet-im-online/std", "pallet-indices/std", "sp-inherents/std", - "sp-npos-elections/std", "pallet-lottery/std", "pallet-membership/std", "pallet-mmr/std", diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 567b2aadb1de8..236a3db8ce1c7 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -186,7 +186,8 @@ //! of doing this would be to have the fallback be another //! [`sp_election_providers::ElectionProvider`]. In this case, this pallet can even have the //! on-chain election provider as fallback, or special _noop_ fallback that simply returns an error, -//! thus replicating [`FallbackStrategy::Nothing`]. +//! thus replicating [`FallbackStrategy::Nothing`]. In this case, we won't need the additional +//! config OnChainAccuracy either. //! //! **Score based on (byte) size**: We should always prioritize small solutions over bigger ones, if //! there is a tie. Even more harsh should be to enforce the bound of the `reduce` algorithm. @@ -194,6 +195,11 @@ //! **Offchain resubmit**: Essentially port https://github.com/paritytech/substrate/pull/7976 to //! this pallet as well. The `OFFCHAIN_REPEAT` also needs to become an adjustable parameter of the //! pallet. +//! +//! **Make the number of nominators configurable from the runtime**. Remove `sp_npos_elections` +//! dependency from staking and the compact solution type. It should be generated at runtime, there +//! it should be encoded how many votes each nominators have. Essentially translate +//! https://github.com/paritytech/substrate/pull/7929 to this pallet. #![cfg_attr(not(feature = "std"), no_std)] diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index f7ea7d5f98b58..c69a680a5267f 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -3115,7 +3115,7 @@ impl Module { /// /// Returns `Err(())` if less than [`MinimumValidatorCount`] validators have been elected, `Ok` /// otherwise. - // TWO_PHASE_NOTE: the deadcode + // TWO_PHASE_NOTE: remove the dead code. #[allow(dead_code)] pub fn process_election( flat_supports: sp_npos_elections::Supports, diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 4f17d05694b88..3d90f412943af 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -1813,7 +1813,6 @@ fn bond_with_duplicate_vote_should_be_ignored_by_npos_election() { .minimum_validator_count(1) .build() .execute_with(|| { - // disable the nominator assert_ok!(Staking::chill(Origin::signed(100))); // make stakes equal. diff --git a/primitives/npos-elections/src/tests.rs b/primitives/npos-elections/src/tests.rs index bc148f118ce42..edfea038ebc50 100644 --- a/primitives/npos-elections/src/tests.rs +++ b/primitives/npos-elections/src/tests.rs @@ -1149,6 +1149,7 @@ mod solution_type { type TestAccuracy = Percent; generate_solution_type!(pub struct TestSolutionCompact::(16)); + #[allow(dead_code)] mod __private { // This is just to make sure that that the compact can be generated in a scope without any From 35f1fafe91026c73803a1f2a3349e6d559b54041 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 27 Jan 2021 12:24:25 +0000 Subject: [PATCH 12/86] Some changes from other PR --- .../src/unsigned.rs | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/frame/election-provider-multi-phase/src/unsigned.rs b/frame/election-provider-multi-phase/src/unsigned.rs index a796986bd12c9..d0b7400cb0f2b 100644 --- a/frame/election-provider-multi-phase/src/unsigned.rs +++ b/frame/election-provider-multi-phase/src/unsigned.rs @@ -53,6 +53,24 @@ impl From for MinerError { } impl Pallet { + /// Mine a new solution, and submit it back to the chain as an unsigned transaction. + pub(crate) fn mine_and_submit() -> Result<(), MinerError> { + let balancing = Self::get_balancing_iters(); + let (raw_solution, witness) = Self::mine_solution(balancing)?; + + // ensure that this will pass the pre-dispatch checks + Self::unsigned_pre_dispatch_checks(&raw_solution).map_err(|e| { + log!(warn, "pre-disaptch-checks failed for mined solution: {:?}", e); + MinerError::PreDispatchChecksFailed + })?; + + // submit the raw solution to the pool. + let call = Call::submit_unsigned(raw_solution, witness).into(); + + SubmitTransaction::>::submit_unsigned_transaction(call) + .map_err(|_| MinerError::PoolSubmissionFailed) + } + /// Mine a new npos solution. pub fn mine_solution( iters: usize, @@ -69,14 +87,6 @@ impl Pallet { ) .map_err(Into::into) .and_then(Self::prepare_election_result) - .and_then(|(raw_solution, size)| { - Self::unsigned_pre_dispatch_checks(&raw_solution) - .map_err(|e| { - log!(warn, "pre-disaptch-checks failed for mined solution: {:?}", e); - MinerError::PreDispatchChecksFailed - })?; - Ok((raw_solution, size)) - }) } /// Convert a raw solution from [`sp_npos_elections::ElectionResult`] to [`RawSolution`], which @@ -324,18 +334,6 @@ impl Pallet { } } - /// Mine a new solution, and submit it back to the chain as an unsigned transaction. - pub(crate) fn mine_and_submit() -> Result<(), MinerError> { - let balancing = Self::get_balancing_iters(); - let (raw_solution, witness) = Self::mine_solution(balancing)?; - - // submit the raw solution to the pool. - let call = Call::submit_unsigned(raw_solution, witness).into(); - - SubmitTransaction::>::submit_unsigned_transaction(call) - .map_err(|_| MinerError::PoolSubmissionFailed) - } - /// Do the basics checks that MUST happen during the validation and pre-dispatch of an unsigned /// transaction. /// @@ -684,13 +682,14 @@ mod tests { #[test] fn miner_will_not_submit_if_not_enough_winners() { - ExtBuilder::default().desired_targets(8).build_and_execute(|| { + let (mut ext, _) = ExtBuilder::default().desired_targets(8).build_offchainify(0); + ext.execute_with(|| { roll_to(25); assert!(TwoPhase::current_phase().is_unsigned()); // mine seq_phragmen solution with 2 iters. assert_eq!( - TwoPhase::mine_solution(2).unwrap_err(), + TwoPhase::mine_and_submit().unwrap_err(), MinerError::PreDispatchChecksFailed, ); }) From e6ffc4da84c6c8a454e68f5e1cbf31ede1a6004e Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 27 Jan 2021 13:04:31 +0000 Subject: [PATCH 13/86] Fix session test --- frame/session/src/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/session/src/tests.rs b/frame/session/src/tests.rs index 7c1d3c9dcdd24..3a665a5cbea17 100644 --- a/frame/session/src/tests.rs +++ b/frame/session/src/tests.rs @@ -274,7 +274,7 @@ fn periodic_session_works() { } assert!(P::should_end_session(13u64)); - assert_eq!(P::estimate_next_session_rotation(13u64).unwrap(), 13); + assert_eq!(P::estimate_next_session_rotation(13u64).unwrap(), 23); assert!(!P::should_end_session(14u64)); assert_eq!(P::estimate_next_session_rotation(14u64).unwrap(), 23); From 54460ebb1005d2275b7016b5e399fb7269e7be86 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Wed, 3 Mar 2021 15:05:46 +0000 Subject: [PATCH 14/86] Update bin/node/runtime/src/lib.rs Co-authored-by: Peter Goodspeed-Niklaus --- bin/node/runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 0170d493e6c3e..cda7397394caf 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -548,7 +548,7 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type SignedDepositWeight = (); type SignedMaxWeight = MinerMaxWeight; type SlashHandler = (); // burn slashes - type RewardHandler = (); // nothing todo upon rewards/ + type RewardHandler = (); // nothing to do upon rewards type DataProvider = Staking; type OnChainAccuracy = Perbill; type CompactSolution = pallet_staking::CompactAssignments; From 33b8e6911f09ac7c195a6e4c5f97a57b2e343132 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 3 Mar 2021 16:30:27 +0100 Subject: [PATCH 15/86] Fix name. --- bin/node/runtime/src/lib.rs | 4 ++-- frame/election-provider-multi-phase/src/benchmarking.rs | 4 ++-- frame/election-provider-multi-phase/src/helpers.rs | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 0170d493e6c3e..7fe06a7b0e236 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -522,7 +522,7 @@ parameter_types! { pub SolutionImprovementThreshold: Perbill = Perbill::from_rational_approximation(1u32, 10_000); // miner configs - pub const TwoPhaseUnsignedPriority: TransactionPriority = StakingUnsignedPriority::get() - 1u64; + pub const MultiPhaseUnsignedPriority: TransactionPriority = StakingUnsignedPriority::get() - 1u64; pub const MinerMaxIterations: u32 = 10; pub MinerMaxWeight: Weight = RuntimeBlockWeights::get() .get(DispatchClass::Normal) @@ -538,7 +538,7 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type SolutionImprovementThreshold = MinSolutionScoreBump; type MinerMaxIterations = MinerMaxIterations; type MinerMaxWeight = MinerMaxWeight; - type MinerTxPriority = TwoPhaseUnsignedPriority; + type MinerTxPriority = MultiPhaseUnsignedPriority; type SignedMaxSubmissions = SignedMaxSubmissions; type SignedRewardBase = SignedRewardBase; type SignedRewardFactor = (); // no score-based reward diff --git a/frame/election-provider-multi-phase/src/benchmarking.rs b/frame/election-provider-multi-phase/src/benchmarking.rs index 50899a3c7813e..1b500a5889f9a 100644 --- a/frame/election-provider-multi-phase/src/benchmarking.rs +++ b/frame/election-provider-multi-phase/src/benchmarking.rs @@ -29,7 +29,7 @@ use sp_arithmetic::traits::One; use sp_runtime::InnerOf; use sp_std::convert::TryInto; -const SEED: u32 = 0; +const SEED: u32 = 999; /// Creates a **valid** solution with exactly the given size. /// @@ -57,7 +57,7 @@ fn solution_with_size( let targets: Vec = (0..size.targets).map(|i| account("Targets", i, SEED)).collect(); - let mut rng = SmallRng::seed_from_u64(999u64); + let mut rng = SmallRng::seed_from_u64(SEED.into()); // decide who are the winners. let winners = targets diff --git a/frame/election-provider-multi-phase/src/helpers.rs b/frame/election-provider-multi-phase/src/helpers.rs index 7375ce017f20b..a14fa8ccc8607 100644 --- a/frame/election-provider-multi-phase/src/helpers.rs +++ b/frame/election-provider-multi-phase/src/helpers.rs @@ -47,13 +47,13 @@ pub fn generate_voter_cache( cache } -/// Create a function the returns the index a voter in the snapshot. +/// Create a function that returns the index of a voter in the snapshot. /// /// The returning index type is the same as the one defined in [`T::CompactSolution::Voter`]. /// /// ## Warning /// -/// The snapshot must be the same is the one used to create `cache`. +/// Note that this will represent the snapshot data from which the `cache` is generated. pub fn voter_index_fn( cache: &BTreeMap, ) -> Box Option> + '_> { @@ -66,7 +66,7 @@ pub fn voter_index_fn( /// /// ## Warning /// -/// The snapshot must be the same is the one used to create `cache`. +/// Note that this will represent the snapshot data from which the `cache` is generated. pub fn voter_index_fn_usize( cache: &BTreeMap, ) -> Box Option + '_> { @@ -90,7 +90,7 @@ pub fn voter_index_fn_linear( }) } -/// Create a function the returns the index a targets in the snapshot. +/// Create a function that returns the index of a targets in the snapshot. /// /// The returning index type is the same as the one defined in [`T::CompactSolution::Target`]. pub fn target_index_fn_linear( From 62f8ce4eefc87e010f6e900887db12fd794fbdc2 Mon Sep 17 00:00:00 2001 From: Parity Benchmarking Bot Date: Wed, 3 Mar 2021 17:01:43 +0000 Subject: [PATCH 16/86] cargo run --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_election_provider_multi_phase --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/election-provider-multi-phase/src/weights.rs --template=./.maintain/frame-weight-template.hbs --- .../src/weights.rs | 233 ++++++++++++++---- 1 file changed, 180 insertions(+), 53 deletions(-) diff --git a/frame/election-provider-multi-phase/src/weights.rs b/frame/election-provider-multi-phase/src/weights.rs index faee884900811..e62f329800caa 100644 --- a/frame/election-provider-multi-phase/src/weights.rs +++ b/frame/election-provider-multi-phase/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_election_provider_multi_phase //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0 -//! DATE: 2021-02-12, STEPS: [50, ], REPEAT: 20, LOW RANGE: [], HIGH RANGE: [] +//! DATE: 2021-03-03, STEPS: [50, ], REPEAT: 20, LOW RANGE: [], HIGH RANGE: [] //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -47,10 +47,19 @@ pub trait WeightInfo { fn on_initialize_nothing() -> Weight; fn on_initialize_open_signed() -> Weight; fn on_initialize_open_unsigned_with_snapshot() -> Weight; + fn finalize_signed_phase_accept_solution() -> Weight; + fn finalize_signed_phase_reject_solution() -> Weight; fn on_initialize_open_unsigned_without_snapshot() -> Weight; + fn submit(c: u32, ) -> Weight; + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight; + fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; + fn on_initialize_nothing() -> Weight; + fn on_initialize_open_signed() -> Weight; + fn on_initialize_open_unsigned_with_snapshot() -> Weight; fn finalize_signed_phase_accept_solution() -> Weight; fn finalize_signed_phase_reject_solution() -> Weight; - fn submit(c: u32) -> Weight; + fn on_initialize_open_unsigned_without_snapshot() -> Weight; + fn submit(c: u32, ) -> Weight; fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight; fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; } @@ -59,64 +68,123 @@ pub trait WeightInfo { pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { fn on_initialize_nothing() -> Weight { - (23_401_000 as Weight) + (24_685_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) } fn on_initialize_open_signed() -> Weight { - (79_260_000 as Weight) + (81_199_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } fn on_initialize_open_unsigned_with_snapshot() -> Weight { - (77_745_000 as Weight) + (79_681_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } + fn finalize_signed_phase_accept_solution() -> Weight { + (47_707_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + fn finalize_signed_phase_reject_solution() -> Weight { + (21_346_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } fn on_initialize_open_unsigned_without_snapshot() -> Weight { - (21_764_000 as Weight) + (21_756_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + fn submit(c: u32, ) -> Weight { + (84_430_000 as Weight) + // Standard Error: 146_000 + .saturating_add((2_758_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 21_000 + .saturating_add((4_090_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 73_000 + .saturating_add((82_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 21_000 + .saturating_add((13_669_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 109_000 + .saturating_add((4_637_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(6 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 12_000 + .saturating_add((4_160_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 40_000 + .saturating_add((567_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 12_000 + .saturating_add((10_362_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 61_000 + .saturating_add((4_521_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + } + fn on_initialize_nothing() -> Weight { + (24_128_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + } + fn on_initialize_open_signed() -> Weight { + (80_951_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + fn on_initialize_open_unsigned_with_snapshot() -> Weight { + (79_888_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } fn finalize_signed_phase_accept_solution() -> Weight { - (38_088_000 as Weight) + (47_783_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } fn finalize_signed_phase_reject_solution() -> Weight { - (17_124_000 as Weight) + (21_277_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn on_initialize_open_unsigned_without_snapshot() -> Weight { + (21_682_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } - fn submit(c: u32) -> Weight { - (52_370_000 as Weight) - // Standard Error: 17_000 - .saturating_add((3_395_000 as Weight).saturating_mul(c as Weight)) + fn submit(c: u32, ) -> Weight { + (78_781_000 as Weight) + // Standard Error: 85_000 + .saturating_add((3_625_000 as Weight).saturating_mul(c as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } - fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 23_000 - .saturating_add((4_171_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 78_000 - .saturating_add((229_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 23_000 - .saturating_add((13_661_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 117_000 - .saturating_add((4_499_000 as Weight).saturating_mul(d as Weight)) + // Standard Error: 21_000 + .saturating_add((3_990_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 21_000 + .saturating_add((13_674_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 107_000 + .saturating_add((4_899_000 as Weight).saturating_mul(d as Weight)) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) // Standard Error: 12_000 - .saturating_add((4_232_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 42_000 - .saturating_add((636_000 as Weight).saturating_mul(t as Weight)) + .saturating_add((4_127_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 41_000 + .saturating_add((477_000 as Weight).saturating_mul(t as Weight)) // Standard Error: 12_000 - .saturating_add((10_294_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 64_000 - .saturating_add((4_428_000 as Weight).saturating_mul(d as Weight)) + .saturating_add((10_359_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 62_000 + .saturating_add((4_745_000 as Weight).saturating_mul(d as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) } } @@ -124,64 +192,123 @@ impl WeightInfo for SubstrateWeight { // For backwards compatibility and tests impl WeightInfo for () { fn on_initialize_nothing() -> Weight { - (23_401_000 as Weight) + (24_685_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) } fn on_initialize_open_signed() -> Weight { - (79_260_000 as Weight) + (81_199_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } fn on_initialize_open_unsigned_with_snapshot() -> Weight { - (77_745_000 as Weight) + (79_681_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } + fn finalize_signed_phase_accept_solution() -> Weight { + (47_707_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + fn finalize_signed_phase_reject_solution() -> Weight { + (21_346_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } fn on_initialize_open_unsigned_without_snapshot() -> Weight { - (21_764_000 as Weight) + (21_756_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + fn submit(c: u32, ) -> Weight { + (84_430_000 as Weight) + // Standard Error: 146_000 + .saturating_add((2_758_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 21_000 + .saturating_add((4_090_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 73_000 + .saturating_add((82_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 21_000 + .saturating_add((13_669_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 109_000 + .saturating_add((4_637_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(6 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 12_000 + .saturating_add((4_160_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 40_000 + .saturating_add((567_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 12_000 + .saturating_add((10_362_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 61_000 + .saturating_add((4_521_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + } + fn on_initialize_nothing() -> Weight { + (24_128_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + } + fn on_initialize_open_signed() -> Weight { + (80_951_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + fn on_initialize_open_unsigned_with_snapshot() -> Weight { + (79_888_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } fn finalize_signed_phase_accept_solution() -> Weight { - (38_088_000 as Weight) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + (47_783_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } fn finalize_signed_phase_reject_solution() -> Weight { - (17_124_000 as Weight) + (21_277_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } - fn submit(c: u32) -> Weight { - (52_370_000 as Weight) - // Standard Error: 17_000 - .saturating_add((3_395_000 as Weight).saturating_mul(c as Weight)) + fn on_initialize_open_unsigned_without_snapshot() -> Weight { + (21_682_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn submit(c: u32, ) -> Weight { + (78_781_000 as Weight) + // Standard Error: 85_000 + .saturating_add((3_625_000 as Weight).saturating_mul(c as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } - fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 23_000 - .saturating_add((4_171_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 78_000 - .saturating_add((229_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 23_000 - .saturating_add((13_661_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 117_000 - .saturating_add((4_499_000 as Weight).saturating_mul(d as Weight)) + // Standard Error: 21_000 + .saturating_add((3_990_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 21_000 + .saturating_add((13_674_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 107_000 + .saturating_add((4_899_000 as Weight).saturating_mul(d as Weight)) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) // Standard Error: 12_000 - .saturating_add((4_232_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 42_000 - .saturating_add((636_000 as Weight).saturating_mul(t as Weight)) + .saturating_add((4_127_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 41_000 + .saturating_add((477_000 as Weight).saturating_mul(t as Weight)) // Standard Error: 12_000 - .saturating_add((10_294_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 64_000 - .saturating_add((4_428_000 as Weight).saturating_mul(d as Weight)) + .saturating_add((10_359_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 62_000 + .saturating_add((4_745_000 as Weight).saturating_mul(d as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) } } From dff422aac9d68484054575b7b28bd8063aa9ad17 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 30 Mar 2021 14:58:28 +0200 Subject: [PATCH 17/86] typos and verbiage --- frame/election-provider-multi-phase/src/lib.rs | 2 +- frame/election-provider-multi-phase/src/signed.rs | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 29c5be48177b8..3342da84658bc 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -527,7 +527,7 @@ pub mod pallet { /// this values, based on [`WeightInfo::submit_unsigned`]. type MinerMaxWeight: Get; - /// Maximum number of singed submissions that can be queued. + /// Maximum number of signed submissions that can be queued. #[pallet::constant] type SignedMaxSubmissions: Get; /// Maximum weight of a signed solution. diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 56d53e699bc76..6655dad9b0a66 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -48,7 +48,7 @@ pub(crate) type NegativeImbalanceOf = <::Currency as Currency< >>::NegativeImbalance; impl Pallet { - /// Finish the singed phase. Process the signed submissions from best to worse until a valid one + /// Finish the signed phase. Process the signed submissions from best to worse until a valid one /// is found, rewarding the best one and slashing the invalid ones along the way. /// /// Returns true if we have a good solution in the signed phase. @@ -68,11 +68,12 @@ impl Pallet { let SolutionOrSnapshotSize { voters, targets } = Self::snapshot_metadata().unwrap_or_default(); let desired_targets = Self::desired_targets().unwrap_or_default(); - let v = voters; - let t = targets; - let a = active_voters; - let w = desired_targets; - T::WeightInfo::feasibility_check(v, t, a, w) + T::WeightInfo::feasibility_check( + voters, + targets, + active_voters, + desired_targets, + ) }; match Self::feasibility_check(solution, ElectionCompute::Signed) { Ok(ready_solution) => { @@ -99,7 +100,7 @@ impl Pallet { } } - // Any unprocessed solution is not pointless to even ponder upon. Feasible or malicious, + // Any unprocessed solution is pointless to even consider. Feasible or malicious, // they didn't end up being used. Unreserve the bonds. all_submission.into_iter().for_each(|not_processed| { let SignedSubmission { who, deposit, .. } = not_processed; From c58b2f9910cc06546ad1f9ce5d573df8b9acc3d6 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 30 Mar 2021 15:48:57 +0200 Subject: [PATCH 18/86] no glob imports in signed.rs --- .../election-provider-multi-phase/src/signed.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 6655dad9b0a66..bc222a2da0952 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -17,11 +17,15 @@ //! The signed phase implementation. -use super::*; -use codec::{Encode, HasCompact}; +use crate::{ + CompactOf, Config, ElectionCompute, Pallet, RawSolution, ReadySolution, SolutionOrSnapshotSize, + Weight, WeightInfo, QueuedSolution, SignedSubmissions, +}; +use codec::{Encode, Decode, HasCompact}; +use frame_support::traits::{Currency, Get, OnUnbalanced, ReservableCurrency}; use sp_arithmetic::traits::SaturatedConversion; use sp_npos_elections::{is_score_better, CompactSolution}; -use sp_runtime::Perbill; +use sp_runtime::{Perbill, RuntimeDebug, traits::Zero}; /// A raw, unchecked signed submission. /// @@ -69,9 +73,9 @@ impl Pallet { Self::snapshot_metadata().unwrap_or_default(); let desired_targets = Self::desired_targets().unwrap_or_default(); T::WeightInfo::feasibility_check( - voters, - targets, - active_voters, + voters, + targets, + active_voters, desired_targets, ) }; From 76c3387ce319d6c938a2932cac7be436a3130e4b Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 30 Mar 2021 15:54:00 +0200 Subject: [PATCH 19/86] meaningful generic type parameters for SignedSubmission --- frame/election-provider-multi-phase/src/signed.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index bc222a2da0952..4fb61dfefb7ec 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -31,15 +31,15 @@ use sp_runtime::{Perbill, RuntimeDebug, traits::Zero}; /// /// This is just a wrapper around [`RawSolution`] and some additional info. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, Default)] -pub struct SignedSubmission { +pub struct SignedSubmission { /// Who submitted this solution. - pub(crate) who: A, + pub(crate) who: AccountId, /// The deposit reserved for storing this solution. - pub(crate) deposit: B, + pub(crate) deposit: Balance, /// The reward that should be given to this solution, if chosen the as the final one. - pub(crate) reward: B, + pub(crate) reward: Balance, /// The raw solution itself. - pub(crate) solution: RawSolution, + pub(crate) solution: RawSolution, } pub(crate) type BalanceOf = From 872a3c8acc197c07b3a3ac2ec53c80e52cfbb936 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 30 Mar 2021 16:02:43 +0200 Subject: [PATCH 20/86] dedup feasibility check weight calculation --- frame/election-provider-multi-phase/src/signed.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 4fb61dfefb7ec..ecac85052e1be 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -79,6 +79,8 @@ impl Pallet { desired_targets, ) }; + // the feasibility check itself has some weight + weight = weight.saturating_add(feasibility_weight); match Self::feasibility_check(solution, ElectionCompute::Signed) { Ok(ready_solution) => { Self::finalize_signed_phase_accept_solution( @@ -89,14 +91,11 @@ impl Pallet { ); found_solution = true; - weight = weight.saturating_add(feasibility_weight); weight = weight .saturating_add(T::WeightInfo::finalize_signed_phase_accept_solution()); break; } Err(_) => { - // we assume a worse case feasibility check happened anyhow. - weight = weight.saturating_add(feasibility_weight); Self::finalize_signed_phase_reject_solution(&who, deposit); weight = weight .saturating_add(T::WeightInfo::finalize_signed_phase_reject_solution()); From f5f4605207977b1d4eae40b97ff5fd4bf3096e6a Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 30 Mar 2021 16:56:08 +0200 Subject: [PATCH 21/86] simplify/optimize fn insert_submission --- .../src/signed.rs | 81 ++++++++++--------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index ecac85052e1be..da723b99cb875 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -26,6 +26,7 @@ use frame_support::traits::{Currency, Get, OnUnbalanced, ReservableCurrency}; use sp_arithmetic::traits::SaturatedConversion; use sp_npos_elections::{is_score_better, CompactSolution}; use sp_runtime::{Perbill, RuntimeDebug, traits::Zero}; +use sp_std::cmp::Ordering; /// A raw, unchecked signed submission. /// @@ -149,10 +150,14 @@ impl Pallet { T::SlashHandler::on_unbalanced(negative_imbalance); } - /// Find a proper position in the queue for the signed queue, whilst maintaining the order of - /// solution quality. If insertion was successful, `Some(index)` is returned where index is the + /// Insert a solution into the queue while maintaining an ordering by solution quality. + /// + /// If insertion was successful, `Some(index)` is returned where index is the /// index of the newly inserted item. /// + /// Note: this function does _not_ maintain the invariant that `queue.len() <= T::SignedMaxSubmissions`. + /// Pruning must happen elsewhere. + /// /// Invariant: The returned index is always a valid index in `queue` and can safely be used to /// inspect the newly inserted element. pub fn insert_submission( @@ -161,45 +166,41 @@ impl Pallet { solution: RawSolution>, size: SolutionOrSnapshotSize, ) -> Option { - // from the last score, compare and see if the current one is better. If none, then the - // awarded index is 0. - let outcome = queue - .iter() - .enumerate() - .rev() - .find_map(|(i, s)| { - if is_score_better::( - solution.score, - s.solution.score, - T::SolutionImprovementThreshold::get(), - ) { - Some(i + 1) - } else { - None - } - }) - .or(Some(0)) - .and_then(|at| { - if at == 0 && queue.len() as u32 >= T::SignedMaxSubmissions::get() { - // if this is worse than all, and the queue is full, don't bother. - None - } else { - // add to the designated spot. If the length is too much, remove one. - let reward = Self::reward_for(&solution); - let deposit = Self::deposit_for(&solution, size); - let submission = - SignedSubmission { who: who.clone(), deposit, reward, solution }; - // Proof: `at` must always less than or equal queue.len() for this not to panic. - // It is either 0 (in which case `0 <= queue.len()`) or one of the queue indices - // + 1. The biggest queue index is `queue.len() - 1`, thus `at <= queue.len()`. - queue.insert(at, submission); - Some(at) - } - }); + let threshold = T::SolutionImprovementThreshold::get(); + // this insertion logic is a bit unusual in that a new solution which beats an existing + // solution by less than the threshold is sorted as "less" than the existing solution. + // this means that the produced ordering depends on the order of insertion, and that + // attempts to produce a total ordering using this comparitor are highly unstable. + // + // this ordering prioritizes earlier solutions over slightly better later ones. + let insertion_position = queue.binary_search_by(|s| { + if is_score_better::( + solution.score, + s.solution.score, + threshold, + ) { + Ordering::Greater + } else { + Ordering::Less + } + }).expect_err("comparitor function never returns Ordering::Equal; qed"); + + let max_submissions = T::SignedMaxSubmissions::get(); + // if this solution is the worst one so far and the queue is full, don't insert + if insertion_position == 0 && queue.len() as u32 >= max_submissions { + return None; + } + + // add to the designated spot. If the length is too much, remove one. + let reward = Self::reward_for(&solution); + let deposit = Self::deposit_for(&solution, size); + let submission = + SignedSubmission { who: who.clone(), deposit, reward, solution }; + queue.insert(insertion_position, submission); - // If the call site is sane and removes the weakest, then this must always be correct. - debug_assert!(queue.len() as u32 <= T::SignedMaxSubmissions::get() + 1); - outcome + // If the queue was within length when this function was called, then this must always be correct. + debug_assert!(queue.len() as u32 <= max_submissions + 1); + Some(insertion_position) } /// Removes the weakest element of the queue, namely the first one, should the length of the From 859f59499d0e407f0541b65c3219f7d04db5737b Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 30 Mar 2021 17:25:46 +0200 Subject: [PATCH 22/86] tests: remove glob, cause to build without error --- .../src/benchmarking.rs | 2 +- frame/election-provider-multi-phase/src/lib.rs | 18 ++++++++++++++++-- .../election-provider-multi-phase/src/mock.rs | 2 ++ .../src/signed.rs | 14 +++++++++----- .../src/unsigned.rs | 14 +++++++++----- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/frame/election-provider-multi-phase/src/benchmarking.rs b/frame/election-provider-multi-phase/src/benchmarking.rs index de17390d46870..a844740ef2995 100644 --- a/frame/election-provider-multi-phase/src/benchmarking.rs +++ b/frame/election-provider-multi-phase/src/benchmarking.rs @@ -19,7 +19,7 @@ use super::*; use crate::Pallet as MultiPhase; -use frame_benchmarking::impl_benchmark_test_suite; +use frame_benchmarking::{account, impl_benchmark_test_suite}; use frame_support::{assert_ok, traits::OnInitialize}; use frame_system::RawOrigin; use rand::{prelude::SliceRandom, rngs::SmallRng, SeedableRng}; diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 23db95218bf1c..0ae85596319d8 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -1262,7 +1262,14 @@ mod feasibility_check { //! more. The best way to audit and review these tests is to try and come up with a solution //! that is invalid, but gets through the system as valid. - use super::{mock::*, *}; + use super::*; + use crate::{ + mock::{ + MultiPhase, Runtime, roll_to, TargetIndex, raw_solution, EpochLength, UnsignedPhase, + SignedPhase, VoterIndex, ExtBuilder, + }, + }; + use frame_support::assert_noop; const COMPUTE: ElectionCompute = ElectionCompute::OnChain; @@ -1429,7 +1436,14 @@ mod feasibility_check { #[cfg(test)] mod tests { - use super::{mock::*, Event, *}; + use super::*; + use crate::{ + Phase, + mock::{ + ExtBuilder, MultiPhase, Runtime, roll_to, MockWeightInfo, AccountId, TargetIndex, + Targets, multi_phase_events, System, + }, + }; use frame_election_provider_support::ElectionProvider; use sp_npos_elections::Support; diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index 0d9c4ceeee0ac..d32adfd874958 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -266,6 +266,8 @@ impl multi_phase::weights::WeightInfo for DualMockWeightInfo { Zero::zero() } else { <() as multi_phase::weights::WeightInfo>::submit(c) + } + } fn elect_queued() -> Weight { if MockWeightInfo::get() { Zero::zero() diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index da723b99cb875..0b020540ac991 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -262,11 +262,15 @@ impl Pallet { #[cfg(test)] mod tests { - use super::{ - mock::{Origin, *}, - Error, Phase, *, + use super::*; + use crate::{ + Phase, Error, + mock::{ + balances, ExtBuilder, MultiPhase, Origin, raw_solution, roll_to, Runtime, + SignedMaxSubmissions, SignedMaxWeight, + }, }; - use frame_support::dispatch::DispatchResultWithPostInfo; + use frame_support::{dispatch::DispatchResultWithPostInfo, assert_noop, assert_ok}; fn submit_with_witness( origin: Origin, @@ -282,7 +286,7 @@ mod tests { assert_eq!(MultiPhase::current_phase(), Phase::Off); // create a temp snapshot only for this test. - MultiPhase::create_snapshot(); + MultiPhase::create_snapshot().unwrap(); let solution = raw_solution(); assert_noop!( diff --git a/frame/election-provider-multi-phase/src/unsigned.rs b/frame/election-provider-multi-phase/src/unsigned.rs index 492f68ec6d043..e0479bd0b17fa 100644 --- a/frame/election-provider-multi-phase/src/unsigned.rs +++ b/frame/election-provider-multi-phase/src/unsigned.rs @@ -61,7 +61,6 @@ impl From for MinerError { } } - impl Pallet { /// Mine a new solution, and submit it back to the chain as an unsigned transaction. pub fn mine_check_and_submit() -> Result<(), MinerError> { @@ -512,11 +511,16 @@ mod max_weight { #[cfg(test)] mod tests { - use super::{ - mock::{Origin, *}, - Call, *, + use super::*; + use crate::{ + CurrentPhase, InvalidTransaction, Phase, QueuedSolution, TransactionSource, + TransactionValidityError, + mock::{ + ExtBuilder, Extrinsic, MinerMaxWeight, MultiPhase, Origin, Runtime, TestCompact, + roll_to, roll_to_with_ocw, witness, + }, }; - use frame_support::{dispatch::Dispatchable, traits::OffchainWorker}; + use frame_support::{assert_noop, assert_ok, dispatch::Dispatchable, traits::OffchainWorker}; use mock::Call as OuterCall; use frame_election_provider_support::Assignment; use sp_runtime::{traits::ValidateUnsigned, PerU16}; From 0fc7a5800bdff3acb414cb2eff8c72e7ace37732 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 31 Mar 2021 09:08:29 +0200 Subject: [PATCH 23/86] use sp_std::vec::Vec --- frame/election-provider-multi-phase/src/signed.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 0b020540ac991..0ed3183168f61 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -26,7 +26,7 @@ use frame_support::traits::{Currency, Get, OnUnbalanced, ReservableCurrency}; use sp_arithmetic::traits::SaturatedConversion; use sp_npos_elections::{is_score_better, CompactSolution}; use sp_runtime::{Perbill, RuntimeDebug, traits::Zero}; -use sp_std::cmp::Ordering; +use sp_std::{cmp::Ordering, vec::Vec}; /// A raw, unchecked signed submission. /// From 1137698533cf60822506060b7920aa7ef754d630 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 31 Mar 2021 09:46:46 +0200 Subject: [PATCH 24/86] maintain invariant within fn insert_submission --- .../election-provider-multi-phase/src/lib.rs | 7 ------- .../src/signed.rs | 20 ++++++++++++++----- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 0ae85596319d8..d6378e41fade8 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -766,7 +766,6 @@ pub mod pallet { // defensive-only: if phase is signed, snapshot will exist. let size = Self::snapshot_metadata().unwrap_or_default(); - // NOTE: we compute this function once in `insert_submission` as well, could optimize. ensure!( Self::feasibility_weight_of(&solution, size) < T::SignedMaxWeight::get(), Error::::SignedTooMuchWeight, @@ -782,12 +781,6 @@ pub mod pallet { let deposit = signed_submissions.get(index).map(|s| s.deposit).unwrap_or_default(); T::Currency::reserve(&who, deposit).map_err(|_| Error::::SignedCannotPayDeposit)?; - // Remove the weakest, if needed. - if signed_submissions.len() as u32 > T::SignedMaxSubmissions::get() { - Self::remove_weakest(&mut signed_submissions); - } - debug_assert!(signed_submissions.len() as u32 <= T::SignedMaxSubmissions::get()); - log!( info, "queued signed solution with (claimed) score {:?}", diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 0ed3183168f61..3db422d064173 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -155,8 +155,8 @@ impl Pallet { /// If insertion was successful, `Some(index)` is returned where index is the /// index of the newly inserted item. /// - /// Note: this function does _not_ maintain the invariant that `queue.len() <= T::SignedMaxSubmissions`. - /// Pruning must happen elsewhere. + /// Note: this function maintains the invariant that `queue.len() <= T::SignedMaxSubmissions`. + /// In the event that insertion would violate that invariant, the weakest element is dropped. /// /// Invariant: The returned index is always a valid index in `queue` and can safely be used to /// inspect the newly inserted element. @@ -173,7 +173,7 @@ impl Pallet { // attempts to produce a total ordering using this comparitor are highly unstable. // // this ordering prioritizes earlier solutions over slightly better later ones. - let insertion_position = queue.binary_search_by(|s| { + let mut insertion_position = queue.binary_search_by(|s| { if is_score_better::( solution.score, s.solution.score, @@ -200,6 +200,16 @@ impl Pallet { // If the queue was within length when this function was called, then this must always be correct. debug_assert!(queue.len() as u32 <= max_submissions + 1); + + // Remove the weakest, if needed. + if queue.len() as u32 > max_submissions { + Self::remove_weakest(queue); + insertion_position -= 1; + // this is sound because remove_weakest always removes the 0th item. + // if `insertion_position` was 0 and this could potentially have triggered, we've + // already short-circuited above. + } + debug_assert!(queue.len() as u32 <= max_submissions); Some(insertion_position) } @@ -207,7 +217,7 @@ impl Pallet { /// queue be enough. /// /// noop if the queue is empty. Bond of the removed solution is returned. - pub fn remove_weakest( + fn remove_weakest( queue: &mut Vec, CompactOf>>, ) { if queue.len() > 0 { @@ -625,7 +635,7 @@ mod tests { } #[test] - fn all_in_one_singed_submission_scenario() { + fn all_in_one_signed_submission_scenario() { // a combination of: // - good_solution_is_rewarded // - bad_solution_is_slashed From 8c26900da11bd3a67df3abe4cd32c5debc56562b Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 31 Mar 2021 10:00:22 +0200 Subject: [PATCH 25/86] fix accidentally ordering the list backward --- frame/election-provider-multi-phase/src/signed.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 3db422d064173..961858794eb6f 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -179,9 +179,9 @@ impl Pallet { s.solution.score, threshold, ) { - Ordering::Greater - } else { Ordering::Less + } else { + Ordering::Greater } }).expect_err("comparitor function never returns Ordering::Equal; qed"); From a882ff52cd8e625513d3699089da3260fd4c0cb8 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 31 Mar 2021 10:50:15 +0200 Subject: [PATCH 26/86] intentionally order the list in reverse --- .../src/signed.rs | 84 +++++++++---------- 1 file changed, 39 insertions(+), 45 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 961858794eb6f..04840c687e9cc 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -61,11 +61,16 @@ impl Pallet { /// This drains the [`SignedSubmissions`], potentially storing the best valid one in /// [`QueuedSolution`]. pub fn finalize_signed_phase() -> (bool, Weight) { - let mut all_submission: Vec> = >::take(); + let mut all_submissions: Vec> = >::take(); let mut found_solution = false; let mut weight = T::DbWeight::get().reads(1); - while let Some(best) = all_submission.pop() { + // Reverse the ordering of submissions: previously it was ordered such that high-scoring + // solutions have low indices. Now, the code flows more cleanly if high-scoring solutions + // have high indices. + all_submissions.reverse(); + + while let Some(best) = all_submissions.pop() { let SignedSubmission { solution, who, deposit, reward } = best; let active_voters = solution.compact.voter_count() as u32; let feasibility_weight = { @@ -106,12 +111,12 @@ impl Pallet { // Any unprocessed solution is pointless to even consider. Feasible or malicious, // they didn't end up being used. Unreserve the bonds. - all_submission.into_iter().for_each(|not_processed| { + for not_processed in all_submissions { let SignedSubmission { who, deposit, .. } = not_processed; let _remaining = T::Currency::unreserve(&who, deposit); weight = weight.saturating_add(T::DbWeight::get().writes(1)); debug_assert!(_remaining.is_zero()); - }); + }; (found_solution, weight) } @@ -152,6 +157,8 @@ impl Pallet { /// Insert a solution into the queue while maintaining an ordering by solution quality. /// + /// Solutions are ordered in reverse: strong solutions have low indices. + /// /// If insertion was successful, `Some(index)` is returned where index is the /// index of the newly inserted item. /// @@ -166,28 +173,31 @@ impl Pallet { solution: RawSolution>, size: SolutionOrSnapshotSize, ) -> Option { + // Let's ensure that our input is valid. + let max_submissions = T::SignedMaxSubmissions::get(); + debug_assert!(queue.len() as u32 <= max_submissions); + let threshold = T::SolutionImprovementThreshold::get(); // this insertion logic is a bit unusual in that a new solution which beats an existing - // solution by less than the threshold is sorted as "less" than the existing solution. + // solution by less than the threshold is sorted as "greater" than the existing solution. // this means that the produced ordering depends on the order of insertion, and that // attempts to produce a total ordering using this comparitor are highly unstable. // // this ordering prioritizes earlier solutions over slightly better later ones. - let mut insertion_position = queue.binary_search_by(|s| { + let insertion_position = queue.binary_search_by(|s| { if is_score_better::( solution.score, s.solution.score, threshold, ) { - Ordering::Less - } else { Ordering::Greater + } else { + Ordering::Less } }).expect_err("comparitor function never returns Ordering::Equal; qed"); - let max_submissions = T::SignedMaxSubmissions::get(); // if this solution is the worst one so far and the queue is full, don't insert - if insertion_position == 0 && queue.len() as u32 >= max_submissions { + if insertion_position == queue.len() && queue.len() as u32 >= max_submissions { return None; } @@ -198,35 +208,19 @@ impl Pallet { SignedSubmission { who: who.clone(), deposit, reward, solution }; queue.insert(insertion_position, submission); - // If the queue was within length when this function was called, then this must always be correct. - debug_assert!(queue.len() as u32 <= max_submissions + 1); - - // Remove the weakest, if needed. + // Remove the weakest if queue is overflowing. + // This doesn't adjust insertion_position: in the event that it might have, we'd have short- + // circuited above. if queue.len() as u32 > max_submissions { - Self::remove_weakest(queue); - insertion_position -= 1; - // this is sound because remove_weakest always removes the 0th item. - // if `insertion_position` was 0 and this could potentially have triggered, we've - // already short-circuited above. + if let Some(SignedSubmission { who, deposit, .. }) = queue.pop() { + let _remainder = T::Currency::unreserve(&who, deposit); + debug_assert!(_remainder.is_zero()); + } } debug_assert!(queue.len() as u32 <= max_submissions); Some(insertion_position) } - /// Removes the weakest element of the queue, namely the first one, should the length of the - /// queue be enough. - /// - /// noop if the queue is empty. Bond of the removed solution is returned. - fn remove_weakest( - queue: &mut Vec, CompactOf>>, - ) { - if queue.len() > 0 { - let SignedSubmission { who, deposit, .. } = queue.remove(0); - let _remainder = T::Currency::unreserve(&who, deposit); - debug_assert!(_remainder.is_zero()); - } - } - /// The feasibility weight of the given raw solution. pub fn feasibility_weight_of( solution: &RawSolution>, @@ -483,7 +477,7 @@ mod tests { .into_iter() .map(|s| s.solution.score[0]) .collect::>(), - vec![5, 6, 7, 8, 9] + vec![9, 8, 7, 6, 5] ); // better. @@ -496,7 +490,7 @@ mod tests { .into_iter() .map(|s| s.solution.score[0]) .collect::>(), - vec![6, 7, 8, 9, 20] + vec![20, 9, 8, 7, 6] ); }) } @@ -521,7 +515,7 @@ mod tests { .into_iter() .map(|s| s.solution.score[0]) .collect::>(), - vec![4, 6, 7, 8, 9] + vec![9, 8, 7, 6, 4], ); // better. @@ -534,7 +528,7 @@ mod tests { .into_iter() .map(|s| s.solution.score[0]) .collect::>(), - vec![5, 6, 7, 8, 9] + vec![9, 8, 7, 6, 5], ); }) } @@ -579,7 +573,7 @@ mod tests { .into_iter() .map(|s| s.solution.score[0]) .collect::>(), - vec![5, 6, 7] + vec![7, 6, 5] ); // 5 is not accepted. This will only cause processing with no benefit. @@ -610,27 +604,27 @@ mod tests { let solution = RawSolution { score: [8, 0, 0], ..Default::default() }; assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![5, 8]); + assert_eq!(scores(), vec![8, 5]); let solution = RawSolution { score: [3, 0, 0], ..Default::default() }; assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![3, 5, 8]); + assert_eq!(scores(), vec![8, 5, 3]); let solution = RawSolution { score: [6, 0, 0], ..Default::default() }; assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![5, 6, 8]); + assert_eq!(scores(), vec![8, 6, 5]); let solution = RawSolution { score: [6, 0, 0], ..Default::default() }; assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![6, 6, 8]); + assert_eq!(scores(), vec![8, 6, 6]); let solution = RawSolution { score: [10, 0, 0], ..Default::default() }; assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![6, 8, 10]); + assert_eq!(scores(), vec![10, 8, 6]); let solution = RawSolution { score: [12, 0, 0], ..Default::default() }; assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![8, 10, 12]); + assert_eq!(scores(), vec![12, 10, 8]); }) } @@ -663,7 +657,7 @@ mod tests { assert_eq!( MultiPhase::signed_submissions().iter().map(|x| x.who).collect::>(), - vec![9999, 99, 999] + vec![999, 99, 9999] ); // _some_ good solution was stored. From d5fe6713319dba94b00d8e3b1461c0a0be108767 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 31 Mar 2021 11:13:58 +0200 Subject: [PATCH 27/86] get rid of unused import --- bin/node/runtime/src/lib.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 2cd16db27a897..1c0131adad663 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -73,6 +73,7 @@ use pallet_session::{historical as pallet_session_historical}; use sp_inherents::{InherentData, CheckInherentsResult}; use static_assertions::const_assert; use pallet_contracts::weights::WeightInfo; +use pallet_election_provider_multi_phase::FallbackStrategy; #[cfg(any(feature = "std", test))] pub use sp_runtime::BuildStorage; @@ -497,7 +498,6 @@ impl pallet_staking::Config for Runtime { type WeightInfo = pallet_staking::weights::SubstrateWeight; } -use pallet_election_provider_multi_phase::FallbackStrategy; parameter_types! { // phase durations. 1/4 of the last session for each. pub const SignedPhase: u32 = EPOCH_DURATION_IN_BLOCKS / 4; @@ -510,8 +510,7 @@ parameter_types! { pub const SignedDepositByte: Balance = 1 * CENTS; // fallback: no need to do on-chain phragmen initially. - pub const Fallback: pallet_election_provider_multi_phase::FallbackStrategy = - pallet_election_provider_multi_phase::FallbackStrategy::OnChain; + pub const Fallback: FallbackStrategy = FallbackStrategy::OnChain; pub SolutionImprovementThreshold: Perbill = Perbill::from_rational(1u32, 10_000); From 75ddba9339f3f622d5ff5eed9f14edde3e63a58c Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 31 Mar 2021 12:02:27 +0200 Subject: [PATCH 28/86] ensure signed submissions are cleared in early elect --- frame/election-provider-multi-phase/src/lib.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index d6378e41fade8..5f5c3ef5a25b9 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -1175,13 +1175,11 @@ impl Pallet { /// 3. Clear all snapshot data. fn post_elect() { // inc round - >::mutate(|r| *r = *r + 1); + >::mutate(|r| *r += 1); // change phase >::put(Phase::Off); - // TODO: clean signed submissions: test case: call `elect` in the middle of the signed phase. - // kill snapshots Self::kill_snapshot(); } @@ -1444,8 +1442,8 @@ mod tests { fn phase_rotation_works() { ExtBuilder::default().build_and_execute(|| { // 0 ------- 15 ------- 25 ------- 30 ------- ------- 45 ------- 55 ------- 60 - // | | | | - // Signed Unsigned Signed Unsigned + // | | | | | | + // Signed Unsigned Elect Signed Unsigned Elect assert_eq!(System::block_number(), 0); assert_eq!(MultiPhase::current_phase(), Phase::Off); @@ -1604,6 +1602,7 @@ mod tests { assert!(MultiPhase::snapshot_metadata().is_none()); assert!(MultiPhase::desired_targets().is_none()); assert!(MultiPhase::queued_solution().is_none()); + assert!(MultiPhase::signed_submissions().is_empty()); }) } From abae6023437d33e721edf5fdcc990267488aad2b Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 31 Mar 2021 13:53:40 +0200 Subject: [PATCH 29/86] finalize the signed phase when appropriate - ensure we don't leave storage lying around, even if elect called prematurely - test that proposition - disable the unsigned phase if a viable solution from the signed phase exists - ensure signed phase finalization weight is accounted for --- .../election-provider-multi-phase/src/lib.rs | 65 +++++++++++++++++-- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 5f5c3ef5a25b9..8692b8f6b6bec 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -632,11 +632,19 @@ pub mod pallet { Phase::Signed | Phase::Off if remaining <= unsigned_deadline && remaining > Zero::zero() => { - // determine if followed by signed or not. + // our needs vary according to whether or not the unsigned phase follows a signed phase let (need_snapshot, enabled, signed_weight) = if current_phase == Phase::Signed { - // followed by a signed phase: close the signed phase, no need for snapshot. - // TODO: proper weight https://github.com/paritytech/substrate/pull/7910. - (false, true, Weight::zero()) + // there was previously a signed phase: close the signed phase, no need for snapshot. + // + // Notes: + // + // - If the signed phase produced a viable solution, we disable the unsigned + // phase. We want to prioritize signed solutions whenever they're available. + // - `Self::finalize_signed_phase()` also appears in `fn do_elect`. This is + // a guard against the case that `elect` is called prematurely. This adds + // a small amount of overhead, but that is unfortunately unavoidable. + let (success, weight) = Self::finalize_signed_phase(); + (false, !success, weight) } else { // no signed phase: create a new snapshot, definitely `enable` the unsigned // phase. @@ -1194,6 +1202,13 @@ impl Pallet { } fn do_elect() -> Result<(Supports, Weight), ElectionError> { + // We have to unconditionally try finalizing the signed phase here. There are only two + // possibilities: + // + // - signed phase was open, in which case this is essential for correct functioning of the system + // - signed phase was complete or not started, in which case finalization is idempotent and + // inexpensive (1 read of an empty vector). + let (_, signed_finalize_weight) = Self::finalize_signed_phase(); >::take() .map_or_else( || match T::Fallback::get() { @@ -1213,7 +1228,7 @@ impl Pallet { if Self::round() != 1 { log!(info, "Finalized election round with compute {:?}.", compute); } - (supports, weight) + (supports, weight.saturating_add(signed_finalize_weight)) }) .map_err(|err| { Self::deposit_event(Event::ElectionFinalized(None)); @@ -1432,10 +1447,11 @@ mod tests { Phase, mock::{ ExtBuilder, MultiPhase, Runtime, roll_to, MockWeightInfo, AccountId, TargetIndex, - Targets, multi_phase_events, System, + Targets, multi_phase_events, System, SignedMaxSubmissions, }, }; use frame_election_provider_support::ElectionProvider; + use frame_support::assert_ok; use sp_npos_elections::Support; #[test] @@ -1606,6 +1622,43 @@ mod tests { }) } + #[test] + fn early_termination_with_submissions() { + // an early termination in the signed phase, with no queued solution. + ExtBuilder::default().build_and_execute(|| { + // signed phase started at block 15 and will end at 25. + roll_to(14); + assert_eq!(MultiPhase::current_phase(), Phase::Off); + + roll_to(15); + assert_eq!(multi_phase_events(), vec![Event::SignedPhaseStarted(1)]); + assert_eq!(MultiPhase::current_phase(), Phase::Signed); + assert_eq!(MultiPhase::round(), 1); + + // fill the queue with signed submissions + for s in 0..SignedMaxSubmissions::get() { + let solution = RawSolution { score: [(5 + s).into(), 0, 0], ..Default::default() }; + assert_ok!(MultiPhase::submit( + crate::mock::Origin::signed(99), + solution, + MultiPhase::signed_submissions().len() as u32 + )); + } + + // an unexpected call to elect. + roll_to(20); + MultiPhase::elect().unwrap(); + + // all storage items must be cleared. + assert_eq!(MultiPhase::round(), 2); + assert!(MultiPhase::snapshot().is_none()); + assert!(MultiPhase::snapshot_metadata().is_none()); + assert!(MultiPhase::desired_targets().is_none()); + assert!(MultiPhase::queued_solution().is_none()); + assert!(MultiPhase::signed_submissions().is_empty()); + }) + } + #[test] fn fallback_strategy_works() { ExtBuilder::default().fallback(FallbackStrategy::OnChain).build_and_execute(|| { From f4aea1d0d40a428c0d43ec3914f1175ef7092937 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 31 Mar 2021 14:16:57 +0200 Subject: [PATCH 30/86] resolve dispatch error todo --- frame/election-provider-multi-phase/src/unsigned.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/frame/election-provider-multi-phase/src/unsigned.rs b/frame/election-provider-multi-phase/src/unsigned.rs index e0479bd0b17fa..0ac33856d84d0 100644 --- a/frame/election-provider-multi-phase/src/unsigned.rs +++ b/frame/election-provider-multi-phase/src/unsigned.rs @@ -43,8 +43,7 @@ pub enum MinerError { /// Submitting a transaction to the pool failed. PoolSubmissionFailed, /// The pre-dispatch checks failed for the mined solution. - // TODO: maybe wrap a DispatchError here to be able to clarify which one failed? - PreDispatchChecksFailed, + PreDispatchChecksFailed(DispatchError), /// The solution generated from the miner is not feasible. Feasibility(FeasibilityError), } @@ -96,7 +95,7 @@ impl Pallet { // ensure that this will pass the pre-dispatch checks Self::unsigned_pre_dispatch_checks(&raw_solution).map_err(|e| { log!(warn, "pre-dispatch-checks failed for mined solution: {:?}", e); - MinerError::PreDispatchChecksFailed + MinerError::PreDispatchChecksFailed(e) })?; // ensure that this is a feasible solution @@ -760,10 +759,10 @@ mod tests { roll_to(25); assert!(MultiPhase::current_phase().is_unsigned()); - assert_eq!( + assert!(matches!( MultiPhase::mine_check_and_submit().unwrap_err(), - MinerError::PreDispatchChecksFailed, - ); + MinerError::PreDispatchChecksFailed(_), + )); }) } From ebface459ab3f41374af7481ec129712b25ebe25 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 31 Mar 2021 16:38:46 +0200 Subject: [PATCH 31/86] update assumptions in submit benchmark --- .../src/benchmarking.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/frame/election-provider-multi-phase/src/benchmarking.rs b/frame/election-provider-multi-phase/src/benchmarking.rs index a844740ef2995..349998cd1107e 100644 --- a/frame/election-provider-multi-phase/src/benchmarking.rs +++ b/frame/election-provider-multi-phase/src/benchmarking.rs @@ -262,21 +262,22 @@ frame_benchmarking::benchmarks! { submit { let c in 1 .. (T::SignedMaxSubmissions::get() - 1); - // the solution will be worse than all of them meaning the score need to be checked against all. + // the solution will be worse than all of them meaning the score need to be checked against ~ log2(c) let solution = RawSolution { score: [(10_000_000u128 - 1).into(), 0, 0], ..Default::default() }; >::put(Phase::Signed); >::put(1); - for i in 0..c { - >::mutate(|queue| { + >::mutate(|queue| { + for i in 0..c { let solution = RawSolution { score: [(10_000_000 + i).into(), 0, 0], ..Default::default() }; let signed_submission = SignedSubmission { solution, ..Default::default() }; - // note: this is quite tricky: we know that the queue will stay sorted here. The - // last will be best. queue.push(signed_submission); - }) - } + } + // as of here, the queue is ordered worst-to-best. + // However, we have an invariant that it should be ordered best-to-worst + queue.reverse(); + }); let caller = frame_benchmarking::whitelisted_caller(); T::Currency::make_free_balance_be(&caller, T::Currency::minimum_balance() * 10u32.into()); From 8d957229084900c60e414b8c5b24515bd61f8686 Mon Sep 17 00:00:00 2001 From: Parity Benchmarking Bot Date: Mon, 12 Apr 2021 12:57:47 +0000 Subject: [PATCH 32/86] cargo run --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_election_provider_multi_phase --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/election-provider-multi-phase/src/weights.rs --template=./.maintain/frame-weight-template.hbs --- .../src/weights.rs | 248 ++++++++++++++---- 1 file changed, 193 insertions(+), 55 deletions(-) diff --git a/frame/election-provider-multi-phase/src/weights.rs b/frame/election-provider-multi-phase/src/weights.rs index c18b0b67caf99..03dedcc35e6ec 100644 --- a/frame/election-provider-multi-phase/src/weights.rs +++ b/frame/election-provider-multi-phase/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_election_provider_multi_phase //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0 -//! DATE: 2021-03-19, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-04-12, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -44,7 +44,11 @@ use sp_std::marker::PhantomData; /// Weight functions needed for pallet_election_provider_multi_phase. pub trait WeightInfo { + fn on_initialize_nothing() -> Weight; + fn on_initialize_open_signed() -> Weight; fn on_initialize_open_unsigned_with_snapshot() -> Weight; + fn finalize_signed_phase_accept_solution() -> Weight; + fn finalize_signed_phase_reject_solution() -> Weight; fn on_initialize_open_unsigned_without_snapshot() -> Weight; fn elect_queued() -> Weight; fn submit(c: u32, ) -> Weight; @@ -52,143 +56,277 @@ pub trait WeightInfo { fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; fn on_initialize_nothing() -> Weight; fn on_initialize_open_signed() -> Weight; + fn on_initialize_open_unsigned_with_snapshot() -> Weight; fn finalize_signed_phase_accept_solution() -> Weight; fn finalize_signed_phase_reject_solution() -> Weight; + fn on_initialize_open_unsigned_without_snapshot() -> Weight; + fn elect_queued() -> Weight; + fn submit(c: u32, ) -> Weight; + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight; + fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; } /// Weights for pallet_election_provider_multi_phase using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { + fn on_initialize_nothing() -> Weight { + (23_021_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + } + fn on_initialize_open_signed() -> Weight { + (112_730_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + fn on_initialize_open_unsigned_with_snapshot() -> Weight { + (112_818_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + fn finalize_signed_phase_accept_solution() -> Weight { + (46_382_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + fn finalize_signed_phase_reject_solution() -> Weight { + (20_567_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } fn on_initialize_open_unsigned_without_snapshot() -> Weight { - (21_039_000 as Weight) + (21_354_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn elect_queued() -> Weight { - (7_362_949_000 as Weight) - .saturating_add(T::DbWeight::get().reads(2 as Weight)) + (7_833_572_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) } fn submit(c: u32, ) -> Weight { - (84_430_000 as Weight) - // Standard Error: 146_000 - .saturating_add((2_758_000 as Weight).saturating_mul(c as Weight)) + (87_180_000 as Weight) + // Standard Error: 24_000 + .saturating_add((3_775_000 as Weight).saturating_mul(c as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 21_000 - .saturating_add((3_933_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 21_000 - .saturating_add((13_520_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 107_000 - .saturating_add((2_880_000 as Weight).saturating_mul(d as Weight)) + // Standard Error: 20_000 + .saturating_add((4_209_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 20_000 + .saturating_add((13_422_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 104_000 + .saturating_add((3_035_000 as Weight).saturating_mul(d as Weight)) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 10_000 - .saturating_add((4_069_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 36_000 - .saturating_add((503_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 10_000 - .saturating_add((10_000_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 54_000 - .saturating_add((3_734_000 as Weight).saturating_mul(d as Weight)) + // Standard Error: 11_000 + .saturating_add((4_381_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 39_000 + .saturating_add((584_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 11_000 + .saturating_add((10_188_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 59_000 + .saturating_add((3_756_000 as Weight).saturating_mul(d as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) } fn on_initialize_nothing() -> Weight { - (24_128_000 as Weight) + (23_161_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) } fn on_initialize_open_signed() -> Weight { - (80_951_000 as Weight) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) + (113_365_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } fn on_initialize_open_unsigned_with_snapshot() -> Weight { - (79_888_000 as Weight) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) + (113_284_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } fn finalize_signed_phase_accept_solution() -> Weight { - (47_783_000 as Weight) + (46_572_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } fn finalize_signed_phase_reject_solution() -> Weight { - (21_277_000 as Weight) + (20_913_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + fn on_initialize_open_unsigned_without_snapshot() -> Weight { + (21_525_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn elect_queued() -> Weight { + (7_857_625_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) + } + fn submit(c: u32, ) -> Weight { + (88_973_000 as Weight) + // Standard Error: 32_000 + .saturating_add((2_982_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 21_000 + .saturating_add((4_196_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 21_000 + .saturating_add((13_460_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 108_000 + .saturating_add((3_306_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(6 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 11_000 + .saturating_add((4_354_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 39_000 + .saturating_add((486_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 11_000 + .saturating_add((10_194_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 58_000 + .saturating_add((3_669_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + } } // For backwards compatibility and tests impl WeightInfo for () { + fn on_initialize_nothing() -> Weight { + (23_021_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + } + fn on_initialize_open_signed() -> Weight { + (112_730_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + fn on_initialize_open_unsigned_with_snapshot() -> Weight { + (112_818_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + fn finalize_signed_phase_accept_solution() -> Weight { + (46_382_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + fn finalize_signed_phase_reject_solution() -> Weight { + (20_567_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } fn on_initialize_open_unsigned_without_snapshot() -> Weight { - (21_039_000 as Weight) + (21_354_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn elect_queued() -> Weight { - (7_362_949_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + (7_833_572_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } fn submit(c: u32, ) -> Weight { - (84_430_000 as Weight) - // Standard Error: 146_000 - .saturating_add((2_758_000 as Weight).saturating_mul(c as Weight)) + (87_180_000 as Weight) + // Standard Error: 24_000 + .saturating_add((3_775_000 as Weight).saturating_mul(c as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 21_000 - .saturating_add((3_933_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 21_000 - .saturating_add((13_520_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 107_000 - .saturating_add((2_880_000 as Weight).saturating_mul(d as Weight)) + // Standard Error: 20_000 + .saturating_add((4_209_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 20_000 + .saturating_add((13_422_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 104_000 + .saturating_add((3_035_000 as Weight).saturating_mul(d as Weight)) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 10_000 - .saturating_add((4_069_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 36_000 - .saturating_add((503_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 10_000 - .saturating_add((10_000_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 54_000 - .saturating_add((3_734_000 as Weight).saturating_mul(d as Weight)) + // Standard Error: 11_000 + .saturating_add((4_381_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 39_000 + .saturating_add((584_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 11_000 + .saturating_add((10_188_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 59_000 + .saturating_add((3_756_000 as Weight).saturating_mul(d as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) } fn on_initialize_nothing() -> Weight { - (24_128_000 as Weight) + (23_161_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) } fn on_initialize_open_signed() -> Weight { - (80_951_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + (113_365_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } fn on_initialize_open_unsigned_with_snapshot() -> Weight { - (79_888_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + (113_284_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } fn finalize_signed_phase_accept_solution() -> Weight { - (47_783_000 as Weight) + (46_572_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } fn finalize_signed_phase_reject_solution() -> Weight { - (21_277_000 as Weight) + (20_913_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + fn on_initialize_open_unsigned_without_snapshot() -> Weight { + (21_525_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn elect_queued() -> Weight { + (7_857_625_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + } + fn submit(c: u32, ) -> Weight { + (88_973_000 as Weight) + // Standard Error: 32_000 + .saturating_add((2_982_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 21_000 + .saturating_add((4_196_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 21_000 + .saturating_add((13_460_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 108_000 + .saturating_add((3_306_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(6 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 11_000 + .saturating_add((4_354_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 39_000 + .saturating_add((486_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 11_000 + .saturating_add((10_194_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 58_000 + .saturating_add((3_669_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + } } From a3574d5d104438a2d7606cd44bbb300716210a89 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 25 May 2021 12:11:25 +0200 Subject: [PATCH 33/86] line length --- .../src/benchmarking.rs | 49 +++++++++---- .../src/signed.rs | 5 +- .../src/unsigned.rs | 69 +++++++++++++++---- 3 files changed, 95 insertions(+), 28 deletions(-) diff --git a/frame/election-provider-multi-phase/src/benchmarking.rs b/frame/election-provider-multi-phase/src/benchmarking.rs index 58991226b8c1f..fa9a1f7a3a126 100644 --- a/frame/election-provider-multi-phase/src/benchmarking.rs +++ b/frame/election-provider-multi-phase/src/benchmarking.rs @@ -263,15 +263,22 @@ frame_benchmarking::benchmarks! { submit { let c in 1 .. (T::SignedMaxSubmissions::get() - 1); - // the solution will be worse than all of them meaning the score need to be checked against ~ log2(c) - let solution = RawSolution { score: [(10_000_000u128 - 1).into(), 0, 0], ..Default::default() }; + // the solution will be worse than all of them meaning the score need to be checked against + // ~ log2(c) + let solution = RawSolution { + score: [(10_000_000u128 - 1).into(), 0, 0], + ..Default::default() + }; >::put(Phase::Signed); >::put(1); >::mutate(|queue| { for i in 0..c { - let solution = RawSolution { score: [(10_000_000 + i).into(), 0, 0], ..Default::default() }; + let solution = RawSolution { + score: [(10_000_000 + i).into(), 0, 0], + ..Default::default() + }; let signed_submission = SignedSubmission { solution, ..Default::default() }; queue.push(signed_submission); } @@ -295,9 +302,12 @@ frame_benchmarking::benchmarks! { let t in (T::BenchmarkingConfig::TARGETS[0]) .. T::BenchmarkingConfig::TARGETS[1]; // number of assignments, i.e. compact.len(). This means the active nominators, thus must be // a subset of `v` component. - let a in (T::BenchmarkingConfig::ACTIVE_VOTERS[0]) .. T::BenchmarkingConfig::ACTIVE_VOTERS[1]; + let a in + (T::BenchmarkingConfig::ACTIVE_VOTERS[0]) .. T::BenchmarkingConfig::ACTIVE_VOTERS[1]; // number of desired targets. Must be a subset of `t` component. - let d in (T::BenchmarkingConfig::DESIRED_TARGETS[0]) .. T::BenchmarkingConfig::DESIRED_TARGETS[1]; + let d in + (T::BenchmarkingConfig::DESIRED_TARGETS[0]) .. + T::BenchmarkingConfig::DESIRED_TARGETS[1]; let witness = SolutionOrSnapshotSize { voters: v, targets: t }; let raw_solution = solution_with_size::(witness, a, d); @@ -310,7 +320,8 @@ frame_benchmarking::benchmarks! { let encoded_call = >::submit_unsigned(raw_solution.clone(), witness).encode(); }: { assert_ok!(>::submit_unsigned(RawOrigin::None.into(), raw_solution, witness)); - let _decoded_snap = as Decode>::decode(&mut &*encoded_snapshot).unwrap(); + let _decoded_snap = as Decode>::decode(&mut &*encoded_snapshot) + .unwrap(); let _decoded_call = as Decode>::decode(&mut &*encoded_call).unwrap(); } verify { assert!(>::queued_solution().is_some()); @@ -324,13 +335,17 @@ frame_benchmarking::benchmarks! { let t in (T::BenchmarkingConfig::TARGETS[0]) .. T::BenchmarkingConfig::TARGETS[1]; // number of assignments, i.e. compact.len(). This means the active nominators, thus must be // a subset of `v` component. - let a in (T::BenchmarkingConfig::ACTIVE_VOTERS[0]) .. T::BenchmarkingConfig::ACTIVE_VOTERS[1]; + let a in + (T::BenchmarkingConfig::ACTIVE_VOTERS[0]) .. T::BenchmarkingConfig::ACTIVE_VOTERS[1]; // number of desired targets. Must be a subset of `t` component. - let d in (T::BenchmarkingConfig::DESIRED_TARGETS[0]) .. T::BenchmarkingConfig::DESIRED_TARGETS[1]; + let d in + (T::BenchmarkingConfig::DESIRED_TARGETS[0]) .. + T::BenchmarkingConfig::DESIRED_TARGETS[1]; // Subtract this percentage from the actual encoded size let f in 0 .. 95; - // Compute a random solution, then work backwards to get the lists of voters, targets, and assignments + // Compute a random solution, then work backwards to get the lists of voters, targets, and + // assignments let witness = SolutionOrSnapshotSize { voters: v, targets: t }; let RawSolution { compact, .. } = solution_with_size::(witness, a, d); let RoundSnapshot { voters, targets } = MultiPhase::::snapshot().unwrap(); @@ -374,7 +389,11 @@ frame_benchmarking::benchmarks! { } verify { let compact = CompactOf::::try_from(index_assignments.as_slice()).unwrap(); let encoding = compact.encode(); - log!(trace, "encoded size prediction = {}", encoded_size_of(index_assignments.as_slice()).unwrap()); + log!( + trace, + "encoded size prediction = {}", + encoded_size_of(index_assignments.as_slice()).unwrap(), + ); log!(trace, "actual encoded size = {}", encoding.len()); assert!(encoding.len() <= desired_size); } @@ -387,9 +406,12 @@ frame_benchmarking::benchmarks! { let t in (T::BenchmarkingConfig::TARGETS[0]) .. T::BenchmarkingConfig::TARGETS[1]; // number of assignments, i.e. compact.len(). This means the active nominators, thus must be // a subset of `v` component. - let a in (T::BenchmarkingConfig::ACTIVE_VOTERS[0]) .. T::BenchmarkingConfig::ACTIVE_VOTERS[1]; + let a in + (T::BenchmarkingConfig::ACTIVE_VOTERS[0]) .. T::BenchmarkingConfig::ACTIVE_VOTERS[1]; // number of desired targets. Must be a subset of `t` component. - let d in (T::BenchmarkingConfig::DESIRED_TARGETS[0]) .. T::BenchmarkingConfig::DESIRED_TARGETS[1]; + let d in + (T::BenchmarkingConfig::DESIRED_TARGETS[0]) .. + T::BenchmarkingConfig::DESIRED_TARGETS[1]; let size = SolutionOrSnapshotSize { voters: v, targets: t }; let raw_solution = solution_with_size::(size, a, d); @@ -401,7 +423,8 @@ frame_benchmarking::benchmarks! { let encoded_snapshot = >::snapshot().unwrap().encode(); }: { assert_ok!(>::feasibility_check(raw_solution, ElectionCompute::Unsigned)); - let _decoded_snap = as Decode>::decode(&mut &*encoded_snapshot).unwrap(); + let _decoded_snap = as Decode>::decode(&mut &*encoded_snapshot) + .unwrap(); } } diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 04840c687e9cc..6a07e1cefeb82 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -241,7 +241,10 @@ impl Pallet { /// 1. base deposit, fixed for all submissions. /// 2. a per-byte deposit, for renting the state usage. /// 3. a per-weight deposit, for the potential weight usage in an upcoming on_initialize - pub fn deposit_for(solution: &RawSolution>, size: SolutionOrSnapshotSize) -> BalanceOf { + pub fn deposit_for( + solution: &RawSolution>, + size: SolutionOrSnapshotSize, + ) -> BalanceOf { let encoded_len: BalanceOf = solution.using_encoded(|e| e.len() as u32).into(); let feasibility_weight = Self::feasibility_weight_of(solution, size); diff --git a/frame/election-provider-multi-phase/src/unsigned.rs b/frame/election-provider-multi-phase/src/unsigned.rs index 3a8dd1db717e1..142f0e2d8eede 100644 --- a/frame/election-provider-multi-phase/src/unsigned.rs +++ b/frame/election-provider-multi-phase/src/unsigned.rs @@ -29,7 +29,12 @@ use sp_npos_elections::{ CompactSolution, ElectionResult, assignment_ratio_to_staked_normalized, assignment_staked_to_ratio_normalized, is_score_better, seq_phragmen, }; -use sp_runtime::{DispatchError, SaturatedConversion, offchain::storage::StorageValueRef, traits::TrailingZeroInput}; +use sp_runtime::{ + DispatchError, + SaturatedConversion, + offchain::storage::StorageValueRef, + traits::TrailingZeroInput, +}; use sp_std::{cmp::Ordering, convert::TryFrom, vec::Vec}; /// Storage key used to store the last block number at which offchain worker ran. @@ -54,7 +59,8 @@ pub type Assignment = sp_npos_elections::Assignment< CompactAccuracyOf, >; -/// The [`IndexAssignment`][sp_npos_elections::IndexAssignment] type specialized for a particular runtime `T`. +/// The [`IndexAssignment`][sp_npos_elections::IndexAssignment] type specialized for a particular +/// runtime `T`. pub type IndexAssignmentOf = sp_npos_elections::IndexAssignmentOf>; #[derive(Debug, Eq, PartialEq)] @@ -343,7 +349,11 @@ impl Pallet { // converting to `Compact`. let mut index_assignments = sorted_assignments .into_iter() - .map(|assignment| IndexAssignmentOf::::new(&assignment, &voter_index, &target_index)) + .map(|assignment| IndexAssignmentOf::::new( + &assignment, + &voter_index, + &target_index, + )) .collect::, _>>()?; // trim assignments list for weight and length. @@ -415,7 +425,9 @@ impl Pallet { size, max_weight, ); - let removing: usize = assignments.len().saturating_sub(maximum_allowed_voters.saturated_into()); + let removing: usize = assignments.len().saturating_sub( + maximum_allowed_voters.saturated_into(), + ); log!( debug, "from {} assignments, truncating to {} for weight, removing {}", @@ -463,7 +475,9 @@ impl Pallet { } } let maximum_allowed_voters = - if low < assignments.len() && encoded_size_of(&assignments[..low + 1])? <= max_allowed_length { + if low < assignments.len() && + encoded_size_of(&assignments[..low + 1])? <= max_allowed_length + { low + 1 } else { low @@ -1001,7 +1015,11 @@ mod tests { assert_eq!( MultiPhase::mine_check_save_submit().unwrap_err(), - MinerError::PreDispatchChecksFailed(DispatchError::Module{ index: 2, error: 1, message: Some("PreDispatchWrongWinnerCount")}), + MinerError::PreDispatchChecksFailed(DispatchError::Module{ + index: 2, + error: 1, + message: Some("PreDispatchWrongWinnerCount"), + }), ); }) } @@ -1206,11 +1224,17 @@ mod tests { let mut storage = StorageValueRef::persistent(&OFFCHAIN_LAST_BLOCK); storage.clear(); - assert!(!ocw_solution_exists::(), "no solution should be present before we mine one"); + assert!( + !ocw_solution_exists::(), + "no solution should be present before we mine one", + ); // creates and cache a solution MultiPhase::offchain_worker(25); - assert!(ocw_solution_exists::(), "a solution must be cached after running the worker"); + assert!( + ocw_solution_exists::(), + "a solution must be cached after running the worker", + ); // after an election, the solution must be cleared // we don't actually care about the result of the election @@ -1336,10 +1360,15 @@ mod tests { _ => panic!("bad call: unexpected submission"), }; - // Custom(3) maps to PreDispatchChecksFailed - let pre_dispatch_check_error = TransactionValidityError::Invalid(InvalidTransaction::Custom(7)); + // Custom(7) maps to PreDispatchChecksFailed + let pre_dispatch_check_error = TransactionValidityError::Invalid( + InvalidTransaction::Custom(7), + ); assert_eq!( - ::validate_unsigned(TransactionSource::Local, &call) + ::validate_unsigned( + TransactionSource::Local, + &call, + ) .unwrap_err(), pre_dispatch_check_error, ); @@ -1366,7 +1395,11 @@ mod tests { let compact_clone = compact.clone(); // when - MultiPhase::trim_assignments_length(encoded_len, &mut assignments, encoded_size_of).unwrap(); + MultiPhase::trim_assignments_length( + encoded_len, + &mut assignments, + encoded_size_of, + ).unwrap(); // then let compact = CompactOf::::try_from(assignments.as_slice()).unwrap(); @@ -1390,7 +1423,11 @@ mod tests { let compact_clone = compact.clone(); // when - MultiPhase::trim_assignments_length(encoded_len as u32 - 1, &mut assignments, encoded_size_of).unwrap(); + MultiPhase::trim_assignments_length( + encoded_len as u32 - 1, + &mut assignments, + encoded_size_of, + ).unwrap(); // then let compact = CompactOf::::try_from(assignments.as_slice()).unwrap(); @@ -1421,7 +1458,11 @@ mod tests { .unwrap(); // when - MultiPhase::trim_assignments_length(encoded_len - 1, &mut assignments, encoded_size_of).unwrap(); + MultiPhase::trim_assignments_length( + encoded_len - 1, + &mut assignments, + encoded_size_of, + ).unwrap(); // then assert_eq!(assignments.len(), count - 1, "we must have removed exactly one assignment"); From ee66c843adfff7ce79dcbbceca5980d5dadbaa11 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 26 May 2021 11:28:14 +0200 Subject: [PATCH 34/86] make a few more things pub --- frame/election-provider-multi-phase/src/signed.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 6a07e1cefeb82..e80ccadcda88d 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -34,13 +34,13 @@ use sp_std::{cmp::Ordering, vec::Vec}; #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, Default)] pub struct SignedSubmission { /// Who submitted this solution. - pub(crate) who: AccountId, + pub who: AccountId, /// The deposit reserved for storing this solution. - pub(crate) deposit: Balance, + pub deposit: Balance, /// The reward that should be given to this solution, if chosen the as the final one. - pub(crate) reward: Balance, + pub reward: Balance, /// The raw solution itself. - pub(crate) solution: RawSolution, + pub solution: RawSolution, } pub(crate) type BalanceOf = From 17faa86580bf2028daf851b84eb013405d2ec648 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 26 May 2021 16:25:21 +0200 Subject: [PATCH 35/86] restore missing import --- frame/election-provider-multi-phase/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 0b826b7d5357e..290ab2cac8dd4 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -1557,7 +1557,7 @@ mod tests { }, }; use frame_election_provider_support::ElectionProvider; - use frame_support::assert_ok; + use frame_support::{assert_noop, assert_ok}; use sp_npos_elections::Support; #[test] From 94314bf7c2ec3d4819ce678ab9210d734a7da1de Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 26 May 2021 16:50:46 +0200 Subject: [PATCH 36/86] update ui test output --- frame/support/test/tests/derive_no_bound_ui/eq.stderr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/support/test/tests/derive_no_bound_ui/eq.stderr b/frame/support/test/tests/derive_no_bound_ui/eq.stderr index fce13d6f17f06..36384178d469b 100644 --- a/frame/support/test/tests/derive_no_bound_ui/eq.stderr +++ b/frame/support/test/tests/derive_no_bound_ui/eq.stderr @@ -7,6 +7,6 @@ error[E0277]: can't compare `Foo` with `Foo` ::: $RUST/core/src/cmp.rs | | pub trait Eq: PartialEq { - | --------------- required by this bound in `std::cmp::Eq` + | --------------- required by this bound in `Eq` | = help: the trait `PartialEq` is not implemented for `Foo` From 9e6b5599b50c7d25689621cbbb9bc6d91910a944 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 26 May 2021 17:22:20 +0200 Subject: [PATCH 37/86] update tests from master branch --- .../test/tests/ui/impl_incorrect_method_signature.stderr | 4 ++-- .../api/test/tests/ui/mock_only_self_reference.stderr | 8 ++++---- .../ui/type_reference_in_impl_runtime_apis_call.stderr | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr b/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr index 6b00b7268672f..fcda69533e3ad 100644 --- a/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr +++ b/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr @@ -23,8 +23,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr 17 | sp_api::impl_runtime_apis! { | ^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found struct `std::string::String` | - = note: expected fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> Result<_, _>` - found fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> Result<_, _>` + = note: expected fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> std::result::Result<_, _>` + found fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> std::result::Result<_, _>` = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) error[E0308]: mismatched types diff --git a/primitives/api/test/tests/ui/mock_only_self_reference.stderr b/primitives/api/test/tests/ui/mock_only_self_reference.stderr index 83cfcf6ca1f9e..73cf936103798 100644 --- a/primitives/api/test/tests/ui/mock_only_self_reference.stderr +++ b/primitives/api/test/tests/ui/mock_only_self_reference.stderr @@ -24,8 +24,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr 12 | sp_api::mock_impl_runtime_apis! { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `()` | - = note: expected fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option, Vec<_>) -> Result<_, _>` - found fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option<()>, Vec<_>) -> Result<_, _>` + = note: expected fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option, Vec<_>) -> std::result::Result<_, _>` + found fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option<()>, Vec<_>) -> std::result::Result<_, _>` = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) error[E0053]: method `Api_test2_runtime_api_impl` has an incompatible type for trait @@ -42,6 +42,6 @@ error[E0053]: method `Api_test2_runtime_api_impl` has an incompatible type for t 12 | sp_api::mock_impl_runtime_apis! { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `()` | - = note: expected fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option, Vec<_>) -> Result<_, _>` - found fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option<()>, Vec<_>) -> Result<_, _>` + = note: expected fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option, Vec<_>) -> std::result::Result<_, _>` + found fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option<()>, Vec<_>) -> std::result::Result<_, _>` = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr b/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr index 689723f8d7509..71f12b415a2b5 100644 --- a/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr +++ b/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr @@ -23,8 +23,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr 17 | sp_api::impl_runtime_apis! { | ^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `&u64` | - = note: expected fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> Result<_, _>` - found fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option<&u64>, Vec<_>) -> Result<_, _>` + = note: expected fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> std::result::Result<_, _>` + found fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option<&u64>, Vec<_>) -> std::result::Result<_, _>` = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) error[E0308]: mismatched types From 6f9cf1067f8e3a35ef27e397cd3a1fd3a6d228e2 Mon Sep 17 00:00:00 2001 From: Parity Bot Date: Thu, 27 May 2021 12:32:54 +0000 Subject: [PATCH 38/86] cargo run --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_election_provider_multi_phase --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/election-provider-multi-phase/src/weights.rs --template=./.maintain/frame-weight-template.hbs --- .../src/weights.rs | 232 ++++++++++++++---- 1 file changed, 187 insertions(+), 45 deletions(-) diff --git a/frame/election-provider-multi-phase/src/weights.rs b/frame/election-provider-multi-phase/src/weights.rs index bff6798d7a2f5..14d8a7cffc5cd 100644 --- a/frame/election-provider-multi-phase/src/weights.rs +++ b/frame/election-provider-multi-phase/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_election_provider_multi_phase //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0 -//! DATE: 2021-04-12, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-05-27, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -54,141 +54,283 @@ pub trait WeightInfo { fn submit(c: u32, ) -> Weight; fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight; fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; + fn on_initialize_nothing() -> Weight; + fn on_initialize_open_signed() -> Weight; + fn on_initialize_open_unsigned_with_snapshot() -> Weight; + fn finalize_signed_phase_accept_solution() -> Weight; + fn finalize_signed_phase_reject_solution() -> Weight; + fn on_initialize_open_unsigned_without_snapshot() -> Weight; + fn elect_queued() -> Weight; + fn submit(c: u32, ) -> Weight; + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight; + fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; } /// Weights for pallet_election_provider_multi_phase using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { fn on_initialize_nothing() -> Weight { - (23_021_000 as Weight) + (28_535_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) } fn on_initialize_open_signed() -> Weight { - (112_730_000 as Weight) + (139_293_000 as Weight) .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } fn on_initialize_open_unsigned_with_snapshot() -> Weight { - (112_818_000 as Weight) + (139_126_000 as Weight) .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } fn finalize_signed_phase_accept_solution() -> Weight { - (46_382_000 as Weight) + (52_340_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } fn finalize_signed_phase_reject_solution() -> Weight { - (20_567_000 as Weight) + (23_222_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn on_initialize_open_unsigned_without_snapshot() -> Weight { - (21_354_000 as Weight) + (25_593_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn elect_queued() -> Weight { - (7_833_572_000 as Weight) + (14_697_445_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) } fn submit(c: u32, ) -> Weight { - (87_180_000 as Weight) - // Standard Error: 24_000 - .saturating_add((3_775_000 as Weight).saturating_mul(c as Weight)) + (94_334_000 as Weight) + // Standard Error: 135_000 + .saturating_add((2_100_000 as Weight).saturating_mul(c as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } - fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) // Standard Error: 20_000 - .saturating_add((4_209_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((4_865_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 67_000 + .saturating_add((49_000 as Weight).saturating_mul(t as Weight)) // Standard Error: 20_000 - .saturating_add((13_422_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 104_000 - .saturating_add((3_035_000 as Weight).saturating_mul(d as Weight)) - .saturating_add(T::DbWeight::get().reads(6 as Weight)) + .saturating_add((13_016_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 101_000 + .saturating_add((2_903_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) // Standard Error: 11_000 - .saturating_add((4_381_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 39_000 - .saturating_add((584_000 as Weight).saturating_mul(t as Weight)) + .saturating_add((5_006_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 38_000 + .saturating_add((549_000 as Weight).saturating_mul(t as Weight)) // Standard Error: 11_000 - .saturating_add((10_188_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 59_000 - .saturating_add((3_756_000 as Weight).saturating_mul(d as Weight)) + .saturating_add((10_349_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 58_000 + .saturating_add((3_832_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + } + fn on_initialize_nothing() -> Weight { + (29_377_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + } + fn on_initialize_open_signed() -> Weight { + (141_740_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + fn on_initialize_open_unsigned_with_snapshot() -> Weight { + (141_477_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + fn finalize_signed_phase_accept_solution() -> Weight { + (52_297_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + fn finalize_signed_phase_reject_solution() -> Weight { + (23_345_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn on_initialize_open_unsigned_without_snapshot() -> Weight { + (25_085_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn elect_queued() -> Weight { + (14_719_594_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) + } + fn submit(c: u32, ) -> Weight { + (89_088_000 as Weight) + // Standard Error: 100_000 + .saturating_add((2_852_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 20_000 + .saturating_add((4_825_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 20_000 + .saturating_add((13_016_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 103_000 + .saturating_add((3_318_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 11_000 + .saturating_add((4_991_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 37_000 + .saturating_add((476_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 11_000 + .saturating_add((10_349_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 56_000 + .saturating_add((3_738_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) } } // For backwards compatibility and tests impl WeightInfo for () { fn on_initialize_nothing() -> Weight { - (23_021_000 as Weight) + (28_535_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) } fn on_initialize_open_signed() -> Weight { - (112_730_000 as Weight) + (139_293_000 as Weight) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } fn on_initialize_open_unsigned_with_snapshot() -> Weight { - (112_818_000 as Weight) + (139_126_000 as Weight) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } fn finalize_signed_phase_accept_solution() -> Weight { - (46_382_000 as Weight) + (52_340_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } fn finalize_signed_phase_reject_solution() -> Weight { - (20_567_000 as Weight) + (23_222_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn on_initialize_open_unsigned_without_snapshot() -> Weight { - (21_354_000 as Weight) + (25_593_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn elect_queued() -> Weight { - (7_833_572_000 as Weight) + (14_697_445_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } fn submit(c: u32, ) -> Weight { - (87_180_000 as Weight) - // Standard Error: 24_000 - .saturating_add((3_775_000 as Weight).saturating_mul(c as Weight)) + (94_334_000 as Weight) + // Standard Error: 135_000 + .saturating_add((2_100_000 as Weight).saturating_mul(c as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } - fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) // Standard Error: 20_000 - .saturating_add((4_209_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((4_865_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 67_000 + .saturating_add((49_000 as Weight).saturating_mul(t as Weight)) // Standard Error: 20_000 - .saturating_add((13_422_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 104_000 - .saturating_add((3_035_000 as Weight).saturating_mul(d as Weight)) - .saturating_add(RocksDbWeight::get().reads(6 as Weight)) + .saturating_add((13_016_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 101_000 + .saturating_add((2_903_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) // Standard Error: 11_000 - .saturating_add((4_381_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 39_000 - .saturating_add((584_000 as Weight).saturating_mul(t as Weight)) + .saturating_add((5_006_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 38_000 + .saturating_add((549_000 as Weight).saturating_mul(t as Weight)) // Standard Error: 11_000 - .saturating_add((10_188_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 59_000 - .saturating_add((3_756_000 as Weight).saturating_mul(d as Weight)) + .saturating_add((10_349_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 58_000 + .saturating_add((3_832_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + } + fn on_initialize_nothing() -> Weight { + (29_377_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + } + fn on_initialize_open_signed() -> Weight { + (141_740_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + fn on_initialize_open_unsigned_with_snapshot() -> Weight { + (141_477_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + fn finalize_signed_phase_accept_solution() -> Weight { + (52_297_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + fn finalize_signed_phase_reject_solution() -> Weight { + (23_345_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn on_initialize_open_unsigned_without_snapshot() -> Weight { + (25_085_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn elect_queued() -> Weight { + (14_719_594_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + } + fn submit(c: u32, ) -> Weight { + (89_088_000 as Weight) + // Standard Error: 100_000 + .saturating_add((2_852_000 as Weight).saturating_mul(c as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 20_000 + .saturating_add((4_825_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 20_000 + .saturating_add((13_016_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 103_000 + .saturating_add((3_318_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 11_000 + .saturating_add((4_991_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 37_000 + .saturating_add((476_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 11_000 + .saturating_add((10_349_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 56_000 + .saturating_add((3_738_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) } } From 134ab4f91d66472aec2fc2dd97a9693880f7c828 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 27 May 2021 15:53:49 +0200 Subject: [PATCH 39/86] remove duplicate definitions --- .../src/weights.rs | 142 ------------------ 1 file changed, 142 deletions(-) diff --git a/frame/election-provider-multi-phase/src/weights.rs b/frame/election-provider-multi-phase/src/weights.rs index 14d8a7cffc5cd..5f2a3f238f218 100644 --- a/frame/election-provider-multi-phase/src/weights.rs +++ b/frame/election-provider-multi-phase/src/weights.rs @@ -54,87 +54,11 @@ pub trait WeightInfo { fn submit(c: u32, ) -> Weight; fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight; fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; - fn on_initialize_nothing() -> Weight; - fn on_initialize_open_signed() -> Weight; - fn on_initialize_open_unsigned_with_snapshot() -> Weight; - fn finalize_signed_phase_accept_solution() -> Weight; - fn finalize_signed_phase_reject_solution() -> Weight; - fn on_initialize_open_unsigned_without_snapshot() -> Weight; - fn elect_queued() -> Weight; - fn submit(c: u32, ) -> Weight; - fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight; - fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; } /// Weights for pallet_election_provider_multi_phase using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { - fn on_initialize_nothing() -> Weight { - (28_535_000 as Weight) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) - } - fn on_initialize_open_signed() -> Weight { - (139_293_000 as Weight) - .saturating_add(T::DbWeight::get().reads(8 as Weight)) - .saturating_add(T::DbWeight::get().writes(4 as Weight)) - } - fn on_initialize_open_unsigned_with_snapshot() -> Weight { - (139_126_000 as Weight) - .saturating_add(T::DbWeight::get().reads(8 as Weight)) - .saturating_add(T::DbWeight::get().writes(4 as Weight)) - } - fn finalize_signed_phase_accept_solution() -> Weight { - (52_340_000 as Weight) - .saturating_add(T::DbWeight::get().reads(1 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) - } - fn finalize_signed_phase_reject_solution() -> Weight { - (23_222_000 as Weight) - .saturating_add(T::DbWeight::get().reads(1 as Weight)) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) - } - fn on_initialize_open_unsigned_without_snapshot() -> Weight { - (25_593_000 as Weight) - .saturating_add(T::DbWeight::get().reads(1 as Weight)) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) - } - fn elect_queued() -> Weight { - (14_697_445_000 as Weight) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) - .saturating_add(T::DbWeight::get().writes(6 as Weight)) - } - fn submit(c: u32, ) -> Weight { - (94_334_000 as Weight) - // Standard Error: 135_000 - .saturating_add((2_100_000 as Weight).saturating_mul(c as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) - } - fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { - (0 as Weight) - // Standard Error: 20_000 - .saturating_add((4_865_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 67_000 - .saturating_add((49_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 20_000 - .saturating_add((13_016_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 101_000 - .saturating_add((2_903_000 as Weight).saturating_mul(d as Weight)) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) - } - fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { - (0 as Weight) - // Standard Error: 11_000 - .saturating_add((5_006_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 38_000 - .saturating_add((549_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 11_000 - .saturating_add((10_349_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 58_000 - .saturating_add((3_832_000 as Weight).saturating_mul(d as Weight)) - .saturating_add(T::DbWeight::get().reads(4 as Weight)) - } fn on_initialize_nothing() -> Weight { (29_377_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) @@ -203,72 +127,6 @@ impl WeightInfo for SubstrateWeight { // For backwards compatibility and tests impl WeightInfo for () { - fn on_initialize_nothing() -> Weight { - (28_535_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) - } - fn on_initialize_open_signed() -> Weight { - (139_293_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(8 as Weight)) - .saturating_add(RocksDbWeight::get().writes(4 as Weight)) - } - fn on_initialize_open_unsigned_with_snapshot() -> Weight { - (139_126_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(8 as Weight)) - .saturating_add(RocksDbWeight::get().writes(4 as Weight)) - } - fn finalize_signed_phase_accept_solution() -> Weight { - (52_340_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) - } - fn finalize_signed_phase_reject_solution() -> Weight { - (23_222_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) - } - fn on_initialize_open_unsigned_without_snapshot() -> Weight { - (25_593_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) - } - fn elect_queued() -> Weight { - (14_697_445_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) - .saturating_add(RocksDbWeight::get().writes(6 as Weight)) - } - fn submit(c: u32, ) -> Weight { - (94_334_000 as Weight) - // Standard Error: 135_000 - .saturating_add((2_100_000 as Weight).saturating_mul(c as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) - } - fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { - (0 as Weight) - // Standard Error: 20_000 - .saturating_add((4_865_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 67_000 - .saturating_add((49_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 20_000 - .saturating_add((13_016_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 101_000 - .saturating_add((2_903_000 as Weight).saturating_mul(d as Weight)) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) - } - fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { - (0 as Weight) - // Standard Error: 11_000 - .saturating_add((5_006_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 38_000 - .saturating_add((549_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 11_000 - .saturating_add((10_349_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 58_000 - .saturating_add((3_832_000 as Weight).saturating_mul(d as Weight)) - .saturating_add(RocksDbWeight::get().reads(4 as Weight)) - } fn on_initialize_nothing() -> Weight { (29_377_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) From 61b76800b8c2c1b5f7c7d68399b3bedd58943d51 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 28 May 2021 13:47:25 +0200 Subject: [PATCH 40/86] remove signed reward factor due to its attack potential --- bin/node/runtime/src/lib.rs | 1 - .../election-provider-multi-phase/src/lib.rs | 3 --- .../election-provider-multi-phase/src/mock.rs | 5 +---- .../src/signed.rs | 19 +++++++++---------- 4 files changed, 10 insertions(+), 18 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 0699fb2145cc6..aa30207f705df 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -557,7 +557,6 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type MinerTxPriority = MultiPhaseUnsignedPriority; type SignedMaxSubmissions = SignedMaxSubmissions; type SignedRewardBase = SignedRewardBase; - type SignedRewardFactor = (); // no score-based reward type SignedRewardMax = SignedRewardBase; type SignedDepositBase = SignedDepositBase; type SignedDepositByte = SignedDepositByte; diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 290ab2cac8dd4..249c946b50d54 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -571,9 +571,6 @@ pub mod pallet { /// Base reward for a signed solution #[pallet::constant] type SignedRewardBase: Get>; - /// Per-score reward for a signed solution. - #[pallet::constant] - type SignedRewardFactor: Get; /// Maximum cap for a signed solution. #[pallet::constant] type SignedRewardMax: Get>>; diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index 2b1c5a095af88..7240efa234c6b 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -263,7 +263,6 @@ parameter_types! { pub static SignedDepositByte: Balance = 0; pub static SignedDepositWeight: Balance = 0; pub static SignedRewardBase: Balance = 7; - pub static SignedRewardFactor: Perbill = Perbill::zero(); pub static SignedRewardMax: Balance = 10; pub static SignedMaxWeight: Weight = BlockWeights::get().max_block; pub static MinerMaxIterations: u32 = 5; @@ -368,7 +367,6 @@ impl crate::Config for Runtime { type MinerMaxLength = MinerMaxLength; type MinerTxPriority = MinerTxPriority; type SignedRewardBase = SignedRewardBase; - type SignedRewardFactor = SignedRewardFactor; type SignedRewardMax = SignedRewardMax; type SignedDepositBase = SignedDepositBase; type SignedDepositByte = (); @@ -485,9 +483,8 @@ impl ExtBuilder { ::set(weight); self } - pub fn reward(self, base: u64, factor: Perbill, max: u64) -> Self { + pub fn reward(self, base: u64, max: u64) -> Self { ::set(base); - ::set(factor); ::set(max); self } diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index e80ccadcda88d..050e6afa7e63c 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -202,7 +202,7 @@ impl Pallet { } // add to the designated spot. If the length is too much, remove one. - let reward = Self::reward_for(&solution); + let reward = Self::reward_for(); let deposit = Self::deposit_for(&solution, size); let submission = SignedSubmission { who: who.clone(), deposit, reward, solution }; @@ -256,9 +256,8 @@ impl Pallet { /// The reward for this solution, if successfully chosen as the best one at the end of the /// signed phase. - pub fn reward_for(solution: &RawSolution>) -> BalanceOf { - let raw_reward = T::SignedRewardBase::get() - + T::SignedRewardFactor::get() * solution.score[0].saturated_into::>(); + pub fn reward_for() -> BalanceOf { + let raw_reward = T::SignedRewardBase::get(); match T::SignedRewardMax::get() { Some(cap) => raw_reward.min(cap), @@ -357,7 +356,7 @@ mod tests { #[test] fn reward_is_capped() { - ExtBuilder::default().reward(5, Perbill::from_percent(25), 10).build_and_execute(|| { + ExtBuilder::default().reward(5, 10).build_and_execute(|| { roll_to(15); assert!(MultiPhase::current_phase().is_signed()); @@ -369,11 +368,11 @@ mod tests { assert_eq!(balances(&99), (95, 5)); assert!(MultiPhase::finalize_signed_phase().0); - // expected reward is 5 + 10 - assert_eq!(balances(&99), (100 + 10, 0)); + // expected reward is 5 + assert_eq!(balances(&99), (100 + 5, 0)); }); - ExtBuilder::default().reward(5, Perbill::from_percent(25), 20).build_and_execute(|| { + ExtBuilder::default().reward(5, 20).build_and_execute(|| { roll_to(15); assert!(MultiPhase::current_phase().is_signed()); @@ -385,8 +384,8 @@ mod tests { assert_eq!(balances(&99), (95, 5)); assert!(MultiPhase::finalize_signed_phase().0); - // expected reward is 5 + 10 - assert_eq!(balances(&99), (100 + 15, 0)); + // expected reward is 5 + assert_eq!(balances(&99), (100 + 5, 0)); }); } From b06f3a8bcbac4cc85b29a93f2ac2e92b19859911 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 28 May 2021 14:23:39 +0200 Subject: [PATCH 41/86] Update frame/election-provider-multi-phase/src/signed.rs Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/election-provider-multi-phase/src/signed.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 050e6afa7e63c..de08b371d96c5 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -315,7 +315,7 @@ mod tests { // now try and cheat by passing a lower queue length assert_noop!( - MultiPhase::submit(Origin::signed(99), solution, 0,), + MultiPhase::submit(Origin::signed(99), solution, 0), Error::::SignedInvalidWitness, ); }) From ca976036072a4cb1c107b2e930ad02aadf8a85f8 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 28 May 2021 14:31:39 +0200 Subject: [PATCH 42/86] remove SignedRewardMax; no longer necessary --- bin/node/runtime/src/lib.rs | 1 - .../election-provider-multi-phase/src/lib.rs | 3 -- .../election-provider-multi-phase/src/mock.rs | 1 - .../src/signed.rs | 48 +------------------ 4 files changed, 1 insertion(+), 52 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index aa30207f705df..3b4bc03d51f2e 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -557,7 +557,6 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type MinerTxPriority = MultiPhaseUnsignedPriority; type SignedMaxSubmissions = SignedMaxSubmissions; type SignedRewardBase = SignedRewardBase; - type SignedRewardMax = SignedRewardBase; type SignedDepositBase = SignedDepositBase; type SignedDepositByte = SignedDepositByte; type SignedDepositWeight = (); diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 249c946b50d54..e71c1d8e61411 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -571,9 +571,6 @@ pub mod pallet { /// Base reward for a signed solution #[pallet::constant] type SignedRewardBase: Get>; - /// Maximum cap for a signed solution. - #[pallet::constant] - type SignedRewardMax: Get>>; /// Base deposit for a signed solution. #[pallet::constant] diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index 7240efa234c6b..0fe9133fb91e3 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -367,7 +367,6 @@ impl crate::Config for Runtime { type MinerMaxLength = MinerMaxLength; type MinerTxPriority = MinerTxPriority; type SignedRewardBase = SignedRewardBase; - type SignedRewardMax = SignedRewardMax; type SignedDepositBase = SignedDepositBase; type SignedDepositByte = (); type SignedDepositWeight = (); diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index de08b371d96c5..393a39642710b 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -202,7 +202,7 @@ impl Pallet { } // add to the designated spot. If the length is too much, remove one. - let reward = Self::reward_for(); + let reward = T::SignedRewardBase::get(); let deposit = Self::deposit_for(&solution, size); let submission = SignedSubmission { who: who.clone(), deposit, reward, solution }; @@ -253,17 +253,6 @@ impl Pallet { T::SignedDepositBase::get() + len_deposit + weight_deposit } - - /// The reward for this solution, if successfully chosen as the best one at the end of the - /// signed phase. - pub fn reward_for() -> BalanceOf { - let raw_reward = T::SignedRewardBase::get(); - - match T::SignedRewardMax::get() { - Some(cap) => raw_reward.min(cap), - None => raw_reward, - } - } } #[cfg(test)] @@ -354,41 +343,6 @@ mod tests { }) } - #[test] - fn reward_is_capped() { - ExtBuilder::default().reward(5, 10).build_and_execute(|| { - roll_to(15); - assert!(MultiPhase::current_phase().is_signed()); - - let solution = raw_solution(); - assert_eq!(solution.score[0], 40); - assert_eq!(balances(&99), (100, 0)); - - assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(balances(&99), (95, 5)); - - assert!(MultiPhase::finalize_signed_phase().0); - // expected reward is 5 - assert_eq!(balances(&99), (100 + 5, 0)); - }); - - ExtBuilder::default().reward(5, 20).build_and_execute(|| { - roll_to(15); - assert!(MultiPhase::current_phase().is_signed()); - - let solution = raw_solution(); - assert_eq!(solution.score[0], 40); - assert_eq!(balances(&99), (100, 0)); - - assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(balances(&99), (95, 5)); - - assert!(MultiPhase::finalize_signed_phase().0); - // expected reward is 5 - assert_eq!(balances(&99), (100 + 5, 0)); - }); - } - #[test] fn bad_solution_is_slashed() { ExtBuilder::default().build_and_execute(|| { From 7ce5afb561bd6ae2c7b905fb3a3969bd79578498 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 28 May 2021 14:39:37 +0200 Subject: [PATCH 43/86] compute the encoded size without actually encoding --- frame/election-provider-multi-phase/src/signed.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 393a39642710b..94b0f70b739a9 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -245,7 +245,8 @@ impl Pallet { solution: &RawSolution>, size: SolutionOrSnapshotSize, ) -> BalanceOf { - let encoded_len: BalanceOf = solution.using_encoded(|e| e.len() as u32).into(); + let encoded_len: u32 = solution.encoded_size().saturated_into(); + let encoded_len: BalanceOf = encoded_len.into(); let feasibility_weight = Self::feasibility_weight_of(solution, size); let len_deposit = T::SignedDepositByte::get() * encoded_len; From bdf45bac190e4525c27bc6487b74c7038815853f Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 28 May 2021 14:47:34 +0200 Subject: [PATCH 44/86] remove unused PostInfo --- frame/election-provider-multi-phase/src/lib.rs | 4 ++-- frame/election-provider-multi-phase/src/signed.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index e71c1d8e61411..17f28d4ae8b66 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -783,7 +783,7 @@ pub mod pallet { origin: OriginFor, solution: RawSolution>, witness_data: u32, - ) -> DispatchResultWithPostInfo { + ) -> DispatchResult { let who = ensure_signed(origin)?; // ensure witness data is correct. @@ -826,7 +826,7 @@ pub mod pallet { // store the new signed submission. >::put(signed_submissions); Self::deposit_event(Event::SolutionStored(ElectionCompute::Signed)); - Ok(None.into()) + Ok(()) } /// Submit a solution for the unsigned phase. diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 94b0f70b739a9..107ea2cfcaefa 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -266,12 +266,12 @@ mod tests { SignedMaxSubmissions, SignedMaxWeight, }, }; - use frame_support::{dispatch::DispatchResultWithPostInfo, assert_noop, assert_ok}; + use frame_support::{dispatch::DispatchResult, assert_noop, assert_ok}; fn submit_with_witness( origin: Origin, solution: RawSolution>, - ) -> DispatchResultWithPostInfo { + ) -> DispatchResult { MultiPhase::submit(origin, solution, MultiPhase::signed_submissions().len() as u32) } From 87b7ff1c6f1d8c81e260d857efb7bc2722c33315 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 28 May 2021 14:49:15 +0200 Subject: [PATCH 45/86] pub use some stuff Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/election-provider-multi-phase/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 17f28d4ae8b66..520ca252fecc1 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -255,7 +255,7 @@ pub mod weights; pub use weights::WeightInfo; -use signed::{SignedSubmission, BalanceOf, NegativeImbalanceOf, PositiveImbalanceOf}; +pub use signed::{SignedSubmission, BalanceOf, NegativeImbalanceOf, PositiveImbalanceOf}; /// The compact solution type used by this crate. pub type CompactOf = ::CompactSolution; From 308b522170b34bc6b55227ea3224fb1db7abbd63 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 28 May 2021 16:29:40 +0200 Subject: [PATCH 46/86] ensure `pub use` things are in fact `pub` --- frame/election-provider-multi-phase/src/signed.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 107ea2cfcaefa..2f7eb7f96e6bd 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -43,12 +43,12 @@ pub struct SignedSubmission { pub solution: RawSolution, } -pub(crate) type BalanceOf = +pub type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; -pub(crate) type PositiveImbalanceOf = <::Currency as Currency< +pub type PositiveImbalanceOf = <::Currency as Currency< ::AccountId, >>::PositiveImbalance; -pub(crate) type NegativeImbalanceOf = <::Currency as Currency< +pub type NegativeImbalanceOf = <::Currency as Currency< ::AccountId, >>::NegativeImbalance; From 0d892f443e05fd96b59ee1ac931fde2ce24d4a52 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 28 May 2021 16:40:20 +0200 Subject: [PATCH 47/86] add event information: was another solution ejected to make room --- frame/election-provider-multi-phase/src/lib.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 520ca252fecc1..24c7ffd0b95dd 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -809,6 +809,8 @@ pub mod pallet { // ensure solution claims is better. let mut signed_submissions = Self::signed_submissions(); + let ejected_a_solution = signed_submissions.len() + == T::SignedMaxSubmissions::get().saturated_into::(); let index = Self::insert_submission(&who, &mut signed_submissions, solution, size) .ok_or(Error::::SignedQueueFull)?; @@ -825,7 +827,7 @@ pub mod pallet { // store the new signed submission. >::put(signed_submissions); - Self::deposit_event(Event::SolutionStored(ElectionCompute::Signed)); + Self::deposit_event(Event::SolutionStored(ElectionCompute::Signed, ejected_a_solution)); Ok(()) } @@ -875,8 +877,12 @@ pub mod pallet { // store the newly received solution. log!(info, "queued unsigned solution with score {:?}", ready.score); + let ejected_a_solution = >::exists(); >::put(ready); - Self::deposit_event(Event::SolutionStored(ElectionCompute::Unsigned)); + Self::deposit_event(Event::SolutionStored( + ElectionCompute::Unsigned, + ejected_a_solution, + )); Ok(None.into()) } @@ -905,7 +911,9 @@ pub mod pallet { /// /// If the solution is signed, this means that it hasn't yet been processed. If the /// solution is unsigned, this means that it has also been processed. - SolutionStored(ElectionCompute), + /// + /// The `bool` is `true` when a previous solution was ejected to make room for this one. + SolutionStored(ElectionCompute, bool), /// The election has been finalized, with `Some` of the given computation, or else if the /// election failed, `None`. ElectionFinalized(Option), From ea2b1abd2f32aac0a640f22ee9b7a0210dfb0e25 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 28 May 2021 16:54:11 +0200 Subject: [PATCH 48/86] unconditionally run the unsigned phase even if signed was successful --- frame/election-provider-multi-phase/src/lib.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 24c7ffd0b95dd..f608fa24a0ef0 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -658,13 +658,14 @@ pub mod pallet { // // Notes: // - // - If the signed phase produced a viable solution, we disable the unsigned - // phase. We want to prioritize signed solutions whenever they're available. // - `Self::finalize_signed_phase()` also appears in `fn do_elect`. This is // a guard against the case that `elect` is called prematurely. This adds // a small amount of overhead, but that is unfortunately unavoidable. - let (success, weight) = Self::finalize_signed_phase(); - (false, !success, weight) + let (_success, weight) = Self::finalize_signed_phase(); + // In the future we can consider disabling the unsigned phase if the signed + // phase completes successfully, but for now we're enabling it unconditionally + // as a defensive measure. + (false, true, weight) } else { // no signed phase: create a new snapshot, definitely `enable` the unsigned // phase. From 22248e175cf6d940db5311a2f271871cc2aca92b Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 28 May 2021 17:02:40 +0200 Subject: [PATCH 49/86] remove dead test code --- frame/election-provider-multi-phase/src/mock.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index 0fe9133fb91e3..323f33f6f93bf 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -482,11 +482,6 @@ impl ExtBuilder { ::set(weight); self } - pub fn reward(self, base: u64, max: u64) -> Self { - ::set(base); - ::set(max); - self - } pub fn signed_weight(self, weight: Weight) -> Self { ::set(weight); self From c651962ea1df495bfbdb9f19051ef1980bfd7aac Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 31 May 2021 09:56:44 +0200 Subject: [PATCH 50/86] meaningful witness data name --- frame/election-provider-multi-phase/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index f608fa24a0ef0..cd0e616fe839c 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -779,17 +779,17 @@ pub mod pallet { /// # /// Queue size must be provided as witness data. /// # - #[pallet::weight(T::WeightInfo::submit(*witness_data))] + #[pallet::weight(T::WeightInfo::submit(*num_signed_submissions))] pub fn submit( origin: OriginFor, solution: RawSolution>, - witness_data: u32, + num_signed_submissions: u32, ) -> DispatchResult { let who = ensure_signed(origin)?; // ensure witness data is correct. ensure!( - witness_data >= >::decode_len().unwrap_or_default() as u32, + num_signed_submissions >= >::decode_len().unwrap_or_default() as u32, Error::::SignedInvalidWitness, ); From 29d9242c5795a7fdeda9388ea6ee189792c879c2 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 31 May 2021 10:15:55 +0200 Subject: [PATCH 51/86] use errors instead of defensive `unwrap_or_default` --- .../src/benchmarking.rs | 2 +- frame/election-provider-multi-phase/src/lib.rs | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/frame/election-provider-multi-phase/src/benchmarking.rs b/frame/election-provider-multi-phase/src/benchmarking.rs index fa9a1f7a3a126..b2f1c9bb4c8a6 100644 --- a/frame/election-provider-multi-phase/src/benchmarking.rs +++ b/frame/election-provider-multi-phase/src/benchmarking.rs @@ -270,7 +270,7 @@ frame_benchmarking::benchmarks! { ..Default::default() }; - >::put(Phase::Signed); + MultiPhase::::on_initialize_open_signed().expect("should be ok to start signed phase"); >::put(1); >::mutate(|queue| { diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index cd0e616fe839c..58c470d599f82 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -800,8 +800,8 @@ pub mod pallet { // because could do just decode_len. But we can create abstractions to do this. // build size. Note: this is not needed for weight calc, thus not input. - // defensive-only: if phase is signed, snapshot will exist. - let size = Self::snapshot_metadata().unwrap_or_default(); + // unlikely to ever return an error: if phase is signed, snapshot will exist. + let size = Self::snapshot_metadata().ok_or(Error::::MissingSnapshotMetadata)?; ensure!( Self::feasibility_weight_of(&solution, size) < T::SignedMaxWeight::get(), @@ -816,8 +816,10 @@ pub mod pallet { .ok_or(Error::::SignedQueueFull)?; // collect deposit. Thereafter, the function cannot fail. - // Defensive -- index is valid. - let deposit = signed_submissions.get(index).map(|s| s.deposit).unwrap_or_default(); + let deposit = signed_submissions + .get(index) + .map(|s| s.deposit) + .ok_or(Error::::InvalidSubmissionIndex)?; T::Currency::reserve(&who, deposit).map_err(|_| Error::::SignedCannotPayDeposit)?; log!( @@ -947,6 +949,10 @@ pub mod pallet { SignedTooMuchWeight, /// OCW submitted solution for wrong round OcwCallWrongEra, + /// Snapshot metadata should exist but didn't. + MissingSnapshotMetadata, + /// `Self::insert_submission` returned an invalid index. + InvalidSubmissionIndex, } #[pallet::origin] From fd383561da9b0fe655f986cd86e231583c78e6a2 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 31 May 2021 10:20:16 +0200 Subject: [PATCH 52/86] get rid of a log message redundant with an event --- frame/election-provider-multi-phase/src/lib.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 58c470d599f82..164d2cf61bcb1 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -822,12 +822,6 @@ pub mod pallet { .ok_or(Error::::InvalidSubmissionIndex)?; T::Currency::reserve(&who, deposit).map_err(|_| Error::::SignedCannotPayDeposit)?; - log!( - info, - "queued signed solution with (claimed) score {:?}", - signed_submissions.get(index).map(|s| s.solution.score).unwrap_or_default() - ); - // store the new signed submission. >::put(signed_submissions); Self::deposit_event(Event::SolutionStored(ElectionCompute::Signed, ejected_a_solution)); From c503fa763b38809b1dd7d875cc9be068e01b61ba Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 31 May 2021 10:38:54 +0200 Subject: [PATCH 53/86] saturating math Co-authored-by: Shawn Tabrizi --- frame/election-provider-multi-phase/src/signed.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 2f7eb7f96e6bd..e83bd96fcf391 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -249,10 +249,10 @@ impl Pallet { let encoded_len: BalanceOf = encoded_len.into(); let feasibility_weight = Self::feasibility_weight_of(solution, size); - let len_deposit = T::SignedDepositByte::get() * encoded_len; - let weight_deposit = T::SignedDepositWeight::get() * feasibility_weight.saturated_into(); + let len_deposit = T::SignedDepositByte::get().saturating_mul(encoded_len); + let weight_deposit = T::SignedDepositWeight::get().saturating_mul(feasibility_weight.saturated_into()); - T::SignedDepositBase::get() + len_deposit + weight_deposit + T::SignedDepositBase::get().saturating_add(len_deposit).saturating_add(weight_deposit) } } From 252f310b036f3f4a7943a7e475349fe4a14db55c Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 31 May 2021 10:40:22 +0200 Subject: [PATCH 54/86] import Saturating --- frame/election-provider-multi-phase/src/signed.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index e83bd96fcf391..b40acba1770ef 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -25,7 +25,7 @@ use codec::{Encode, Decode, HasCompact}; use frame_support::traits::{Currency, Get, OnUnbalanced, ReservableCurrency}; use sp_arithmetic::traits::SaturatedConversion; use sp_npos_elections::{is_score_better, CompactSolution}; -use sp_runtime::{Perbill, RuntimeDebug, traits::Zero}; +use sp_runtime::{Perbill, RuntimeDebug, traits::{Saturating, Zero}}; use sp_std::{cmp::Ordering, vec::Vec}; /// A raw, unchecked signed submission. From 725cadfe6116a5f5a4446409779242500d7bd6a1 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 31 May 2021 10:44:17 +0200 Subject: [PATCH 55/86] mv `fn submit` to end of call --- .../election-provider-multi-phase/src/lib.rs | 124 +++++++++--------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 164d2cf61bcb1..2d8fee6b1e2ef 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -766,68 +766,6 @@ pub mod pallet { #[pallet::call] impl Pallet { - /// Submit a solution for the signed phase. - /// - /// The dispatch origin fo this call must be __signed__. - /// - /// The solution is potentially queued, based on the claimed score and processed at the end - /// of the signed phase. - /// - /// A deposit is reserved and recorded for the solution. Based on the outcome, the solution - /// might be rewarded, slashed, or get all or a part of the deposit back. - /// - /// # - /// Queue size must be provided as witness data. - /// # - #[pallet::weight(T::WeightInfo::submit(*num_signed_submissions))] - pub fn submit( - origin: OriginFor, - solution: RawSolution>, - num_signed_submissions: u32, - ) -> DispatchResult { - let who = ensure_signed(origin)?; - - // ensure witness data is correct. - ensure!( - num_signed_submissions >= >::decode_len().unwrap_or_default() as u32, - Error::::SignedInvalidWitness, - ); - - // ensure solution is timely. - ensure!(Self::current_phase().is_signed(), Error::::PreDispatchEarlySubmission); - - // NOTE: this is the only case where having separate snapshot would have been better - // because could do just decode_len. But we can create abstractions to do this. - - // build size. Note: this is not needed for weight calc, thus not input. - // unlikely to ever return an error: if phase is signed, snapshot will exist. - let size = Self::snapshot_metadata().ok_or(Error::::MissingSnapshotMetadata)?; - - ensure!( - Self::feasibility_weight_of(&solution, size) < T::SignedMaxWeight::get(), - Error::::SignedTooMuchWeight, - ); - - // ensure solution claims is better. - let mut signed_submissions = Self::signed_submissions(); - let ejected_a_solution = signed_submissions.len() - == T::SignedMaxSubmissions::get().saturated_into::(); - let index = Self::insert_submission(&who, &mut signed_submissions, solution, size) - .ok_or(Error::::SignedQueueFull)?; - - // collect deposit. Thereafter, the function cannot fail. - let deposit = signed_submissions - .get(index) - .map(|s| s.deposit) - .ok_or(Error::::InvalidSubmissionIndex)?; - T::Currency::reserve(&who, deposit).map_err(|_| Error::::SignedCannotPayDeposit)?; - - // store the new signed submission. - >::put(signed_submissions); - Self::deposit_event(Event::SolutionStored(ElectionCompute::Signed, ejected_a_solution)); - Ok(()) - } - /// Submit a solution for the unsigned phase. /// /// The dispatch origin fo this call must be __none__. @@ -898,6 +836,68 @@ pub mod pallet { >::set(maybe_next_score); Ok(()) } + + /// Submit a solution for the signed phase. + /// + /// The dispatch origin fo this call must be __signed__. + /// + /// The solution is potentially queued, based on the claimed score and processed at the end + /// of the signed phase. + /// + /// A deposit is reserved and recorded for the solution. Based on the outcome, the solution + /// might be rewarded, slashed, or get all or a part of the deposit back. + /// + /// # + /// Queue size must be provided as witness data. + /// # + #[pallet::weight(T::WeightInfo::submit(*num_signed_submissions))] + pub fn submit( + origin: OriginFor, + solution: RawSolution>, + num_signed_submissions: u32, + ) -> DispatchResult { + let who = ensure_signed(origin)?; + + // ensure witness data is correct. + ensure!( + num_signed_submissions >= >::decode_len().unwrap_or_default() as u32, + Error::::SignedInvalidWitness, + ); + + // ensure solution is timely. + ensure!(Self::current_phase().is_signed(), Error::::PreDispatchEarlySubmission); + + // NOTE: this is the only case where having separate snapshot would have been better + // because could do just decode_len. But we can create abstractions to do this. + + // build size. Note: this is not needed for weight calc, thus not input. + // unlikely to ever return an error: if phase is signed, snapshot will exist. + let size = Self::snapshot_metadata().ok_or(Error::::MissingSnapshotMetadata)?; + + ensure!( + Self::feasibility_weight_of(&solution, size) < T::SignedMaxWeight::get(), + Error::::SignedTooMuchWeight, + ); + + // ensure solution claims is better. + let mut signed_submissions = Self::signed_submissions(); + let ejected_a_solution = signed_submissions.len() + == T::SignedMaxSubmissions::get().saturated_into::(); + let index = Self::insert_submission(&who, &mut signed_submissions, solution, size) + .ok_or(Error::::SignedQueueFull)?; + + // collect deposit. Thereafter, the function cannot fail. + let deposit = signed_submissions + .get(index) + .map(|s| s.deposit) + .ok_or(Error::::InvalidSubmissionIndex)?; + T::Currency::reserve(&who, deposit).map_err(|_| Error::::SignedCannotPayDeposit)?; + + // store the new signed submission. + >::put(signed_submissions); + Self::deposit_event(Event::SolutionStored(ElectionCompute::Signed, ejected_a_solution)); + Ok(()) + } } #[pallet::event] From 88f856a2d05ac4b04b582d56207a3646ee90c8ea Mon Sep 17 00:00:00 2001 From: kianenigma Date: Tue, 1 Jun 2021 15:09:23 +0200 Subject: [PATCH 56/86] add log line --- frame/election-provider-multi-phase/src/signed.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index b40acba1770ef..e0f1a82919b92 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -111,6 +111,7 @@ impl Pallet { // Any unprocessed solution is pointless to even consider. Feasible or malicious, // they didn't end up being used. Unreserve the bonds. + let discarded = all_submissions.len(); for not_processed in all_submissions { let SignedSubmission { who, deposit, .. } = not_processed; let _remaining = T::Currency::unreserve(&who, deposit); @@ -118,6 +119,7 @@ impl Pallet { debug_assert!(_remaining.is_zero()); }; + log!(debug, "closed signed phase, found solution? {}, discarded {}", found_solution, discarded); (found_solution, weight) } From ce0974f8b15937ed1534203b1bd981f8b6ea23b7 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 18 Jun 2021 09:29:44 +0200 Subject: [PATCH 57/86] Use a better data structure for SignedSubmissions instead of Vec (#8933) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove: (#8748) * `NetworkStatusSinks` * `sc_service::SpawnTasksParams::network_status_sinks` Also: * `sc_service::build_network()` does not return `network_status_sinks` * CI: fix simnet trigger (#8927) * CI: chore * CI: pin simnet version * More sc-service config reexports (#8887) * Reexport ExecutionStrategies and ExecutionStrategy * Reexport more of the network * Reexport the ExecutionStrategy as it's used within ExecutionStrategies * Fix check runtime CI (#8930) * Fix check_runtime.sh script * contracts: Remove confusing "Related Modules" doc * Bump parity-wasm and pwasm-utils to the newest versions everywhere (#8928) * BROKEN: convert SignedSubmissions to BoundedBTreeSet Eventually, once it works, this change should improve overall performance. However, in the meantime, the trait bounds aren't playing nicely, and this is turning into too much of a pain to handle right now as part of /#7910. We can take care of it later. * Simple `MaxBoundedLen` Implementations (#8793) * implement max_values + storages info * some formatting + doc * sudo sanity check * timestamp * assets (not working) * fix assets * impl for proxy * update balances * rename StoragesInfo -> PalletStorageInfo * merge both StorageInfoTrait and PalletStorageInfo I think it is more future proof. In the future some storage could make use of multiple prefix. Like one to store how much value has been inserted, etc... * Update frame/support/procedural/src/storage/parse.rs Co-authored-by: Peter Goodspeed-Niklaus * Update frame/support/procedural/src/storage/storage_struct.rs Co-authored-by: Peter Goodspeed-Niklaus * Fix max_size using hasher information hasher now expose `max_len` which allows to computes their maximum len. For hasher without concatenation, it is the size of the hash part, for hasher with concatenation, it is the size of the hash part + max encoded len of the key. * fix tests * fix ui tests * Move `MaxBoundedLen` into its own crate (#8814) * move MaxEncodedLen into its own crate * remove MaxEncodedLen impl from frame-support * add to assets and balances * try more fixes * fix compile Co-authored-by: Shawn Tabrizi * nits * fix compile * line width * fix max-values-macro merge * Add some derive, needed for test and other purpose * use weak bounded vec in some cases * Update lib.rs * move max-encoded-len crate * fix * remove app crypto for now * width * Revert "remove app crypto for now" This reverts commit 73623e9933d50648e0e7fe90b6171a8e45d7f5a2. * unused variable * more unused variables * more fixes * Add #[max_encoded_len_crate(...)] helper attribute The purpose of this attribute is to reduce the surface area of max_encoded_len changes. Crates deriving `MaxEncodedLen` do not need to add it to `Cargo.toml`; they can instead just do ```rust \#[derive(Encode, MaxEncodedLen)] \#[max_encoded_len_crate(frame_support::max_encoded_len)] struct Example; ``` * fix a ui test * use #[max_encoded_len_crate(...)] helper in app_crypto * remove max_encoded_len import where not necessary * update lockfile * fix ui test * ui * newline * fix merge * try fix ui again * Update max-encoded-len/derive/src/lib.rs Co-authored-by: Peter Goodspeed-Niklaus * extract generate_crate_access_2018 * Update lib.rs * compiler isnt smart enough Co-authored-by: thiolliere Co-authored-by: Peter Goodspeed-Niklaus Co-authored-by: Peter Goodspeed-Niklaus * remove duplicate Issued/Burned events (#8935) * weather -> whether (#8938) * make remote ext use batch ws-client (#8916) * make remote ext use batch ws-client * Add debug log for key length * better assertions * new sanity_checl * try and make it work with batch * update test * remove exctra uri * add missing at * remove unused rpc stuff * improve Co-authored-by: emostov <32168567+emostov@users.noreply.github.com> * Make `Schedule` fields public to allow for customization (#8924) * Make `Schedule` fields public for customization * Fix doc typo Co-authored-by: Andrew Jones Co-authored-by: Andrew Jones * Session key should be settable at genesis even for non-endowed accounts (#8942) * Session key should be settable at genesis even for non-endowed accounts * Docs * Migrate pallet-scored-pool to pallet attribute macro (#8825) * Migrate pallet-scored-pool to pallet attribute macro. * Remove dummy event. * Apply review suggestions. * Bump retain_mut from 0.1.2 to 0.1.3 (#8951) Bumps [retain_mut](https://github.com/upsuper/retain_mut) from 0.1.2 to 0.1.3. - [Release notes](https://github.com/upsuper/retain_mut/releases) - [Commits](https://github.com/upsuper/retain_mut/compare/v0.1.2...v0.1.3) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Use correct CreateInherentDataProviders impl for manual seal (#8852) * use correct CreateInherentDataProviders impl for manual seal * add babe inherent provider * move client into factory fn * Refactor code a little bit (#8932) * Optimize `next_storage_key` (#8956) * Optimize `next_storage_key` - Do not rely on recursion - Use an iterator over the overlay to not always call the same method * Fix bug * Add deserialize for TransactionValidityError in std. (#8961) * Add deserialize for TransactionValidityError in std. * Fix derives * Bump getrandom from 0.2.2 to 0.2.3 (#8952) Bumps [getrandom](https://github.com/rust-random/getrandom) from 0.2.2 to 0.2.3. - [Release notes](https://github.com/rust-random/getrandom/releases) - [Changelog](https://github.com/rust-random/getrandom/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-random/getrandom/compare/v0.2.2...v0.2.3) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Allow usage of path in construct_runtime! (#8801) * Allow usage of path in construct_runtime! * Fix whitespace * Fix whitespace * Make expand_runtime_metadata accept slice instead of Iterator * Include Call and Event in construct_runtime for testing * Migrate impl_outer_event to proc macro * Fix integrity_test_works * Update UI test expectations * Factor in module path while generating enum variant or fn names * Use ParseStream::lookahead for more helpful error messages * Remove generating outer_event_metadata * Ensure pallets with different paths but same last path segment can coexist * Remove unnecessary generated function * Migrate decl_outer_config to proc macro * Add default_filter test for expand_outer_origin * Allow crate, self and super keywords to appear in pallet path * Add UI test for specifying empty pallet paths in construct_runtime * Reduce cargo doc warnings (#8947) Co-authored-by: Bastian Köcher * Update wasmtime to 0.27 (#8913) * Update wasmtime to 0.27 A couple of notes: - Now we are fair about unsafeness of runtime creation via an compiled artifact. This change was prompted by the change in wasmtime which made `deserialize` rightfully unsafe. Now `CodeSupplyMode` was hidden and the `create_runtime` now takes the blob again and there is now a new fn for creating a runtime with a compiled artifact. - This is a big change for wasmtime. They switched to the modern backend for code generation. While this can bring performance improvements, it can also introduce some problems. In fact, 0.27 fixed a serious issue that could lead to sandbox escape. Hence we need a proper burn in. This would require a change to PVF validation host as well. * Filter regalloc logging * Spellling corrections (no code changes) (#8971) * Spelling corrections * As this might break let's do as a separate PR * Dependabot use correct label (#8973) * Inject hashed prefix for remote-ext (#8960) * Inject for remote-ext * Update utils/frame/remote-externalities/src/lib.rs Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com> * Update utils/frame/remote-externalities/src/lib.rs Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com> * Apply suggestions from code review * Apply suggestions from code review Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com> * Use `SpawnTaskHandle`s for spawning tasks in the tx pool (#8958) * Remove futures-diagnose * Use `SpawnTaskHandle`s for spawning tasks in the tx pool * Box the spawner * Fix tests * Use the testing task executor * Do not spend time on verifying the signatures before calling Runtime (#8980) * Revert "Use `SpawnTaskHandle`s for spawning tasks in the tx pool (#8958)" (#8983) This reverts commit bfef07c0d22ead3ab3c4e0e90ddf9b0e3537566e. * Uniques: An economically-secure basic-featured NFT pallet (#8813) * Uniques: An economically-secure basic-featured NFT pallet * force_transfer * freeze/thaw * team management * approvals * Fixes * force_asset_status * class_metadata * instance metadata * Fixes * use nmap * Fixes * class metadata has information field * Intiial mock/tests and a fix * Remove impl_non_fungibles * Docs * Update frame/uniques/src/lib.rs Co-authored-by: Shawn Tabrizi * Update frame/uniques/src/lib.rs Co-authored-by: Shawn Tabrizi * Update frame/uniques/src/lib.rs Co-authored-by: Shawn Tabrizi * Update frame/uniques/src/lib.rs Co-authored-by: Shawn Tabrizi * Reserve, don't transfer. * Fixes * Tests * Tests * refresh_deposit * Tests and proper handling of metdata destruction * test burn * Tests * Update impl_fungibles.rs * Initial benchmarking * benchmark * Fixes * cargo run --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_uniques --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/uniques/src/weights.rs --template=./.maintain/frame-weight-template.hbs * Attributes * Attribute metadata * Fixes * Update frame/uniques/README.md * Docs * Docs * Docs * Simple metadata * Use BoundedVec * cargo run --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_uniques --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/uniques/src/weights.rs --template=./.maintain/frame-weight-template.hbs * Update frame/uniques/src/lib.rs Co-authored-by: Lohann Paterno Coutinho Ferreira * Update frame/uniques/src/lib.rs Co-authored-by: Lohann Paterno Coutinho Ferreira * Update frame/uniques/src/lib.rs Co-authored-by: Lohann Paterno Coutinho Ferreira * Update frame/uniques/src/lib.rs Co-authored-by: Lohann Paterno Coutinho Ferreira * Update frame/uniques/src/lib.rs Co-authored-by: Lohann Paterno Coutinho Ferreira * Fixes * Update frame/uniques/README.md Co-authored-by: Alexander Popiak * Update frame/uniques/README.md Co-authored-by: Alexander Popiak * Update frame/uniques/README.md Co-authored-by: Alexander Popiak * Docs * Bump Co-authored-by: Shawn Tabrizi Co-authored-by: Parity Bot Co-authored-by: Lohann Paterno Coutinho Ferreira Co-authored-by: Alexander Popiak * Update WeakBoundedVec's remove and swap_remove (#8985) Co-authored-by: Boiethios * Convert another instance of Into impl to From in the macros (#8986) * Convert another instance of Into impl to From in the macros * Convert another location * also fix bounded vec (#8987) * fix most compiler errors Mostly the work so far has been in tracking down where precisely to insert appropriate trait bounds, and updating `fn insert_submission`. However, there's still a compiler error remaining: ``` error[E0275]: overflow evaluating the requirement `Compact<_>: Decode` | = help: consider adding a `#![recursion_limit="256"]` attribute to your crate (`pallet_election_provider_multi_phase`) = note: required because of the requirements on the impl of `Decode` for `Compact<_>` = note: 126 redundant requirements hidden = note: required because of the requirements on the impl of `Decode` for `Compact<_>` ``` Next up: figure out how we ended up with that recursive bound, and fix it. * extract type SignedSubmissionsOf Weirdly, we still encounter the recursive trait definition error here, despite removing the trait bounds. Something weird is happening. * impl Decode bounds on BoundedBTreeMap/Set on T, not predecessor Otherwise, Rust gets confused and decides that the trait bound is infinitely recursive. For that matter, it _still_ gets confused somehow and decides that the trait bound is infinitely recursive, but at least this should somewhat simplify the matter. * fix recursive trait bound problem * minor fixes * more little fixes * correct semantics for try_insert * more fixes * derive Ord for SolutionType * tests compile * fix most tests, rm unnecessary one * Transactionpool: Make `ready_at` return earlier (#8995) `ready_at` returns when we have processed the requested block. However, on startup we already have processed the best block and there are no transactions in the pool on startup anyway. So, we can set `updated_at` to the best block on startup. Besides that `ready_at` now returns early when there are no ready nor any future transactions in the pool. * Discard notifications if we have failed to parse handshake (#8806) * Migrate pallet-democracy to pallet attribute macro (#8824) * Migrate pallet-democracy to pallet attribute macro. * Metadata fix. * Trigger CI. * Add ecdsa::Pair::verify_prehashed() (#8996) * Add ecdsa::Pair::verify_prehashed() * turn verify_prehashed() into an associated function * add Signature::recover_prehashed() * Non-fungible token traits (#8993) * Non-fungible token traits * Docs * Fixes * Implement non-fungible trait for Uniques * Update frame/uniques/src/impl_nonfungibles.rs Co-authored-by: Shawn Tabrizi * Update frame/uniques/src/impl_nonfungibles.rs Co-authored-by: Shawn Tabrizi Co-authored-by: Shawn Tabrizi * Removes unused import (#9007) * Add Call Filter That Prevents Nested `batch_all` (#9009) * add filter preventing nested `batch_all` * more tests * fix test * cargo run --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_utility --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/utility/src/weights.rs --template=./.maintain/frame-weight-template.hbs Co-authored-by: Parity Bot * Transaction pool: Ensure that we prune transactions properly (#8963) * Transaction pool: Ensure that we prune transactions properly There was a bug in the transaction pool that we didn't pruned transactions properly because we called `prune_known`, instead of `prune`. This bug was introduced by: https://github.com/paritytech/substrate/pull/4629 This is required to have stale extrinsics being removed properly, so that they don't fill up the tx pool. * Fix compilation * Fix benches * ... * Storage chain: Runtime module (#8624) * Transaction storage runtime module * WIP: Tests * Tests, benchmarks and docs * Made check_proof mandatory * Typo * Renamed a crate * Apply suggestions from code review Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> * Added weight for on_finalize * Fixed counter mutations * Reorganized tests * Fixed build * Update for the new inherent API * Reworked for the new inherents API * Apply suggestions from code review Co-authored-by: cheme Co-authored-by: Alexander Popiak Co-authored-by: Shawn Tabrizi * Store transactions in a Vec * Added FeeDestination * Get rid of constants * Fixed node runtime build * Fixed benches * Update frame/transaction-storage/src/lib.rs Co-authored-by: cheme Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Co-authored-by: cheme Co-authored-by: Alexander Popiak Co-authored-by: Shawn Tabrizi * more useful error message (#9014) * Named reserve (#7778) * add NamedReservableCurrency * move currency related trait and types into a new file * implement NamedReservableCurrency * remove empty reserves * Update frame/support/src/traits.rs Co-authored-by: Shawn Tabrizi * fix build * bump year * add MaxReserves * repatriate_reserved_named should put reserved fund into named reserved * add tests * add some docs * fix warning * Update lib.rs * fix test * fix test * fix * fix * triggier CI * Move NamedReservableCurrency. * Use strongly bounded vec for reserves. * Fix test. * remove duplicated file * trigger CI * Make `ReserveIdentifier` assosicated type * add helpers * make ReserveIdentifier assosicated type * fix * update * trigger CI * Apply suggestions from code review Co-authored-by: Shawn Tabrizi * trigger CI * Apply suggestions from code review Co-authored-by: Shawn Tabrizi Co-authored-by: Gavin Wood Co-authored-by: Shaun Wang * update ss58 type to u16 (#8955) * Fixed build (#9021) * Bump parity-db (#9024) * consensus: handle justification sync for blocks authored locally (#8698) * consensus: add trait to control justification sync process * network: implement JustificationSyncLink for NetworkService * slots: handle justification sync in slot worker * babe: fix slot worker instantiation * aura: fix slot worker instantiation * pow: handle justification sync in miner * babe: fix tests * aura: fix tests * node: fix compilation * node-template: fix compilation * consensus: rename justification sync link parameter * aura: fix test compilation * consensus: slots: move JustificationSyncLink out of on_slot * arithmetic: fix PerThing pow (#9030) * arithmetic: add failing test for pow * arithmetic: fix PerThing::pow * Revert back to previous optimisations Co-authored-by: Gav Wood * Compact proof utilities in sp_trie. (#8574) * validation extension in sp_io * need paths * arc impl * missing host function in executor * io to pkdot * decode function. * encode primitive. * trailing tab * multiple patch * fix child trie logic * restore master versionning * bench compact proof size * trie-db 22.3 is needed * line width * split line * fixes for bench (additional root may not be needed as original issue was with empty proof). * revert compact from block size calculation. * New error type for compression. * Adding test (incomplete (failing)). Also lacking real proof checking (no good primitives in sp-trie crate). * There is currently no proof recording utility in sp_trie, removing test. * small test of child root in proof without a child proof. * remove empty test. * remove non compact proof size * Missing revert. * proof method to encode decode. * Don't inlucde nominaotrs that back no one in the snapshot. (#9017) * fix all_in_one test which had a logic error * use sp_std, not std * Periodically call `Peerset::alloc_slots` on all sets (#9025) * Periodically call alloc_slots on all slots * Add test * contracts: Add new `seal_call` that offers new features (#8909) * Add new `seal_call` that offers new features * Fix doc typo Co-authored-by: Michael Müller * Fix doc typos Co-authored-by: Michael Müller * Fix comment on assert * Update CHANGELOG.md Co-authored-by: Michael Müller * fix unreserve_all_named (#9042) * Delete legacy runtime metadata macros (#9043) * `rpc-http-threads` cli arg (#8890) * Add optional `rpc-http-threads` cli arg * Update `http::ServerBuilder`threads * allow inserting equal items into bounded map/set * refactor: only load one solution at a time This increases the database read load, because we read one solution at a time. On the other hand, it substantially decreases the overall memory load, because we _only_ read one solution at a time instead of reading all of them. * Emit `Bonded` event when rebonding (#9040) * Emit `Bonded` event when rebonding * fix borrow checker * cargo run --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_staking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/staking/src/weights.rs --template=./.maintain/frame-weight-template.hbs Co-authored-by: Parity Bot * fix tests * Revert "Merge remote-tracking branch 'origin/master' into prgn-election-provider-multi-phase-bounded-btree-set-signed-submissions" This reverts commit de92b1e8e0e44a74c24e270d02b6e8e6a2c37032, reversing changes made to dae31f2018593b60dbf1d96ec96cdc35c374bb9e. * only derive debug when std * write after check * SignedSubmissions doesn't ever modify storage until .put() This makes a true check-before-write pattern possible. * REVERT ME: demo that Drop impl doesn't work * Revert "REVERT ME: demo that Drop impl doesn't work" This reverts commit 3317a4bb4de2e77d5a7fff2154552a81ec081763. * doc note about decode_len * rename get_submission, take_submission for clarity * add test which fails for current incorrect behavior * inline fn insert_submission This fixes a tricky check-before-write error, ensuring that we really only ever modify anything if we have in fact succeeded. Co-authored-by: Roman Proskuryakov Co-authored-by: Denis Pisarev Co-authored-by: MOZGIII Co-authored-by: Alexander Theißen Co-authored-by: Shawn Tabrizi Co-authored-by: thiolliere Co-authored-by: joe petrowski <25483142+joepetrowski@users.noreply.github.com> Co-authored-by: Sebastian Müller Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Co-authored-by: emostov <32168567+emostov@users.noreply.github.com> Co-authored-by: Andrew Jones Co-authored-by: Gavin Wood Co-authored-by: Shaun Wang Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Seun Lanlege Co-authored-by: Bastian Köcher Co-authored-by: Keith Yeung Co-authored-by: Squirrel Co-authored-by: Sergei Shulepov Co-authored-by: Ashley Co-authored-by: Parity Bot Co-authored-by: Lohann Paterno Coutinho Ferreira Co-authored-by: Alexander Popiak Co-authored-by: Boiethios Co-authored-by: Boiethios Co-authored-by: Pierre Krieger Co-authored-by: Andreas Doerr Co-authored-by: Dmitry Kashitsyn Co-authored-by: Arkadiy Paronyan Co-authored-by: cheme Co-authored-by: Andronik Ordian Co-authored-by: Xiliang Chen Co-authored-by: Gavin Wood Co-authored-by: Jakub Pánik Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com> Co-authored-by: Michael Müller Co-authored-by: tgmichel --- .../src/benchmarking.rs | 23 +- .../election-provider-multi-phase/src/lib.rs | 91 ++-- .../src/signed.rs | 432 ++++++++++++------ .../support/src/storage/bounded_btree_map.rs | 64 ++- .../support/src/storage/bounded_btree_set.rs | 62 ++- primitives/npos-elections/compact/src/lib.rs | 2 +- 6 files changed, 477 insertions(+), 197 deletions(-) diff --git a/frame/election-provider-multi-phase/src/benchmarking.rs b/frame/election-provider-multi-phase/src/benchmarking.rs index b2f1c9bb4c8a6..7988163e98f65 100644 --- a/frame/election-provider-multi-phase/src/benchmarking.rs +++ b/frame/election-provider-multi-phase/src/benchmarking.rs @@ -273,19 +273,16 @@ frame_benchmarking::benchmarks! { MultiPhase::::on_initialize_open_signed().expect("should be ok to start signed phase"); >::put(1); - >::mutate(|queue| { - for i in 0..c { - let solution = RawSolution { - score: [(10_000_000 + i).into(), 0, 0], - ..Default::default() - }; - let signed_submission = SignedSubmission { solution, ..Default::default() }; - queue.push(signed_submission); - } - // as of here, the queue is ordered worst-to-best. - // However, we have an invariant that it should be ordered best-to-worst - queue.reverse(); - }); + let mut signed_submissions = SignedSubmissions::::get(); + for i in 0..c { + let solution = RawSolution { + score: [(10_000_000 + i).into(), 0, 0], + ..Default::default() + }; + let signed_submission = SignedSubmission { solution, ..Default::default() }; + signed_submissions.insert(signed_submission); + } + signed_submissions.put(); let caller = frame_benchmarking::whitelisted_caller(); T::Currency::make_free_balance_be(&caller, T::Currency::minimum_balance() * 10u32.into()); diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 2d8fee6b1e2ef..e599f4cfaa039 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -249,14 +249,16 @@ pub mod helpers; const LOG_TARGET: &'static str = "runtime::election-provider"; -pub mod unsigned; pub mod signed; +pub mod unsigned; pub mod weights; +pub use signed::{ + BalanceOf, NegativeImbalanceOf, PositiveImbalanceOf, SignedSubmission, SignedSubmissionOf, + SignedSubmissions, SubmissionIndicesOf, +}; pub use weights::WeightInfo; -pub use signed::{SignedSubmission, BalanceOf, NegativeImbalanceOf, PositiveImbalanceOf}; - /// The compact solution type used by this crate. pub type CompactOf = ::CompactSolution; @@ -387,7 +389,7 @@ impl Default for ElectionCompute { /// /// Such a solution should never become effective in anyway before being checked by the /// `Pallet::feasibility_check` -#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, PartialOrd, Ord)] pub struct RawSolution { /// Compact election edges. pub compact: C, @@ -562,6 +564,7 @@ pub mod pallet { /// Maximum number of signed submissions that can be queued. #[pallet::constant] type SignedMaxSubmissions: Get; + /// Maximum weight of a signed solution. /// /// This should probably be similar to [`Config::MinerMaxWeight`]. @@ -603,6 +606,7 @@ pub mod pallet { + Eq + Clone + sp_std::fmt::Debug + + Ord + CompactSolution; /// Accuracy used for fallback on-chain election. @@ -879,22 +883,37 @@ pub mod pallet { Error::::SignedTooMuchWeight, ); - // ensure solution claims is better. + // create the submission + let reward = T::SignedRewardBase::get(); + let deposit = Self::deposit_for(&solution, size); + let submission = SignedSubmission { who: who.clone(), deposit, reward, solution }; + + // insert the submission if the queue has space or it's better than the weakest + // eject the weakest if the queue was full let mut signed_submissions = Self::signed_submissions(); - let ejected_a_solution = signed_submissions.len() - == T::SignedMaxSubmissions::get().saturated_into::(); - let index = Self::insert_submission(&who, &mut signed_submissions, solution, size) - .ok_or(Error::::SignedQueueFull)?; - - // collect deposit. Thereafter, the function cannot fail. - let deposit = signed_submissions - .get(index) - .map(|s| s.deposit) - .ok_or(Error::::InvalidSubmissionIndex)?; - T::Currency::reserve(&who, deposit).map_err(|_| Error::::SignedCannotPayDeposit)?; - - // store the new signed submission. - >::put(signed_submissions); + let (inserted, maybe_weakest) = signed_submissions.insert(submission); + let ejected_a_solution = maybe_weakest.is_some(); + + // it's an error if we neither inserted nor removed any submissions: this indicates + // the queue was full but our solution had insufficient score to eject any solution + ensure!( + (false, false) != (inserted, ejected_a_solution), + Error::::SignedQueueFull, + ); + + if inserted { + // collect deposit. Thereafter, the function cannot fail. + T::Currency::reserve(&who, deposit) + .map_err(|_| Error::::SignedCannotPayDeposit)?; + } + + // if we had to remove the weakest solution, unreserve its deposit + if let Some(weakest) = maybe_weakest { + let _remainder = T::Currency::unreserve(&weakest.who, weakest.deposit); + debug_assert!(_remainder.is_zero()); + } + + signed_submissions.put(); Self::deposit_event(Event::SolutionStored(ElectionCompute::Signed, ejected_a_solution)); Ok(()) } @@ -1048,14 +1067,34 @@ pub mod pallet { #[pallet::getter(fn snapshot_metadata)] pub type SnapshotMetadata = StorageValue<_, SolutionOrSnapshotSize>; - /// Sorted (worse -> best) list of unchecked, signed solutions. + /// The next index to be assigned to an incoming signed submission. + /// + /// We can't just use `SignedSubmissionIndices.len()`, because that's a bounded set; past its + /// capacity, it will simply saturate. We can't just iterate over `SignedSubmissionsMap`, + /// because iteration is slow. Instead, we store the value here. + #[pallet::storage] + pub(crate) type SignedSubmissionNextIndex = StorageValue<_, u32, ValueQuery>; + + /// A sorted, bounded set of `(score, index)`, where each `index` points to a value in + /// `SignedSubmissions`. + /// + /// We never need to process more than a single signed submission at a time. Signed submissions + /// can be quite large, so we're willing to pay the cost of multiple database accesses to access + /// them one at a time instead of reading and decoding all of them at once. + #[pallet::storage] + pub(crate) type SignedSubmissionIndices = + StorageValue<_, SubmissionIndicesOf, ValueQuery>; + + /// Unchecked, signed solutions. + /// + /// Together with `SubmissionIndices`, this stores a bounded set of `SignedSubmissions` while + /// allowing us to keep only a single one in memory at a time. + /// + /// Twox note: the key of the map is an auto-incrementing index which users cannot inspect or + /// affect; we shouldn't need a cryptographically secure hasher. #[pallet::storage] - #[pallet::getter(fn signed_submissions)] - pub type SignedSubmissions = StorageValue< - _, - Vec, CompactOf>>, - ValueQuery, - >; + pub(crate) type SignedSubmissionsMap = + StorageMap<_, Twox64Concat, u32, SignedSubmissionOf, ValueQuery>; /// The minimum score that each 'untrusted' solution must attain in order to be considered /// feasible. diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index e0f1a82919b92..97633b28967f4 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -19,14 +19,26 @@ use crate::{ CompactOf, Config, ElectionCompute, Pallet, RawSolution, ReadySolution, SolutionOrSnapshotSize, - Weight, WeightInfo, QueuedSolution, SignedSubmissions, + Weight, WeightInfo, QueuedSolution, SignedSubmissionsMap, SignedSubmissionIndices, + SignedSubmissionNextIndex, }; use codec::{Encode, Decode, HasCompact}; -use frame_support::traits::{Currency, Get, OnUnbalanced, ReservableCurrency}; +use frame_support::{ + storage::bounded_btree_map::BoundedBTreeMap, + traits::{Currency, Get, OnUnbalanced, ReservableCurrency}, + DebugNoBound, +}; use sp_arithmetic::traits::SaturatedConversion; -use sp_npos_elections::{is_score_better, CompactSolution}; -use sp_runtime::{Perbill, RuntimeDebug, traits::{Saturating, Zero}}; -use sp_std::{cmp::Ordering, vec::Vec}; +use sp_npos_elections::{is_score_better, CompactSolution, ElectionScore}; +use sp_runtime::{ + RuntimeDebug, + traits::{Saturating, Zero}, +}; +use sp_std::{ + cmp::Ordering, + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + ops::Deref, +}; /// A raw, unchecked signed submission. /// @@ -43,6 +55,38 @@ pub struct SignedSubmission { pub solution: RawSolution, } +impl Ord + for SignedSubmission +where + AccountId: Ord, + Balance: Ord + HasCompact, + CompactSolution: Ord, + RawSolution: Ord, +{ + fn cmp(&self, other: &Self) -> Ordering { + self.solution + .score + .cmp(&other.solution.score) + .then_with(|| self.solution.cmp(&other.solution)) + .then_with(|| self.deposit.cmp(&other.deposit)) + .then_with(|| self.reward.cmp(&other.reward)) + .then_with(|| self.who.cmp(&other.who)) + } +} + +impl PartialOrd + for SignedSubmission +where + AccountId: Ord, + Balance: Ord + HasCompact, + CompactSolution: Ord, + RawSolution: Ord, +{ + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + pub type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; pub type PositiveImbalanceOf = <::Currency as Currency< @@ -51,8 +95,175 @@ pub type PositiveImbalanceOf = <::Currency as Currency< pub type NegativeImbalanceOf = <::Currency as Currency< ::AccountId, >>::NegativeImbalance; +pub type SignedSubmissionOf = + SignedSubmission<::AccountId, BalanceOf, CompactOf>; + +pub type SubmissionIndicesOf = + BoundedBTreeMap::SignedMaxSubmissions>; + +/// Mask type which pretends to be a set of `SignedSubmissionOf`, while in fact delegating to the +/// actual implementations in `SignedSubmissionIndices`, `SignedSubmissionsMap`, and +/// `SignedSubmissionNextIndex`. +#[cfg_attr(feature = "std", derive(DebugNoBound))] +pub struct SignedSubmissions { + indices: SubmissionIndicesOf, + next_idx: u32, + insertion_overlay: BTreeMap>, + deletion_overlay: BTreeSet, +} + +impl SignedSubmissions { + /// Get the signed submissions from storage. + pub fn get() -> Self { + SignedSubmissions { + indices: SignedSubmissionIndices::::get(), + next_idx: SignedSubmissionNextIndex::::get(), + insertion_overlay: BTreeMap::new(), + deletion_overlay: BTreeSet::new(), + } + } + + /// Put the signed submissions back into storage. + pub fn put(self) { + SignedSubmissionIndices::::put(self.indices); + SignedSubmissionNextIndex::::put(self.next_idx); + for key in self.deletion_overlay { + SignedSubmissionsMap::::remove(key); + } + for (key, value) in self.insertion_overlay { + SignedSubmissionsMap::::insert(key, value); + } + } + + /// Get the submission at a particular index. + fn get_submission(&self, idx: u32) -> Option> { + self.insertion_overlay + .get(&idx) + .cloned() + .or_else(|| SignedSubmissionsMap::::try_get(idx).ok()) + } + + /// Take the submission at a particular index. + fn take_submission(&mut self, idx: u32) -> Option> { + self.insertion_overlay.remove(&idx).or_else(|| { + if self.deletion_overlay.contains(&idx) { + None + } else { + self.deletion_overlay.insert(idx); + SignedSubmissionsMap::::try_get(idx).ok() + } + }) + } + + /// Iterate through the set of signed submissions in order of increasing score. + pub fn iter(&self) -> impl '_ + Iterator> { + self.indices.iter().map(move |(_score, idx)| { + self.get_submission(*idx).expect("SignedSubmissions must remain internally consistent") + }) + } + + /// Empty the set of signed submissions, returning an iterator of signed submissions in + /// arbitrary order + pub fn drain(&mut self) -> impl '_ + Iterator> { + self.indices.clear(); + SignedSubmissionNextIndex::::kill(); + let insertion_overlay = sp_std::mem::take(&mut self.insertion_overlay); + SignedSubmissionsMap::::drain() + .filter(move |(k, _v)| !self.deletion_overlay.contains(k)) + .map(|(_k, v)| v) + .chain(insertion_overlay.into_iter().map(|(_k, v)| v)) + } + + /// Decode the length of the signed submissions without actually reading the entire struct into + /// memory. + /// + /// Note that if you hold an instance of `SignedSubmissions`, this function does _not_ + /// track its current length. This only decodes what is currently stored in memory. + pub fn decode_len() -> Option { + SignedSubmissionIndices::::decode_len() + } + + /// Insert a new signed submission into the set. + /// + /// Returns `(inserted, removed)`. `inserted` is true when the submission was inserted. + /// `removed` is the removed weakest submission, if any. + /// + /// In the event that the new submission is not better than the current weakest according + /// to `is_score_better`, we do not change anything. + pub fn insert( + &mut self, + submission: SignedSubmissionOf, + ) -> (bool, Option>) { + let weakest = match self.indices.try_insert(submission.solution.score, self.next_idx) { + Ok(Some(prev_idx)) => { + // a submission of equal score was already present in the set; + // no point editing the actual backing map as we know that the newer solution can't + // be better than the old. However, we do need to put the old value back. + self.indices + .try_insert(submission.solution.score, prev_idx) + .expect("didn't change the map size; qed"); + return (false, None); + } + Ok(None) => { + // successfully inserted into the set; no need to take out weakest member + None + } + Err((score, insert_idx)) => { + // could not insert into the set because it is full. + // note that we short-circuit return here in case the iteration produces `None`. + // If there wasn't a weakest entry to remove, then there must be a capacity of 0, + // which means that we can't meaningfully proceed. + let (weakest_score, weakest_idx) = match self.indices.iter().next() { + None => return (false, None), + Some((score, idx)) => (*score, *idx), + }; + let threshold = T::SolutionImprovementThreshold::get(); + + // if we haven't improved on the weakest score, don't change anything. + if !is_score_better(score, weakest_score, threshold) { + return (false, None); + } + + self.indices.remove(&weakest_score); + self.indices + .try_insert(score, insert_idx) + .expect("just removed an item, we must be under capacity; qed"); + + // ensure that SignedSubmissionsMap never grows past capacity by taking out the + // weakest member here. + self.take_submission(weakest_idx) + } + }; + + // we've taken out the weakest, so update the storage map and the next index + self.insertion_overlay.insert(self.next_idx, submission); + self.next_idx += 1; + (true, weakest) + } + + /// Remove the signed submission with the highest score from the set. + pub fn pop_last(&mut self) -> Option> { + let (highest_score, idx) = self.indices.iter().rev().next()?; + let (highest_score, idx) = (*highest_score, *idx); + self.indices.remove(&highest_score); + self.take_submission(idx) + } +} + +impl Deref for SignedSubmissions { + type Target = SubmissionIndicesOf; + + fn deref(&self) -> &Self::Target { + &self.indices + } +} impl Pallet { + /// `Self` accessor for `SignedSubmission`. + pub fn signed_submissions() -> SignedSubmissions { + SignedSubmissions::::get() + } + /// Finish the signed phase. Process the signed submissions from best to worse until a valid one /// is found, rewarding the best one and slashing the invalid ones along the way. /// @@ -61,16 +272,11 @@ impl Pallet { /// This drains the [`SignedSubmissions`], potentially storing the best valid one in /// [`QueuedSolution`]. pub fn finalize_signed_phase() -> (bool, Weight) { - let mut all_submissions: Vec> = >::take(); + let mut all_submissions = Self::signed_submissions(); let mut found_solution = false; let mut weight = T::DbWeight::get().reads(1); - // Reverse the ordering of submissions: previously it was ordered such that high-scoring - // solutions have low indices. Now, the code flows more cleanly if high-scoring solutions - // have high indices. - all_submissions.reverse(); - - while let Some(best) = all_submissions.pop() { + while let Some(best) = all_submissions.pop_last() { let SignedSubmission { solution, who, deposit, reward } = best; let active_voters = solution.compact.voter_count() as u32; let feasibility_weight = { @@ -112,12 +318,13 @@ impl Pallet { // Any unprocessed solution is pointless to even consider. Feasible or malicious, // they didn't end up being used. Unreserve the bonds. let discarded = all_submissions.len(); - for not_processed in all_submissions { - let SignedSubmission { who, deposit, .. } = not_processed; + for SignedSubmission { who, deposit, .. } in all_submissions.drain() { let _remaining = T::Currency::unreserve(&who, deposit); weight = weight.saturating_add(T::DbWeight::get().writes(1)); debug_assert!(_remaining.is_zero()); - }; + } + + all_submissions.put(); log!(debug, "closed signed phase, found solution? {}, discarded {}", found_solution, discarded); (found_solution, weight) @@ -157,72 +364,6 @@ impl Pallet { T::SlashHandler::on_unbalanced(negative_imbalance); } - /// Insert a solution into the queue while maintaining an ordering by solution quality. - /// - /// Solutions are ordered in reverse: strong solutions have low indices. - /// - /// If insertion was successful, `Some(index)` is returned where index is the - /// index of the newly inserted item. - /// - /// Note: this function maintains the invariant that `queue.len() <= T::SignedMaxSubmissions`. - /// In the event that insertion would violate that invariant, the weakest element is dropped. - /// - /// Invariant: The returned index is always a valid index in `queue` and can safely be used to - /// inspect the newly inserted element. - pub fn insert_submission( - who: &T::AccountId, - queue: &mut Vec, CompactOf>>, - solution: RawSolution>, - size: SolutionOrSnapshotSize, - ) -> Option { - // Let's ensure that our input is valid. - let max_submissions = T::SignedMaxSubmissions::get(); - debug_assert!(queue.len() as u32 <= max_submissions); - - let threshold = T::SolutionImprovementThreshold::get(); - // this insertion logic is a bit unusual in that a new solution which beats an existing - // solution by less than the threshold is sorted as "greater" than the existing solution. - // this means that the produced ordering depends on the order of insertion, and that - // attempts to produce a total ordering using this comparitor are highly unstable. - // - // this ordering prioritizes earlier solutions over slightly better later ones. - let insertion_position = queue.binary_search_by(|s| { - if is_score_better::( - solution.score, - s.solution.score, - threshold, - ) { - Ordering::Greater - } else { - Ordering::Less - } - }).expect_err("comparitor function never returns Ordering::Equal; qed"); - - // if this solution is the worst one so far and the queue is full, don't insert - if insertion_position == queue.len() && queue.len() as u32 >= max_submissions { - return None; - } - - // add to the designated spot. If the length is too much, remove one. - let reward = T::SignedRewardBase::get(); - let deposit = Self::deposit_for(&solution, size); - let submission = - SignedSubmission { who: who.clone(), deposit, reward, solution }; - queue.insert(insertion_position, submission); - - // Remove the weakest if queue is overflowing. - // This doesn't adjust insertion_position: in the event that it might have, we'd have short- - // circuited above. - if queue.len() as u32 > max_submissions { - if let Some(SignedSubmission { who, deposit, .. }) = queue.pop() { - let _remainder = T::Currency::unreserve(&who, deposit); - debug_assert!(_remainder.is_zero()); - } - } - debug_assert!(queue.len() as u32 <= max_submissions); - Some(insertion_position) - } - /// The feasibility weight of the given raw solution. pub fn feasibility_weight_of( solution: &RawSolution>, @@ -325,7 +466,7 @@ mod tests { assert_ok!(submit_with_witness(Origin::signed(99), solution)); assert_eq!(balances(&99), (95, 5)); - assert_eq!(MultiPhase::signed_submissions().first().unwrap().deposit, 5); + assert_eq!(MultiPhase::signed_submissions().iter().next().unwrap().deposit, 5); }) } @@ -409,6 +550,8 @@ mod tests { assert_ok!(submit_with_witness(Origin::signed(99), solution)); } + dbg!(MultiPhase::signed_submissions().len(), SignedMaxSubmissions::get()); + // weaker. let solution = RawSolution { score: [4, 0, 0], ..Default::default() }; @@ -433,10 +576,10 @@ mod tests { assert_eq!( MultiPhase::signed_submissions() - .into_iter() + .iter() .map(|s| s.solution.score[0]) .collect::>(), - vec![9, 8, 7, 6, 5] + vec![5, 6, 7, 8, 9] ); // better. @@ -446,10 +589,10 @@ mod tests { // the one with score 5 was rejected, the new one inserted. assert_eq!( MultiPhase::signed_submissions() - .into_iter() + .iter() .map(|s| s.solution.score[0]) .collect::>(), - vec![20, 9, 8, 7, 6] + vec![6, 7, 8, 9, 20] ); }) } @@ -471,10 +614,10 @@ mod tests { assert_eq!( MultiPhase::signed_submissions() - .into_iter() + .iter() .map(|s| s.solution.score[0]) .collect::>(), - vec![9, 8, 7, 6, 4], + vec![4, 6, 7, 8, 9], ); // better. @@ -484,10 +627,10 @@ mod tests { // the one with score 5 was rejected, the new one inserted. assert_eq!( MultiPhase::signed_submissions() - .into_iter() + .iter() .map(|s| s.solution.score[0]) .collect::>(), - vec![9, 8, 7, 6, 5], + vec![5, 6, 7, 8, 9], ); }) } @@ -529,10 +672,10 @@ mod tests { } assert_eq!( MultiPhase::signed_submissions() - .into_iter() + .iter() .map(|s| s.solution.score[0]) .collect::>(), - vec![7, 6, 5] + vec![5, 6, 7] ); // 5 is not accepted. This will only cause processing with no benefit. @@ -544,49 +687,6 @@ mod tests { }) } - #[test] - fn solutions_are_always_sorted() { - ExtBuilder::default().signed_max_submission(3).build_and_execute(|| { - let scores = || { - MultiPhase::signed_submissions() - .into_iter() - .map(|s| s.solution.score[0]) - .collect::>() - }; - - roll_to(15); - assert!(MultiPhase::current_phase().is_signed()); - - let solution = RawSolution { score: [5, 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![5]); - - let solution = RawSolution { score: [8, 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![8, 5]); - - let solution = RawSolution { score: [3, 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![8, 5, 3]); - - let solution = RawSolution { score: [6, 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![8, 6, 5]); - - let solution = RawSolution { score: [6, 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![8, 6, 6]); - - let solution = RawSolution { score: [10, 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![10, 8, 6]); - - let solution = RawSolution { score: [12, 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); - assert_eq!(scores(), vec![12, 10, 8]); - }) - } - #[test] fn all_in_one_signed_submission_scenario() { // a combination of: @@ -600,23 +700,25 @@ mod tests { assert_eq!(balances(&99), (100, 0)); assert_eq!(balances(&999), (100, 0)); assert_eq!(balances(&9999), (100, 0)); - let mut solution = raw_solution(); + let solution = raw_solution(); // submit a correct one. assert_ok!(submit_with_witness(Origin::signed(99), solution.clone())); // make the solution invalidly better and submit. This ought to be slashed. - solution.score[0] += 1; - assert_ok!(submit_with_witness(Origin::signed(999), solution.clone())); + let mut solution_999 = solution.clone(); + solution_999.score[0] += 1; + assert_ok!(submit_with_witness(Origin::signed(999), solution_999)); // make the solution invalidly worse and submit. This ought to be suppressed and // returned. - solution.score[0] -= 1; - assert_ok!(submit_with_witness(Origin::signed(9999), solution)); + let mut solution_9999 = solution.clone(); + solution_9999.score[0] -= 1; + assert_ok!(submit_with_witness(Origin::signed(9999), solution_9999)); assert_eq!( MultiPhase::signed_submissions().iter().map(|x| x.who).collect::>(), - vec![999, 99, 9999] + vec![9999, 99, 999] ); // _some_ good solution was stored. @@ -661,4 +763,52 @@ mod tests { ); }) } + + #[test] + fn insufficient_deposit_doesnt_store_submission() { + ExtBuilder::default().build_and_execute(|| { + roll_to(15); + assert!(MultiPhase::current_phase().is_signed()); + + let solution = raw_solution(); + + assert_eq!(balances(&123), (0, 0)); + assert_noop!( + submit_with_witness(Origin::signed(123), solution), + Error::::SignedCannotPayDeposit, + ); + + assert_eq!(balances(&123), (0, 0)); + }) + } + + // given a full queue, and a solution which _should_ be allowed in, but the proposer of this + // new solution has insufficient deposit, we should not modify storage at all + #[test] + fn insufficient_deposit_with_full_queue_works_properly() { + ExtBuilder::default().build_and_execute(|| { + roll_to(15); + assert!(MultiPhase::current_phase().is_signed()); + + for s in 0..SignedMaxSubmissions::get() { + // score is always getting better + let solution = RawSolution { score: [(5 + s).into(), 0, 0], ..Default::default() }; + assert_ok!(submit_with_witness(Origin::signed(99), solution)); + } + + // this solution has a higher score than any in the queue + let solution = RawSolution { + score: [(5 + SignedMaxSubmissions::get()).into(), 0, 0], + ..Default::default() + }; + + assert_eq!(balances(&123), (0, 0)); + assert_noop!( + submit_with_witness(Origin::signed(123), solution), + Error::::SignedCannotPayDeposit, + ); + + assert_eq!(balances(&123), (0, 0)); + }) + } } diff --git a/frame/support/src/storage/bounded_btree_map.rs b/frame/support/src/storage/bounded_btree_map.rs index 8c50557618eec..0c1994d63a35d 100644 --- a/frame/support/src/storage/bounded_btree_map.rs +++ b/frame/support/src/storage/bounded_btree_map.rs @@ -39,7 +39,8 @@ pub struct BoundedBTreeMap(BTreeMap, PhantomData); impl Decode for BoundedBTreeMap where - BTreeMap: Decode, + K: Decode + Ord, + V: Decode, S: Get, { fn decode(input: &mut I) -> Result { @@ -115,14 +116,15 @@ where self.0.get_mut(key) } - /// Exactly the same semantics as [`BTreeMap::insert`], but returns an `Err` (and is a noop) if the - /// new length of the map exceeds `S`. - pub fn try_insert(&mut self, key: K, value: V) -> Result<(), ()> { - if self.len() < Self::bound() { - self.0.insert(key, value); - Ok(()) + /// Exactly the same semantics as [`BTreeMap::insert`], but returns an `Err` (and is a noop) if + /// the new length of the map exceeds `S`. + /// + /// In the `Err` case, returns the inserted pair so it can be further used without cloning. + pub fn try_insert(&mut self, key: K, value: V) -> Result, (K, V)> { + if self.len() < Self::bound() || self.0.contains_key(&key) { + Ok(self.0.insert(key, value)) } else { - Err(()) + Err((key, value)) } } @@ -407,4 +409,50 @@ pub mod test { Err("BoundedBTreeMap exceeds its limit".into()), ); } + + #[test] + fn unequal_eq_impl_insert_works() { + // given a struct with a strange notion of equality + #[derive(Debug)] + struct Unequal(u32, bool); + + impl PartialEq for Unequal { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } + } + impl Eq for Unequal {} + + impl Ord for Unequal { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.0.cmp(&other.0) + } + } + + impl PartialOrd for Unequal { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + let mut map = BoundedBTreeMap::::new(); + + // when the set is full + + for i in 0..4 { + map.try_insert(Unequal(i, false), i).unwrap(); + } + + // can't insert a new distinct member + map.try_insert(Unequal(5, false), 5).unwrap_err(); + + // but _can_ insert a distinct member which compares equal, though per the documentation, + // neither the set length nor the actual member are changed, but the value is + map.try_insert(Unequal(0, true), 6).unwrap(); + assert_eq!(map.len(), 4); + let (zero_key, zero_value) = map.get_key_value(&Unequal(0, true)).unwrap(); + assert_eq!(zero_key.0, 0); + assert_eq!(zero_key.1, false); + assert_eq!(*zero_value, 6); + } } diff --git a/frame/support/src/storage/bounded_btree_set.rs b/frame/support/src/storage/bounded_btree_set.rs index f551a3cbfa38e..10c2300a08a09 100644 --- a/frame/support/src/storage/bounded_btree_set.rs +++ b/frame/support/src/storage/bounded_btree_set.rs @@ -39,7 +39,7 @@ pub struct BoundedBTreeSet(BTreeSet, PhantomData); impl Decode for BoundedBTreeSet where - BTreeSet: Decode, + T: Decode + Ord, S: Get, { fn decode(input: &mut I) -> Result { @@ -103,14 +103,15 @@ where self.0.clear() } - /// Exactly the same semantics as [`BTreeSet::insert`], but returns an `Err` (and is a noop) if the - /// new length of the set exceeds `S`. - pub fn try_insert(&mut self, item: T) -> Result<(), ()> { - if self.len() < Self::bound() { - self.0.insert(item); - Ok(()) + /// Exactly the same semantics as [`BTreeSet::insert`], but returns an `Err` (and is a noop) if + /// the new length of the set exceeds `S`. + /// + /// In the `Err` case, returns the inserted item so it can be further used without cloning. + pub fn try_insert(&mut self, item: T) -> Result { + if self.len() < Self::bound() || self.0.contains(&item) { + Ok(self.0.insert(item)) } else { - Err(()) + Err(item) } } @@ -393,4 +394,49 @@ pub mod test { Err("BoundedBTreeSet exceeds its limit".into()), ); } + + #[test] + fn unequal_eq_impl_insert_works() { + // given a struct with a strange notion of equality + #[derive(Debug)] + struct Unequal(u32, bool); + + impl PartialEq for Unequal { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } + } + impl Eq for Unequal {} + + impl Ord for Unequal { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.0.cmp(&other.0) + } + } + + impl PartialOrd for Unequal { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + let mut set = BoundedBTreeSet::::new(); + + // when the set is full + + for i in 0..4 { + set.try_insert(Unequal(i, false)).unwrap(); + } + + // can't insert a new distinct member + set.try_insert(Unequal(5, false)).unwrap_err(); + + // but _can_ insert a distinct member which compares equal, though per the documentation, + // neither the set length nor the actual member are changed + set.try_insert(Unequal(0, true)).unwrap(); + assert_eq!(set.len(), 4); + let zero_item = set.get(&Unequal(0, true)).unwrap(); + assert_eq!(zero_item.0, 0); + assert_eq!(zero_item.1, false); + } } diff --git a/primitives/npos-elections/compact/src/lib.rs b/primitives/npos-elections/compact/src/lib.rs index e49518cc25cc7..14aefecef6daa 100644 --- a/primitives/npos-elections/compact/src/lib.rs +++ b/primitives/npos-elections/compact/src/lib.rs @@ -169,7 +169,7 @@ fn struct_def( ); quote!{ #compact_impl - #[derive(Default, PartialEq, Eq, Clone, Debug)] + #[derive(Default, PartialEq, Eq, Clone, Debug, PartialOrd, Ord)] } } else { // automatically derived. From 2f26f1693828a5e03dcd0a9996b140cd4437d065 Mon Sep 17 00:00:00 2001 From: Parity Bot Date: Sun, 20 Jun 2021 13:29:20 +0000 Subject: [PATCH 58/86] cargo run --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_election_provider_multi_phase --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/election-provider-multi-phase/src/weights.rs --template=./.maintain/frame-weight-template.hbs --- .../src/weights.rs | 272 ++++++++++++++---- 1 file changed, 209 insertions(+), 63 deletions(-) diff --git a/frame/election-provider-multi-phase/src/weights.rs b/frame/election-provider-multi-phase/src/weights.rs index 5f2a3f238f218..38c321327123b 100644 --- a/frame/election-provider-multi-phase/src/weights.rs +++ b/frame/election-provider-multi-phase/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_election_provider_multi_phase //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0 -//! DATE: 2021-05-27, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-06-20, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -54,73 +54,151 @@ pub trait WeightInfo { fn submit(c: u32, ) -> Weight; fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight; fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; + fn on_initialize_nothing() -> Weight; + fn on_initialize_open_signed() -> Weight; + fn on_initialize_open_unsigned_with_snapshot() -> Weight; + fn finalize_signed_phase_accept_solution() -> Weight; + fn finalize_signed_phase_reject_solution() -> Weight; + fn on_initialize_open_unsigned_without_snapshot() -> Weight; + fn elect_queued() -> Weight; + fn submit(c: u32, ) -> Weight; + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight; + fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; } /// Weights for pallet_election_provider_multi_phase using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { fn on_initialize_nothing() -> Weight { - (29_377_000 as Weight) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) + (33_090_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) } fn on_initialize_open_signed() -> Weight { - (141_740_000 as Weight) - .saturating_add(T::DbWeight::get().reads(8 as Weight)) + (114_048_000 as Weight) + .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } fn on_initialize_open_unsigned_with_snapshot() -> Weight { - (141_477_000 as Weight) - .saturating_add(T::DbWeight::get().reads(8 as Weight)) + (113_442_000 as Weight) + .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } fn finalize_signed_phase_accept_solution() -> Weight { - (52_297_000 as Weight) + (51_475_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } fn finalize_signed_phase_reject_solution() -> Weight { - (23_345_000 as Weight) + (23_115_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn on_initialize_open_unsigned_without_snapshot() -> Weight { - (25_085_000 as Weight) + (24_024_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn elect_queued() -> Weight { - (14_719_594_000 as Weight) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) - .saturating_add(T::DbWeight::get().writes(6 as Weight)) + (6_114_135_000 as Weight) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) + .saturating_add(T::DbWeight::get().writes(8 as Weight)) } fn submit(c: u32, ) -> Weight { - (89_088_000 as Weight) - // Standard Error: 100_000 - .saturating_add((2_852_000 as Weight).saturating_mul(c as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (78_965_000 as Weight) + // Standard Error: 16_000 + .saturating_add((311_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 12_000 + .saturating_add((3_721_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 42_000 + .saturating_add((339_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 12_000 + .saturating_add((11_483_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 63_000 + .saturating_add((2_891_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 8_000 + .saturating_add((3_680_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 27_000 + .saturating_add((406_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 8_000 + .saturating_add((9_593_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 41_000 + .saturating_add((3_289_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + } + fn on_initialize_nothing() -> Weight { + (33_392_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + } + fn on_initialize_open_signed() -> Weight { + (115_659_000 as Weight) + .saturating_add(T::DbWeight::get().reads(10 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + fn on_initialize_open_unsigned_with_snapshot() -> Weight { + (114_970_000 as Weight) + .saturating_add(T::DbWeight::get().reads(10 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + fn finalize_signed_phase_accept_solution() -> Weight { + (51_442_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + fn finalize_signed_phase_reject_solution() -> Weight { + (23_160_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn on_initialize_open_unsigned_without_snapshot() -> Weight { + (24_101_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } - fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { + fn elect_queued() -> Weight { + (6_153_604_000 as Weight) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) + .saturating_add(T::DbWeight::get().writes(8 as Weight)) + } + fn submit(c: u32, ) -> Weight { + (78_972_000 as Weight) + // Standard Error: 16_000 + .saturating_add((308_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 20_000 - .saturating_add((4_825_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 20_000 - .saturating_add((13_016_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 103_000 - .saturating_add((3_318_000 as Weight).saturating_mul(d as Weight)) + // Standard Error: 12_000 + .saturating_add((3_572_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 42_000 + .saturating_add((23_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 12_000 + .saturating_add((11_529_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 63_000 + .saturating_add((3_333_000 as Weight).saturating_mul(d as Weight)) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 11_000 - .saturating_add((4_991_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 37_000 - .saturating_add((476_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 11_000 - .saturating_add((10_349_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 56_000 - .saturating_add((3_738_000 as Weight).saturating_mul(d as Weight)) + // Standard Error: 7_000 + .saturating_add((3_647_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 23_000 + .saturating_add((390_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 7_000 + .saturating_add((9_614_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 35_000 + .saturating_add((3_405_000 as Weight).saturating_mul(d as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) } } @@ -128,67 +206,135 @@ impl WeightInfo for SubstrateWeight { // For backwards compatibility and tests impl WeightInfo for () { fn on_initialize_nothing() -> Weight { - (29_377_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + (33_090_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) } fn on_initialize_open_signed() -> Weight { - (141_740_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + (114_048_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } fn on_initialize_open_unsigned_with_snapshot() -> Weight { - (141_477_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + (113_442_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } fn finalize_signed_phase_accept_solution() -> Weight { - (52_297_000 as Weight) + (51_475_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } fn finalize_signed_phase_reject_solution() -> Weight { - (23_345_000 as Weight) + (23_115_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn on_initialize_open_unsigned_without_snapshot() -> Weight { - (25_085_000 as Weight) + (24_024_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn elect_queued() -> Weight { - (14_719_594_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) - .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + (6_114_135_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) + .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } fn submit(c: u32, ) -> Weight { - (89_088_000 as Weight) - // Standard Error: 100_000 - .saturating_add((2_852_000 as Weight).saturating_mul(c as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (78_965_000 as Weight) + // Standard Error: 16_000 + .saturating_add((311_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 12_000 + .saturating_add((3_721_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 42_000 + .saturating_add((339_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 12_000 + .saturating_add((11_483_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 63_000 + .saturating_add((2_891_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } - fn submit_unsigned(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { + fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 8_000 + .saturating_add((3_680_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 27_000 + .saturating_add((406_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 8_000 + .saturating_add((9_593_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 41_000 + .saturating_add((3_289_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + } + fn on_initialize_nothing() -> Weight { + (33_392_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + } + fn on_initialize_open_signed() -> Weight { + (115_659_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(10 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + fn on_initialize_open_unsigned_with_snapshot() -> Weight { + (114_970_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(10 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + fn finalize_signed_phase_accept_solution() -> Weight { + (51_442_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + fn finalize_signed_phase_reject_solution() -> Weight { + (23_160_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn on_initialize_open_unsigned_without_snapshot() -> Weight { + (24_101_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn elect_queued() -> Weight { + (6_153_604_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) + .saturating_add(RocksDbWeight::get().writes(8 as Weight)) + } + fn submit(c: u32, ) -> Weight { + (78_972_000 as Weight) + // Standard Error: 16_000 + .saturating_add((308_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 20_000 - .saturating_add((4_825_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 20_000 - .saturating_add((13_016_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 103_000 - .saturating_add((3_318_000 as Weight).saturating_mul(d as Weight)) + // Standard Error: 12_000 + .saturating_add((3_572_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 42_000 + .saturating_add((23_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 12_000 + .saturating_add((11_529_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 63_000 + .saturating_add((3_333_000 as Weight).saturating_mul(d as Weight)) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 11_000 - .saturating_add((4_991_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 37_000 - .saturating_add((476_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 11_000 - .saturating_add((10_349_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 56_000 - .saturating_add((3_738_000 as Weight).saturating_mul(d as Weight)) + // Standard Error: 7_000 + .saturating_add((3_647_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 23_000 + .saturating_add((390_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 7_000 + .saturating_add((9_614_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 35_000 + .saturating_add((3_405_000 as Weight).saturating_mul(d as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) } } From eb07d0b009e0c50e630584be3434a4c2aeeb77df Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 11:14:23 +0200 Subject: [PATCH 59/86] remove duplicate weight definitions injected by benchmark bot --- .../src/weights.rs | 142 ------------------ 1 file changed, 142 deletions(-) diff --git a/frame/election-provider-multi-phase/src/weights.rs b/frame/election-provider-multi-phase/src/weights.rs index 38c321327123b..6a245ebb51254 100644 --- a/frame/election-provider-multi-phase/src/weights.rs +++ b/frame/election-provider-multi-phase/src/weights.rs @@ -54,87 +54,11 @@ pub trait WeightInfo { fn submit(c: u32, ) -> Weight; fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight; fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; - fn on_initialize_nothing() -> Weight; - fn on_initialize_open_signed() -> Weight; - fn on_initialize_open_unsigned_with_snapshot() -> Weight; - fn finalize_signed_phase_accept_solution() -> Weight; - fn finalize_signed_phase_reject_solution() -> Weight; - fn on_initialize_open_unsigned_without_snapshot() -> Weight; - fn elect_queued() -> Weight; - fn submit(c: u32, ) -> Weight; - fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight; - fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; } /// Weights for pallet_election_provider_multi_phase using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { - fn on_initialize_nothing() -> Weight { - (33_090_000 as Weight) - .saturating_add(T::DbWeight::get().reads(8 as Weight)) - } - fn on_initialize_open_signed() -> Weight { - (114_048_000 as Weight) - .saturating_add(T::DbWeight::get().reads(10 as Weight)) - .saturating_add(T::DbWeight::get().writes(4 as Weight)) - } - fn on_initialize_open_unsigned_with_snapshot() -> Weight { - (113_442_000 as Weight) - .saturating_add(T::DbWeight::get().reads(10 as Weight)) - .saturating_add(T::DbWeight::get().writes(4 as Weight)) - } - fn finalize_signed_phase_accept_solution() -> Weight { - (51_475_000 as Weight) - .saturating_add(T::DbWeight::get().reads(1 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) - } - fn finalize_signed_phase_reject_solution() -> Weight { - (23_115_000 as Weight) - .saturating_add(T::DbWeight::get().reads(1 as Weight)) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) - } - fn on_initialize_open_unsigned_without_snapshot() -> Weight { - (24_024_000 as Weight) - .saturating_add(T::DbWeight::get().reads(1 as Weight)) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) - } - fn elect_queued() -> Weight { - (6_114_135_000 as Weight) - .saturating_add(T::DbWeight::get().reads(5 as Weight)) - .saturating_add(T::DbWeight::get().writes(8 as Weight)) - } - fn submit(c: u32, ) -> Weight { - (78_965_000 as Weight) - // Standard Error: 16_000 - .saturating_add((311_000 as Weight).saturating_mul(c as Weight)) - .saturating_add(T::DbWeight::get().reads(4 as Weight)) - .saturating_add(T::DbWeight::get().writes(3 as Weight)) - } - fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { - (0 as Weight) - // Standard Error: 12_000 - .saturating_add((3_721_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 42_000 - .saturating_add((339_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 12_000 - .saturating_add((11_483_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 63_000 - .saturating_add((2_891_000 as Weight).saturating_mul(d as Weight)) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) - } - fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { - (0 as Weight) - // Standard Error: 8_000 - .saturating_add((3_680_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 27_000 - .saturating_add((406_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 8_000 - .saturating_add((9_593_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 41_000 - .saturating_add((3_289_000 as Weight).saturating_mul(d as Weight)) - .saturating_add(T::DbWeight::get().reads(4 as Weight)) - } fn on_initialize_nothing() -> Weight { (33_392_000 as Weight) .saturating_add(T::DbWeight::get().reads(8 as Weight)) @@ -205,72 +129,6 @@ impl WeightInfo for SubstrateWeight { // For backwards compatibility and tests impl WeightInfo for () { - fn on_initialize_nothing() -> Weight { - (33_090_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(8 as Weight)) - } - fn on_initialize_open_signed() -> Weight { - (114_048_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(10 as Weight)) - .saturating_add(RocksDbWeight::get().writes(4 as Weight)) - } - fn on_initialize_open_unsigned_with_snapshot() -> Weight { - (113_442_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(10 as Weight)) - .saturating_add(RocksDbWeight::get().writes(4 as Weight)) - } - fn finalize_signed_phase_accept_solution() -> Weight { - (51_475_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) - } - fn finalize_signed_phase_reject_solution() -> Weight { - (23_115_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) - } - fn on_initialize_open_unsigned_without_snapshot() -> Weight { - (24_024_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) - } - fn elect_queued() -> Weight { - (6_114_135_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(5 as Weight)) - .saturating_add(RocksDbWeight::get().writes(8 as Weight)) - } - fn submit(c: u32, ) -> Weight { - (78_965_000 as Weight) - // Standard Error: 16_000 - .saturating_add((311_000 as Weight).saturating_mul(c as Weight)) - .saturating_add(RocksDbWeight::get().reads(4 as Weight)) - .saturating_add(RocksDbWeight::get().writes(3 as Weight)) - } - fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { - (0 as Weight) - // Standard Error: 12_000 - .saturating_add((3_721_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 42_000 - .saturating_add((339_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 12_000 - .saturating_add((11_483_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 63_000 - .saturating_add((2_891_000 as Weight).saturating_mul(d as Weight)) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) - } - fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { - (0 as Weight) - // Standard Error: 8_000 - .saturating_add((3_680_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 27_000 - .saturating_add((406_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 8_000 - .saturating_add((9_593_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 41_000 - .saturating_add((3_289_000 as Weight).saturating_mul(d as Weight)) - .saturating_add(RocksDbWeight::get().reads(4 as Weight)) - } fn on_initialize_nothing() -> Weight { (33_392_000 as Weight) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) From fb187fc48daf3cd9ca76dc05f162dad55c214249 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 17:04:31 +0200 Subject: [PATCH 60/86] check deletion overlay before getting --- .../src/signed.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 97633b28967f4..b3015ad4fd874 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -124,10 +124,11 @@ impl SignedSubmissions { } /// Put the signed submissions back into storage. - pub fn put(self) { + pub fn put(mut self) { SignedSubmissionIndices::::put(self.indices); SignedSubmissionNextIndex::::put(self.next_idx); for key in self.deletion_overlay { + self.insertion_overlay.remove(&key); SignedSubmissionsMap::::remove(key); } for (key, value) in self.insertion_overlay { @@ -137,10 +138,17 @@ impl SignedSubmissions { /// Get the submission at a particular index. fn get_submission(&self, idx: u32) -> Option> { - self.insertion_overlay - .get(&idx) - .cloned() - .or_else(|| SignedSubmissionsMap::::try_get(idx).ok()) + if self.deletion_overlay.contains(&idx) { + // Note: can't actually remove the item from the insertion overlay (if present) + // because we don't want to use `&mut self` here. There may be some kind of + // `RefCell` optimization possible here in the future. + None + } else { + self.insertion_overlay + .get(&idx) + .cloned() + .or_else(|| SignedSubmissionsMap::::try_get(idx).ok()) + } } /// Take the submission at a particular index. From 2ed81bcf94c67653289aaddd8ad32ea51210d3a2 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 17:05:52 +0200 Subject: [PATCH 61/86] clarify non-conflict between delete, insert overlays --- frame/election-provider-multi-phase/src/signed.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index b3015ad4fd874..b6cff61ec673a 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -245,6 +245,7 @@ impl SignedSubmissions { // we've taken out the weakest, so update the storage map and the next index self.insertion_overlay.insert(self.next_idx, submission); + debug_assert!(!self.deletion_overlay.contains(&self.next_idx)); self.next_idx += 1; (true, weakest) } From 8f0c16e4c433795ec09e6f5df88a9744537f1222 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 17:12:56 +0200 Subject: [PATCH 62/86] drain can be used wrong so is private --- frame/election-provider-multi-phase/src/signed.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index b6cff61ec673a..6c0ccd28497ab 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -171,8 +171,10 @@ impl SignedSubmissions { } /// Empty the set of signed submissions, returning an iterator of signed submissions in - /// arbitrary order - pub fn drain(&mut self) -> impl '_ + Iterator> { + /// arbitrary order. + /// + /// Note that if the iterator is dropped without consuming all elements, not all may be removed. + fn drain(&mut self) -> impl '_ + Iterator> { self.indices.clear(); SignedSubmissionNextIndex::::kill(); let insertion_overlay = sp_std::mem::take(&mut self.insertion_overlay); From 0b0abb8c7ab839493147ed822b67d08da56eaba6 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 17:20:04 +0200 Subject: [PATCH 63/86] update take_submission docs --- frame/election-provider-multi-phase/src/signed.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 6c0ccd28497ab..54e0d016d0cb7 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -152,6 +152,9 @@ impl SignedSubmissions { } /// Take the submission at a particular index. + /// + /// Note that this function does not examine the insertion overlay. If that behavior is desired, + /// try `self.insetion_overlay.remove(&idx).or_else(|| self.take_submission(idx))`. fn take_submission(&mut self, idx: u32) -> Option> { self.insertion_overlay.remove(&idx).or_else(|| { if self.deletion_overlay.contains(&idx) { From 73071aaebba06036e4e566650e8a5ed47fc2fa97 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 22 Jun 2021 09:34:12 +0200 Subject: [PATCH 64/86] more drain improvements --- frame/election-provider-multi-phase/src/signed.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 54e0d016d0cb7..9fc4021d30fa7 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -176,9 +176,12 @@ impl SignedSubmissions { /// Empty the set of signed submissions, returning an iterator of signed submissions in /// arbitrary order. /// - /// Note that if the iterator is dropped without consuming all elements, not all may be removed. - fn drain(&mut self) -> impl '_ + Iterator> { - self.indices.clear(); + /// Note that if the iterator is dropped without consuming all elements, not all may be removed + /// from the underlying `SignedSubmissionsMap`, putting the struct into an invalid state. + /// + /// Note that, like `put`, this function consumes `Self` and modifies storage. + fn drain(mut self) -> impl Iterator> { + SignedSubmissionIndices::::kill(); SignedSubmissionNextIndex::::kill(); let insertion_overlay = sp_std::mem::take(&mut self.insertion_overlay); SignedSubmissionsMap::::drain() @@ -338,8 +341,6 @@ impl Pallet { debug_assert!(_remaining.is_zero()); } - all_submissions.put(); - log!(debug, "closed signed phase, found solution? {}, discarded {}", found_solution, discarded); (found_solution, weight) } From 955f5bccea40e74ca63ca4d0c55c6c1214fdedbe Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 22 Jun 2021 09:42:39 +0200 Subject: [PATCH 65/86] more take_submission docs --- frame/election-provider-multi-phase/src/signed.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 9fc4021d30fa7..1ec2044c3c8ec 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -155,6 +155,9 @@ impl SignedSubmissions { /// /// Note that this function does not examine the insertion overlay. If that behavior is desired, /// try `self.insetion_overlay.remove(&idx).or_else(|| self.take_submission(idx))`. + /// + /// Note that though this function updates the fields `insertion_overlay` and `deletion_overlay`, + /// it does not update the field `indices`. That must be handled separately. fn take_submission(&mut self, idx: u32) -> Option> { self.insertion_overlay.remove(&idx).or_else(|| { if self.deletion_overlay.contains(&idx) { From 65b6926c547b5a539b5a895a09e2689059619f73 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 22 Jun 2021 10:00:42 +0200 Subject: [PATCH 66/86] debug assertion helps prove expectation is valid --- frame/election-provider-multi-phase/src/signed.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 1ec2044c3c8ec..94eccc671c26e 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -243,7 +243,11 @@ impl SignedSubmissions { return (false, None); } - self.indices.remove(&weakest_score); + let _removed = self.indices.remove(&weakest_score); + debug_assert!( + _removed.is_some(), + "we mut have really removed an index to validate our expectation", + ); self.indices .try_insert(score, insert_idx) .expect("just removed an item, we must be under capacity; qed"); From 7b673d166327b97c8304207ec0f7a8872993432e Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 22 Jun 2021 10:06:39 +0200 Subject: [PATCH 67/86] doc on changing SignedMaxSubmissions --- frame/election-provider-multi-phase/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index ad13b20ee42e2..d29dd6f38a45a 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -588,6 +588,12 @@ pub mod pallet { type MinerMaxWeight: Get; /// Maximum number of signed submissions that can be queued. + /// + /// It is best to avoid adjusting this during an election, as it impacts downstream data + /// structures. In particular, `SignedSubmissionIndices` is bounded on this value. If you + /// update this value during an election, you _must_ ensure that + /// `SignedSubmissionIndices.len()` is less than or equal to the new value. Otherwise, + /// attempts to submit new solutions may cause a runtime panic. #[pallet::constant] type SignedMaxSubmissions: Get; From 300c5f6e0a978f77d9aaf212b8702168949add59 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 22 Jun 2021 10:09:30 +0200 Subject: [PATCH 68/86] take_submission inner doc on system properties --- frame/election-provider-multi-phase/src/signed.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 94eccc671c26e..a67ece993ee3c 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -159,6 +159,9 @@ impl SignedSubmissions { /// Note that though this function updates the fields `insertion_overlay` and `deletion_overlay`, /// it does not update the field `indices`. That must be handled separately. fn take_submission(&mut self, idx: u32) -> Option> { + // We never override an existing index in `self.insertion_overlay`, so if we successfully + // removed an item from the insertion overlay, we can be confident that it is not also + // present in `SignedSubmissionsMap`. self.insertion_overlay.remove(&idx).or_else(|| { if self.deletion_overlay.contains(&idx) { None From f84093b15968390c0fb504bda48537ea3f95e8d8 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 22 Jun 2021 10:23:05 +0200 Subject: [PATCH 69/86] Apply suggestions from code review Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com> Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/election-provider-multi-phase/src/lib.rs | 5 ++++- frame/election-provider-multi-phase/src/signed.rs | 5 ++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index d29dd6f38a45a..48f3770ec0c54 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -610,15 +610,18 @@ pub mod pallet { /// Base deposit for a signed solution. #[pallet::constant] type SignedDepositBase: Get>; + /// Per-byte deposit for a signed solution. #[pallet::constant] type SignedDepositByte: Get>; + /// Per-weight deposit for a signed solution. #[pallet::constant] type SignedDepositWeight: Get>; /// Handler for the slashed deposits. type SlashHandler: OnUnbalanced>; + /// Handler for the rewards. type RewardHandler: OnUnbalanced>; @@ -1019,7 +1022,7 @@ pub mod pallet { SignedQueueFull, /// The origin failed to pay the deposit. SignedCannotPayDeposit, - /// witness data to dispatchable is invalid. + /// Witness data to dispatchable is invalid. SignedInvalidWitness, /// The signed submission consumes too much weight SignedTooMuchWeight, diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index a67ece993ee3c..1c7419d8d8b73 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -402,7 +402,7 @@ impl Pallet { ) } - /// Collect sufficient deposit to store this solution this chain. + /// Collect a sufficient deposit to store this solution. /// /// The deposit is composed of 3 main elements: /// @@ -575,7 +575,6 @@ mod tests { assert_ok!(submit_with_witness(Origin::signed(99), solution)); } - dbg!(MultiPhase::signed_submissions().len(), SignedMaxSubmissions::get()); // weaker. let solution = RawSolution { score: [4, 0, 0], ..Default::default() }; @@ -623,7 +622,7 @@ mod tests { } #[test] - fn weakest_is_removed_if_better_provided_wont_remove_self() { + fn replace_weakest_works() { ExtBuilder::default().build_and_execute(|| { roll_to(15); assert!(MultiPhase::current_phase().is_signed()); From 2355a784451b4c59b9c2ef4f1ec22ae8f04db4d1 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 22 Jun 2021 10:25:33 +0200 Subject: [PATCH 70/86] get SolutionOrSnapshotSize out of the loop Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com> --- frame/election-provider-multi-phase/src/signed.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 1c7419d8d8b73..7a9531b636b37 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -303,13 +303,14 @@ impl Pallet { let mut found_solution = false; let mut weight = T::DbWeight::get().reads(1); + let SolutionOrSnapshotSize { voters, targets } = + Self::snapshot_metadata().unwrap_or_default(); + while let Some(best) = all_submissions.pop_last() { let SignedSubmission { solution, who, deposit, reward } = best; let active_voters = solution.compact.voter_count() as u32; let feasibility_weight = { // defensive only: at the end of signed phase, snapshot will exits. - let SolutionOrSnapshotSize { voters, targets } = - Self::snapshot_metadata().unwrap_or_default(); let desired_targets = Self::desired_targets().unwrap_or_default(); T::WeightInfo::feasibility_check( voters, From e4e9edac76ab1787399f7d56e31749133b430d7e Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 22 Jun 2021 10:33:07 +0200 Subject: [PATCH 71/86] doc which items comprise `SignedSubmissions` --- frame/election-provider-multi-phase/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 48f3770ec0c54..385cf91272b7e 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -1135,6 +1135,10 @@ pub mod pallet { #[pallet::getter(fn snapshot_metadata)] pub type SnapshotMetadata = StorageValue<_, SolutionOrSnapshotSize>; + // The following storage items collectively comprise `SignedSubmissions`, and should never be + // accessed independently. Instead, get `Self::signed_submissions()`, modify it as desired, and + // then do `signed_submissions.put()` when you're done with it. + /// The next index to be assigned to an incoming signed submission. /// /// We can't just use `SignedSubmissionIndices.len()`, because that's a bounded set; past its @@ -1164,6 +1168,8 @@ pub mod pallet { pub(crate) type SignedSubmissionsMap = StorageMap<_, Twox64Concat, u32, SignedSubmissionOf, ValueQuery>; + // `SignedSubmissions` items end here. + /// The minimum score that each 'untrusted' solution must attain in order to be considered /// feasible. /// From 65d7af60ac8b5294c6ee00466edc44f33cced6b7 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 22 Jun 2021 10:35:18 +0200 Subject: [PATCH 72/86] add doc about index as unique identifier --- frame/election-provider-multi-phase/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 385cf91272b7e..dceb770ac9d69 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -1141,6 +1141,10 @@ pub mod pallet { /// The next index to be assigned to an incoming signed submission. /// + /// Every accepted submission is assigned a unique index; that index is bound to that particular + /// submission for the duration of the election. On election finalization, the next index is + /// reset to 0. + /// /// We can't just use `SignedSubmissionIndices.len()`, because that's a bounded set; past its /// capacity, it will simply saturate. We can't just iterate over `SignedSubmissionsMap`, /// because iteration is slow. Instead, we store the value here. From 2c61152ab4c71948b2ed627b9e704cd50fe2e1ba Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 22 Jun 2021 10:45:55 +0200 Subject: [PATCH 73/86] Add debug assertions to prove drain worked properly Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/election-provider-multi-phase/src/signed.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 7a9531b636b37..8432d19ef8ddf 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -352,6 +352,10 @@ impl Pallet { debug_assert!(_remaining.is_zero()); } + debug_assert!(!SignedSubmissionIndices::::exists()); + debug_assert!(!SignedSubmissionNextIndex::::exists()); + debug_assert!(SignedSubmissionsMap::::iter().next().is_none()); + log!(debug, "closed signed phase, found solution? {}, discarded {}", found_solution, discarded); (found_solution, weight) } From 368e97bd2a3aec3ff99bea19d04ff3dc7d8ea3af Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 23 Jun 2021 09:54:38 +0200 Subject: [PATCH 74/86] replace take_submission with swap_out_submission --- .../src/signed.rs | 71 +++++++++---------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 8432d19ef8ddf..c3949c9f5f426 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -151,24 +151,34 @@ impl SignedSubmissions { } } - /// Take the submission at a particular index. + /// Perform three operations: /// - /// Note that this function does not examine the insertion overlay. If that behavior is desired, - /// try `self.insetion_overlay.remove(&idx).or_else(|| self.take_submission(idx))`. + /// - Remove a submission (identified by score) + /// - Insert a new submission (identified by score and insertion index) + /// - Return the weakest submission /// - /// Note that though this function updates the fields `insertion_overlay` and `deletion_overlay`, - /// it does not update the field `indices`. That must be handled separately. - fn take_submission(&mut self, idx: u32) -> Option> { - // We never override an existing index in `self.insertion_overlay`, so if we successfully - // removed an item from the insertion overlay, we can be confident that it is not also - // present in `SignedSubmissionsMap`. - self.insertion_overlay.remove(&idx).or_else(|| { - if self.deletion_overlay.contains(&idx) { - None - } else { - self.deletion_overlay.insert(idx); - SignedSubmissionsMap::::try_get(idx).ok() - } + /// Note: in the case that `weakest_score` is not present in `self.indices`, this will return + /// `None` without inserting the new submission and without further notice. + /// + /// Note: this does not enforce any ordering relation between the submission removed and that + /// inserted. + fn swap_out_submission( + &mut self, + remove_score: ElectionScore, + insert: Option<(ElectionScore, u32)>, + ) -> Option> { + let remove_idx = self.indices.remove(&remove_score)?; + if let Some((insert_score, insert_idx)) = insert { + self.indices + .try_insert(insert_score, insert_idx) + .expect("just removed an item, we must be under capacity; qed"); + } + + self.insertion_overlay.remove(&remove_idx).or_else(|| { + (!self.deletion_overlay.contains(&remove_idx)).then(|| { + self.deletion_overlay.insert(remove_idx); + SignedSubmissionsMap::::try_get(remove_idx).ok() + }).flatten() }) } @@ -230,34 +240,23 @@ impl SignedSubmissions { // successfully inserted into the set; no need to take out weakest member None } - Err((score, insert_idx)) => { + Err((insert_score, insert_idx)) => { // could not insert into the set because it is full. // note that we short-circuit return here in case the iteration produces `None`. // If there wasn't a weakest entry to remove, then there must be a capacity of 0, // which means that we can't meaningfully proceed. - let (weakest_score, weakest_idx) = match self.indices.iter().next() { + let weakest_score = match self.indices.iter().next() { None => return (false, None), - Some((score, idx)) => (*score, *idx), + Some((score, _)) => *score, }; let threshold = T::SolutionImprovementThreshold::get(); // if we haven't improved on the weakest score, don't change anything. - if !is_score_better(score, weakest_score, threshold) { + if !is_score_better(insert_score, weakest_score, threshold) { return (false, None); } - let _removed = self.indices.remove(&weakest_score); - debug_assert!( - _removed.is_some(), - "we mut have really removed an index to validate our expectation", - ); - self.indices - .try_insert(score, insert_idx) - .expect("just removed an item, we must be under capacity; qed"); - - // ensure that SignedSubmissionsMap never grows past capacity by taking out the - // weakest member here. - self.take_submission(weakest_idx) + self.swap_out_submission(weakest_score, Some((insert_score, insert_idx))) } }; @@ -270,10 +269,10 @@ impl SignedSubmissions { /// Remove the signed submission with the highest score from the set. pub fn pop_last(&mut self) -> Option> { - let (highest_score, idx) = self.indices.iter().rev().next()?; - let (highest_score, idx) = (*highest_score, *idx); - self.indices.remove(&highest_score); - self.take_submission(idx) + let (score, _) = self.indices.iter().rev().next()?; + // deref in advance to prevent mutable-immutable borrow conflict + let score = *score; + self.swap_out_submission(score, None) } } From 2491916a33c714f0e3c33d3808d7c07016184118 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 23 Jun 2021 10:15:34 +0200 Subject: [PATCH 75/86] use a match to demonstrate all cases from signed_submissions.insert --- .../election-provider-multi-phase/src/lib.rs | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index dceb770ac9d69..0a852a45003a6 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -934,27 +934,31 @@ pub mod pallet { // insert the submission if the queue has space or it's better than the weakest // eject the weakest if the queue was full let mut signed_submissions = Self::signed_submissions(); - let (inserted, maybe_weakest) = signed_submissions.insert(submission); - let ejected_a_solution = maybe_weakest.is_some(); - - // it's an error if we neither inserted nor removed any submissions: this indicates - // the queue was full but our solution had insufficient score to eject any solution - ensure!( - (false, false) != (inserted, ejected_a_solution), - Error::::SignedQueueFull, - ); - - if inserted { - // collect deposit. Thereafter, the function cannot fail. - T::Currency::reserve(&who, deposit) - .map_err(|_| Error::::SignedCannotPayDeposit)?; - } + let ejected_a_solution = match signed_submissions.insert(submission) { + (false, None) => { + // it's an error if we neither inserted nor removed any submissions: this + // indicates the queue was full but our solution had insufficient score to eject + // any solution + return Err(Error::::SignedQueueFull.into()); + } + (false, Some(_)) => { + unreachable!("`signed_submissions.insert` never returns this pattern") + } + (true, maybe_removed) => { + // collect deposit. Thereafter, the function cannot fail. + T::Currency::reserve(&who, deposit) + .map_err(|_| Error::::SignedCannotPayDeposit)?; + + let ejected_a_solution = maybe_removed.is_some(); + // if we had to remove the weakest solution, unreserve its deposit + if let Some(removed) = maybe_removed { + let _remainder = T::Currency::unreserve(&removed.who, removed.deposit); + debug_assert!(_remainder.is_zero()); + } - // if we had to remove the weakest solution, unreserve its deposit - if let Some(weakest) = maybe_weakest { - let _remainder = T::Currency::unreserve(&weakest.who, weakest.deposit); - debug_assert!(_remainder.is_zero()); - } + ejected_a_solution + } + }; signed_submissions.put(); Self::deposit_event(Event::SolutionStored(ElectionCompute::Signed, ejected_a_solution)); From e12e542e08bf7ec4330e6739064c807ead4c780f Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 23 Jun 2021 10:37:01 +0200 Subject: [PATCH 76/86] refactor signed_submissions.insert return type --- .../election-provider-multi-phase/src/lib.rs | 41 ++++++++----------- .../src/signed.rs | 16 ++++---- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 0a852a45003a6..ba45a30a0e9e8 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -934,31 +934,22 @@ pub mod pallet { // insert the submission if the queue has space or it's better than the weakest // eject the weakest if the queue was full let mut signed_submissions = Self::signed_submissions(); - let ejected_a_solution = match signed_submissions.insert(submission) { - (false, None) => { - // it's an error if we neither inserted nor removed any submissions: this - // indicates the queue was full but our solution had insufficient score to eject - // any solution - return Err(Error::::SignedQueueFull.into()); - } - (false, Some(_)) => { - unreachable!("`signed_submissions.insert` never returns this pattern") - } - (true, maybe_removed) => { - // collect deposit. Thereafter, the function cannot fail. - T::Currency::reserve(&who, deposit) - .map_err(|_| Error::::SignedCannotPayDeposit)?; - - let ejected_a_solution = maybe_removed.is_some(); - // if we had to remove the weakest solution, unreserve its deposit - if let Some(removed) = maybe_removed { - let _remainder = T::Currency::unreserve(&removed.who, removed.deposit); - debug_assert!(_remainder.is_zero()); - } - - ejected_a_solution - } - }; + let maybe_removed = signed_submissions + .insert(submission) + // it's an error if we failed to insert a submission: this indicates the queue was + // full but our solution had insufficient score to eject any solution + .ok_or(Error::::SignedQueueFull)?; + + // collect deposit. Thereafter, the function cannot fail. + T::Currency::reserve(&who, deposit) + .map_err(|_| Error::::SignedCannotPayDeposit)?; + + let ejected_a_solution = maybe_removed.is_some(); + // if we had to remove the weakest solution, unreserve its deposit + if let Some(removed) = maybe_removed { + let _remainder = T::Currency::unreserve(&removed.who, removed.deposit); + debug_assert!(_remainder.is_zero()); + } signed_submissions.put(); Self::deposit_event(Event::SolutionStored(ElectionCompute::Signed, ejected_a_solution)); diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index c3949c9f5f426..15b90586cf60d 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -217,15 +217,17 @@ impl SignedSubmissions { /// Insert a new signed submission into the set. /// - /// Returns `(inserted, removed)`. `inserted` is true when the submission was inserted. - /// `removed` is the removed weakest submission, if any. + /// The outer `Option` of the return value indicates whether a solution was inserted. In the + /// case that nothing was inserted, nothing is ever removed. + /// + /// The inner `Option` of the return value contains the submission which was removed, if any. /// /// In the event that the new submission is not better than the current weakest according /// to `is_score_better`, we do not change anything. pub fn insert( &mut self, submission: SignedSubmissionOf, - ) -> (bool, Option>) { + ) -> Option>> { let weakest = match self.indices.try_insert(submission.solution.score, self.next_idx) { Ok(Some(prev_idx)) => { // a submission of equal score was already present in the set; @@ -234,7 +236,7 @@ impl SignedSubmissions { self.indices .try_insert(submission.solution.score, prev_idx) .expect("didn't change the map size; qed"); - return (false, None); + return None; } Ok(None) => { // successfully inserted into the set; no need to take out weakest member @@ -246,14 +248,14 @@ impl SignedSubmissions { // If there wasn't a weakest entry to remove, then there must be a capacity of 0, // which means that we can't meaningfully proceed. let weakest_score = match self.indices.iter().next() { - None => return (false, None), + None => return None, Some((score, _)) => *score, }; let threshold = T::SolutionImprovementThreshold::get(); // if we haven't improved on the weakest score, don't change anything. if !is_score_better(insert_score, weakest_score, threshold) { - return (false, None); + return None; } self.swap_out_submission(weakest_score, Some((insert_score, insert_idx))) @@ -264,7 +266,7 @@ impl SignedSubmissions { self.insertion_overlay.insert(self.next_idx, submission); debug_assert!(!self.deletion_overlay.contains(&self.next_idx)); self.next_idx += 1; - (true, weakest) + Some(weakest) } /// Remove the signed submission with the highest score from the set. From a195ed83eed85f8d26e461dd633b1ab544ca33fe Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 23 Jun 2021 10:52:08 +0200 Subject: [PATCH 77/86] prettify test assertion Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/election-provider-multi-phase/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index ba45a30a0e9e8..4ec1c0d52f46d 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -1876,7 +1876,7 @@ mod tests { // an unexpected call to elect. roll_to(20); - MultiPhase::elect().unwrap(); + assert!(MultiPhase::elect().is_ok()); // all storage items must be cleared. assert_eq!(MultiPhase::round(), 2); From b0a57da14c3c569fa764267682b8a3aa73b6b0c9 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 23 Jun 2021 10:53:10 +0200 Subject: [PATCH 78/86] improve docs Co-authored-by: Guillaume Thiolliere --- frame/election-provider-multi-phase/src/signed.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 15b90586cf60d..0d3d0a97846fc 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -162,6 +162,8 @@ impl SignedSubmissions { /// /// Note: this does not enforce any ordering relation between the submission removed and that /// inserted. + /// + /// Note: this doesn't insert into `insertion_overlay`, the optional new insertion must be inserted into `insertion_overlay` to keep the variable `self` in a valid state. fn swap_out_submission( &mut self, remove_score: ElectionScore, @@ -193,7 +195,7 @@ impl SignedSubmissions { /// arbitrary order. /// /// Note that if the iterator is dropped without consuming all elements, not all may be removed - /// from the underlying `SignedSubmissionsMap`, putting the struct into an invalid state. + /// from the underlying `SignedSubmissionsMap`, putting the storages into an invalid state. /// /// Note that, like `put`, this function consumes `Self` and modifies storage. fn drain(mut self) -> impl Iterator> { From 3325691f01b90186c2b6ca21250212c9431e3a52 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 23 Jun 2021 11:21:30 +0200 Subject: [PATCH 79/86] add tests that finalize_signed_phase is idempotent --- .../src/signed.rs | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 0d3d0a97846fc..932810dc65f00 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -442,7 +442,7 @@ mod tests { SignedMaxSubmissions, SignedMaxWeight, }, }; - use frame_support::{dispatch::DispatchResult, assert_noop, assert_ok}; + use frame_support::{dispatch::DispatchResult, assert_noop, assert_storage_noop, assert_ok}; fn submit_with_witness( origin: Origin, @@ -843,4 +843,35 @@ mod tests { assert_eq!(balances(&123), (0, 0)); }) } + + #[test] + fn finalize_signed_phase_is_idempotent_given_no_submissions() { + ExtBuilder::default().build_and_execute(|| { + for block_number in 0..25 { + roll_to(block_number); + + assert_eq!(SignedSubmissions::::decode_len().unwrap_or_default(), 0); + assert_storage_noop!(MultiPhase::finalize_signed_phase()); + } + }) + } + + #[test] + fn finalize_signed_phase_is_idempotent_given_submissions() { + ExtBuilder::default().build_and_execute(|| { + roll_to(15); + assert!(MultiPhase::current_phase().is_signed()); + + let solution = raw_solution(); + + // submit a correct one. + assert_ok!(submit_with_witness(Origin::signed(99), solution.clone())); + + // _some_ good solution was stored. + assert!(MultiPhase::finalize_signed_phase().0); + + // calling it again doesn't change anything + assert_storage_noop!(MultiPhase::finalize_signed_phase()); + }) + } } From e0b3a52671641bd69a9937735a74d03cfd4c83fe Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 23 Jun 2021 12:00:08 +0200 Subject: [PATCH 80/86] add some debug assertions to guard against misuse of storage --- .../src/signed.rs | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 932810dc65f00..c339c1cd4ac82 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -115,16 +115,32 @@ pub struct SignedSubmissions { impl SignedSubmissions { /// Get the signed submissions from storage. pub fn get() -> Self { - SignedSubmissions { + let submissions = SignedSubmissions { indices: SignedSubmissionIndices::::get(), next_idx: SignedSubmissionNextIndex::::get(), insertion_overlay: BTreeMap::new(), deletion_overlay: BTreeSet::new(), - } + }; + // validate that the stored state is sane + debug_assert!(submissions.indices.values().copied().max().map_or( + true, + |max_idx| submissions.next_idx > max_idx, + )); + submissions } /// Put the signed submissions back into storage. pub fn put(mut self) { + // validate that we're going to write only sane things to storage + debug_assert!(self.insertion_overlay.keys().copied().max().map_or( + true, + |max_idx| self.next_idx > max_idx, + )); + debug_assert!(self.indices.values().copied().max().map_or( + true, + |max_idx| self.next_idx > max_idx, + )); + SignedSubmissionIndices::::put(self.indices); SignedSubmissionNextIndex::::put(self.next_idx); for key in self.deletion_overlay { @@ -163,7 +179,8 @@ impl SignedSubmissions { /// Note: this does not enforce any ordering relation between the submission removed and that /// inserted. /// - /// Note: this doesn't insert into `insertion_overlay`, the optional new insertion must be inserted into `insertion_overlay` to keep the variable `self` in a valid state. + /// Note: this doesn't insert into `insertion_overlay`, the optional new insertion must be + /// inserted into `insertion_overlay` to keep the variable `self` in a valid state. fn swap_out_submission( &mut self, remove_score: ElectionScore, @@ -230,6 +247,9 @@ impl SignedSubmissions { &mut self, submission: SignedSubmissionOf, ) -> Option>> { + // verify the expectation that we never reuse an index + debug_assert!(!self.indices.values().any(|&idx| idx == self.next_idx)); + let weakest = match self.indices.try_insert(submission.solution.score, self.next_idx) { Ok(Some(prev_idx)) => { // a submission of equal score was already present in the set; @@ -265,6 +285,7 @@ impl SignedSubmissions { }; // we've taken out the weakest, so update the storage map and the next index + debug_assert!(!self.insertion_overlay.contains_key(&self.next_idx)); self.insertion_overlay.insert(self.next_idx, submission); debug_assert!(!self.deletion_overlay.contains(&self.next_idx)); self.next_idx += 1; From ad9d095b3035c8926142e2924a62e08118ca169b Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 23 Jun 2021 12:16:36 +0200 Subject: [PATCH 81/86] log internal logic errors instead of panicing --- frame/election-provider-multi-phase/src/signed.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index c339c1cd4ac82..e18602be999a2 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -203,8 +203,17 @@ impl SignedSubmissions { /// Iterate through the set of signed submissions in order of increasing score. pub fn iter(&self) -> impl '_ + Iterator> { - self.indices.iter().map(move |(_score, idx)| { - self.get_submission(*idx).expect("SignedSubmissions must remain internally consistent") + self.indices.iter().filter_map(move |(_score, &idx)| { + let maybe_submission = self.get_submission(idx); + if maybe_submission.is_none() { + log!( + error, + "SignedSubmissions internal state is invalid (idx {}); \ + there is a logic error in code handling signed solution submissions", + idx, + ) + } + maybe_submission }) } From 0ddfedaca8088266771ffab81e8eea194aead710 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 28 Jun 2021 10:11:46 +0200 Subject: [PATCH 82/86] don't store the reward with each signed submission The signed reward base can be treated as a constant. It can in principle change, but even if it's updated in the middle of an election, it's appropriate to use the current value for the winner. --- frame/election-provider-multi-phase/src/lib.rs | 3 +-- frame/election-provider-multi-phase/src/signed.rs | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 4ec1c0d52f46d..9a9b28432c9fa 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -927,9 +927,8 @@ pub mod pallet { ); // create the submission - let reward = T::SignedRewardBase::get(); let deposit = Self::deposit_for(&solution, size); - let submission = SignedSubmission { who: who.clone(), deposit, reward, solution }; + let submission = SignedSubmission { who: who.clone(), deposit, solution }; // insert the submission if the queue has space or it's better than the weakest // eject the weakest if the queue was full diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index e18602be999a2..c44d2aebba419 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -49,8 +49,6 @@ pub struct SignedSubmission { pub who: AccountId, /// The deposit reserved for storing this solution. pub deposit: Balance, - /// The reward that should be given to this solution, if chosen the as the final one. - pub reward: Balance, /// The raw solution itself. pub solution: RawSolution, } @@ -69,7 +67,6 @@ where .cmp(&other.solution.score) .then_with(|| self.solution.cmp(&other.solution)) .then_with(|| self.deposit.cmp(&other.deposit)) - .then_with(|| self.reward.cmp(&other.reward)) .then_with(|| self.who.cmp(&other.who)) } } @@ -339,8 +336,10 @@ impl Pallet { let SolutionOrSnapshotSize { voters, targets } = Self::snapshot_metadata().unwrap_or_default(); + let reward = T::SignedRewardBase::get(); + while let Some(best) = all_submissions.pop_last() { - let SignedSubmission { solution, who, deposit, reward } = best; + let SignedSubmission { solution, who, deposit} = best; let active_voters = solution.compact.voter_count() as u32; let feasibility_weight = { // defensive only: at the end of signed phase, snapshot will exits. From bf4572c967a489b5a07d617aa8374887ec62f7a9 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 28 Jun 2021 10:21:19 +0200 Subject: [PATCH 83/86] emit Rewarded, Slashed events as appropriate Makes it easier to see who won/lost with signed submissions. --- frame/election-provider-multi-phase/src/signed.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index c44d2aebba419..e7bb65fe961ad 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -406,6 +406,9 @@ impl Pallet { // write this ready solution. >::put(ready_solution); + // emit reward event + Self::deposit_event(crate::Event::Rewarded(who.clone())); + // unreserve deposit. let _remaining = T::Currency::unreserve(who, deposit); debug_assert!(_remaining.is_zero()); @@ -421,6 +424,7 @@ impl Pallet { /// /// Infallible pub fn finalize_signed_phase_reject_solution(who: &T::AccountId, deposit: BalanceOf) { + Self::deposit_event(crate::Event::Slashed(who.clone())); let (negative_imbalance, _remaining) = T::Currency::slash_reserved(who, deposit); debug_assert!(_remaining.is_zero()); T::SlashHandler::on_unbalanced(negative_imbalance); From dfd0bf19d0a12560e3a897c8d98377d271bac552 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 28 Jun 2021 10:28:23 +0200 Subject: [PATCH 84/86] update docs --- frame/election-provider-multi-phase/src/signed.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index e7bb65fe961ad..b4cb540a4a7aa 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -168,7 +168,7 @@ impl SignedSubmissions { /// /// - Remove a submission (identified by score) /// - Insert a new submission (identified by score and insertion index) - /// - Return the weakest submission + /// - Return the submission which was removed. /// /// Note: in the case that `weakest_score` is not present in `self.indices`, this will return /// `None` without inserting the new submission and without further notice. From 786898df9d2a4957d891493d85dadc6b836eee40 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 28 Jun 2021 10:41:42 +0200 Subject: [PATCH 85/86] use a custom enum to be explicit about the outcome of insertion --- .../election-provider-multi-phase/src/lib.rs | 8 +++--- .../src/signed.rs | 25 +++++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 9a9b28432c9fa..45e04a757f0b3 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -933,11 +933,13 @@ pub mod pallet { // insert the submission if the queue has space or it's better than the weakest // eject the weakest if the queue was full let mut signed_submissions = Self::signed_submissions(); - let maybe_removed = signed_submissions - .insert(submission) + let maybe_removed = match signed_submissions.insert(submission) { // it's an error if we failed to insert a submission: this indicates the queue was // full but our solution had insufficient score to eject any solution - .ok_or(Error::::SignedQueueFull)?; + signed::InsertResult::NotInserted => return Err(Error::::SignedQueueFull.into()), + signed::InsertResult::Inserted => None, + signed::InsertResult::InsertedEjecting(weakest) => Some(weakest), + }; // collect deposit. Thereafter, the function cannot fail. T::Currency::reserve(&who, deposit) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index b4cb540a4a7aa..04350af044d18 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -98,6 +98,18 @@ pub type SignedSubmissionOf = pub type SubmissionIndicesOf = BoundedBTreeMap::SignedMaxSubmissions>; +/// Outcome of [`SignedSubmissions::insert`]. +pub enum InsertResult { + /// The submission was not inserted because the queue was full and the submission had + /// insufficient score to eject a prior solution from the queue. + NotInserted, + /// The submission was inserted successfully without ejecting a solution. + Inserted, + /// The submission was inserted successfully. As the queue was full, this operation ejected a + /// prior solution, contained in this variant. + InsertedEjecting(SignedSubmissionOf), +} + /// Mask type which pretends to be a set of `SignedSubmissionOf`, while in fact delegating to the /// actual implementations in `SignedSubmissionIndices`, `SignedSubmissionsMap`, and /// `SignedSubmissionNextIndex`. @@ -252,7 +264,7 @@ impl SignedSubmissions { pub fn insert( &mut self, submission: SignedSubmissionOf, - ) -> Option>> { + ) -> InsertResult { // verify the expectation that we never reuse an index debug_assert!(!self.indices.values().any(|&idx| idx == self.next_idx)); @@ -264,7 +276,7 @@ impl SignedSubmissions { self.indices .try_insert(submission.solution.score, prev_idx) .expect("didn't change the map size; qed"); - return None; + return InsertResult::NotInserted; } Ok(None) => { // successfully inserted into the set; no need to take out weakest member @@ -276,14 +288,14 @@ impl SignedSubmissions { // If there wasn't a weakest entry to remove, then there must be a capacity of 0, // which means that we can't meaningfully proceed. let weakest_score = match self.indices.iter().next() { - None => return None, + None => return InsertResult::NotInserted, Some((score, _)) => *score, }; let threshold = T::SolutionImprovementThreshold::get(); // if we haven't improved on the weakest score, don't change anything. if !is_score_better(insert_score, weakest_score, threshold) { - return None; + return InsertResult::NotInserted; } self.swap_out_submission(weakest_score, Some((insert_score, insert_idx))) @@ -295,7 +307,10 @@ impl SignedSubmissions { self.insertion_overlay.insert(self.next_idx, submission); debug_assert!(!self.deletion_overlay.contains(&self.next_idx)); self.next_idx += 1; - Some(weakest) + match weakest { + Some(weakest) => InsertResult::InsertedEjecting(weakest), + None => InsertResult::Inserted, + } } /// Remove the signed submission with the highest score from the set. From 9e6e75fad126e0eabfc07c7bb025b4b46f910748 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 28 Jun 2021 10:45:34 +0200 Subject: [PATCH 86/86] remove outdated docs --- frame/election-provider-multi-phase/src/signed.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 04350af044d18..ba1123c1331ad 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -254,11 +254,6 @@ impl SignedSubmissions { /// Insert a new signed submission into the set. /// - /// The outer `Option` of the return value indicates whether a solution was inserted. In the - /// case that nothing was inserted, nothing is ever removed. - /// - /// The inner `Option` of the return value contains the submission which was removed, if any. - /// /// In the event that the new submission is not better than the current weakest according /// to `is_score_better`, we do not change anything. pub fn insert(