Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Update disputes prioritisation in dispute-coordinator #6130

Merged
merged 39 commits into from
Nov 11, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
56731cf
Scraper processes CandidateBacked events
tdimitrov Oct 5, 2022
056a606
Change definition of best-effort
tdimitrov Oct 5, 2022
9794e99
Fix `dispute-coordinator` tests
tdimitrov Oct 7, 2022
d44055e
Unit test for dispute filtering
tdimitrov Oct 7, 2022
032475a
Clarification comment
tdimitrov Oct 8, 2022
a70d7d8
Add tests
tdimitrov Oct 8, 2022
8bafd7a
Fix logic
tdimitrov Oct 10, 2022
dd6fbc0
Add metrics for refrained participations
tdimitrov Oct 10, 2022
baa7e20
Revert "Add tests"
tdimitrov Oct 10, 2022
d7af3f7
Revert "Unit test for dispute filtering"
tdimitrov Oct 10, 2022
18f7a14
fix dispute-coordinator tests
tdimitrov Oct 10, 2022
b82df59
Fix scraping
tdimitrov Oct 11, 2022
1523bce
new tests
tdimitrov Oct 11, 2022
2c6272f
Small fixes in guide
tdimitrov Oct 13, 2022
cd42bcb
Apply suggestions from code review
tdimitrov Oct 20, 2022
0551b7c
Fix some comments and remove a pointless test
tdimitrov Oct 20, 2022
2906d2a
Code review feedback
tdimitrov Oct 31, 2022
eaa250f
Clarification comment in tests
tdimitrov Nov 2, 2022
9b40f11
Some tests
tdimitrov Nov 7, 2022
7e3040b
Reference counted `CandidateHash` in scraper
tdimitrov Nov 7, 2022
77cc49e
Proper handling for Backed and Included candidates in scraper
tdimitrov Nov 8, 2022
dcd0465
Update comments in tests
tdimitrov Nov 9, 2022
f925dd3
Guide update
tdimitrov Nov 9, 2022
9dd091a
Fix cleanup logic for `backed_candidates_by_block_number`
tdimitrov Nov 9, 2022
e533e6d
Simplify cleanup
tdimitrov Nov 9, 2022
fcb99d0
Merge branch 'master' into disputes-backed
tdimitrov Nov 9, 2022
56f4e2a
Make spellcheck happy
tdimitrov Nov 9, 2022
aecc3e0
Update tests
tdimitrov Nov 9, 2022
a97e29a
Extract candidate backing logic in separate struct
tdimitrov Nov 10, 2022
d29b300
Code review feedback
tdimitrov Nov 10, 2022
0d498dc
Treat backed and included candidates in the same fashion
tdimitrov Nov 11, 2022
40b4115
Update some comments
tdimitrov Nov 11, 2022
4c030a1
Small improvements in test
tdimitrov Nov 11, 2022
76bca96
spell check
tdimitrov Nov 11, 2022
a57ffaa
Fix some more comments
tdimitrov Nov 11, 2022
f9a4407
clean -> prune
tdimitrov Nov 11, 2022
c441720
Code review feedback
tdimitrov Nov 11, 2022
f283517
Reword comment
tdimitrov Nov 11, 2022
f7393a7
spelling
tdimitrov Nov 11, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update some comments
  • Loading branch information
tdimitrov committed Nov 11, 2022
commit 40b41157f3d6a2d99ac827ba33d8d847c0dbf83d
33 changes: 13 additions & 20 deletions node/core/dispute-coordinator/src/scraping/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,24 +57,16 @@ const LRU_OBSERVED_BLOCKS_CAPACITY: NonZeroUsize = match NonZeroUsize::new(20) {
///
/// Concretely:
///
/// - Monitors for `CandidateIncluded` events to keep track of candidates that have been included on chains.
/// - Monitors for `CandidateIncluded` events to keep track of candidates that have been
/// included on chains.
/// - Monitors for `CandidateBacked` events to keep track of all backed candidates.
/// - Calls `FetchOnChainVotes` for each block to gather potentially missed votes from chain.
///
/// With this information it provides a `CandidateComparator` and as a return value of
/// `process_active_leaves_update` any scraped votes.
///
/// If a candidate was backed and included - it will be removed once the block at its height
/// is finalized. However a candidate can be backed and never included. To avoid
/// leaking memory in such cases we keep track at which block height the candidate was backed.
/// We remove backed (and
/// not included) candidates when `CANDIDATE_MAX_LIFETIME` more blocks are
/// finalized after the block they were initially backed in.
/// E.g. if `CANDIDATE_MAX_LIFETIME = 2` when block 4 is finalized we will remove
/// all candidates backed in block 2 which were not already cleaned up on finalization.
///
/// Please note that if a candidate is backed AND included it will be cleaned up on
/// finalization and won't be kept for `CANDIDATE_MAX_LIFETIME` blocks.
/// Scraped candidates are available `CANDIDATE_LIFETIME_AFTER_FINALIZATION` more blocks
/// after finalization as a precaution not to clean them prematurely.
pub struct ChainScraper {
/// All candidates we have seen included, which not yet have been finalized.
included_candidates: candidates::ScrapedCandidates,
Expand All @@ -95,9 +87,9 @@ impl ChainScraper {
/// As long as we have `MAX_FINALITY_LAG` this makes sense as a value.
pub(crate) const ANCESTRY_SIZE_LIMIT: u32 = MAX_FINALITY_LAG;

/// How many blocks after finalization a backed and not included candidate should
/// be kept.
pub(crate) const CANDIDATE_MAX_LIFETIME: BlockNumber = 2;
/// How many blocks after finalization a backed and not included candidate
/// should be kept.
pub(crate) const CANDIDATE_LIFETIME_AFTER_FINALIZATION: BlockNumber = 2;
tdimitrov marked this conversation as resolved.
Show resolved Hide resolved

/// Create a properly initialized `OrderingProvider`.
///
Expand Down Expand Up @@ -177,19 +169,20 @@ impl ChainScraper {

/// Prune finalized candidates.
///
/// Once a candidate lives in a relay chain block that's behind `CANDIDATE_MAX_LIFETIME` blocks
/// after finalization we can treat it as low priority.
/// Once a candidate lives in a relay chain block that's behind `CANDIDATE_LIFETIME_AFTER_FINALIZATION`
/// blocks after finalization we can treat it as low priority.
pub fn process_finalized_block(&mut self, finalized_block_number: &BlockNumber) {
// `CANDIDATE_MAX_LIFETIME - 1` because `finalized_block_number` counts to the candidate lifetime.
// `CANDIDATE_LIFETIME_AFTER_FINALIZATION - 1` because `finalized_block_number`
// counts to the candidate lifetime.
// Example:
// finalized_block_number = 4; CANDIDATE_MAX_LIFETIME = 2;
// finalized_block_number = 4; CANDIDATE_LIFETIME_AFTER_FINALIZATION = 2;
// key_to_clean = 4 - (2 - 1) = 3
// After `remove_all_at_block_height` at 3:
// 0, 1, 2 will be removed
// 3, 4 will be kept
// => We keep candidates in the last two finalized blocks
finalized_block_number
.checked_sub(Self::CANDIDATE_MAX_LIFETIME - 1)
.checked_sub(Self::CANDIDATE_LIFETIME_AFTER_FINALIZATION - 1)
.map(|key_to_clean| {
tdimitrov marked this conversation as resolved.
Show resolved Hide resolved
self.backed_candidates.remove_up_to_height(&key_to_clean);
self.included_candidates.remove_up_to_height(&key_to_clean);
Expand Down
11 changes: 6 additions & 5 deletions node/core/dispute-coordinator/src/scraping/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,7 @@ fn scraper_cleans_finalized_candidates() {
// Finalize the next block and verify that candidate is neither backed nor included.
// Backed AND Included candidates should be cleaned on finalization.
// `scraper_handles_backed_but_not_included_candidate` covers Backed and NOT Included candidates.
finalized_block_number += ChainScraper::CANDIDATE_MAX_LIFETIME + 1;
finalized_block_number += ChainScraper::CANDIDATE_LIFETIME_AFTER_FINALIZATION + 1;
process_finalized_block(&mut scraper, &finalized_block_number);

assert!(!scraper.is_candidate_backed(&candidate.hash()));
Expand Down Expand Up @@ -492,15 +492,16 @@ fn scraper_handles_backed_but_not_included_candidate() {
// Finalize blocks to enforce pruning of scraped events
process_finalized_block(&mut scraper, &finalized_block_number);

// `FIRST_TEST_BLOCK` is finalized, which is within `BACKED_CANDIDATE_MAX_LIFETIME` window.
// `FIRST_TEST_BLOCK` is finalized, which is within `BACKED_CANDIDATE_LIFETIME_AFTER_FINALIZATION` window.
// The candidate should still be backed.
let candidate = make_candidate_receipt(get_block_number_hash(BLOCK_WITH_EVENTS as u32));
assert!(!scraper.is_candidate_included(&candidate.hash()));
assert!(scraper.is_candidate_backed(&candidate.hash()));

// Bump the finalized block outside `BACKED_CANDIDATE_MAX_LIFETIME`.
// Bump the finalized block outside `BACKED_CANDIDATE_LIFETIME_AFTER_FINALIZATION`.
// The candidate should be removed.
finalized_block_number += FIRST_TEST_BLOCK + ChainScraper::CANDIDATE_MAX_LIFETIME + 1;
finalized_block_number +=
FIRST_TEST_BLOCK + ChainScraper::CANDIDATE_LIFETIME_AFTER_FINALIZATION + 1;
process_finalized_block(&mut scraper, &finalized_block_number);

assert!(!scraper.is_candidate_included(&candidate.hash()));
Expand Down Expand Up @@ -537,7 +538,7 @@ fn scraper_handles_the_same_candidate_incuded_in_two_different_block_heights() {

// Finalize blocks to enforce pruning of scraped events.
// The magic candidate was added twice, so it shouldn't be removed if we finalize two more blocks.
finalized_block_number += ChainScraper::CANDIDATE_MAX_LIFETIME + 1; //BLOCKS_TO_SKIP as u32 - finalized_block_number;
finalized_block_number += ChainScraper::CANDIDATE_LIFETIME_AFTER_FINALIZATION + 1; //BLOCKS_TO_SKIP as u32 - finalized_block_number;
process_finalized_block(&mut scraper, &finalized_block_number);

let magic_candidate = make_candidate_receipt(get_magic_candidate_hash());
Expand Down