From 25e85170f08e5764e6ed2d2ad16452ae79e23815 Mon Sep 17 00:00:00 2001 From: Fedor Partanskiy Date: Wed, 18 Dec 2024 09:26:55 +0300 Subject: [PATCH] fix nodes panic during synchronization Signed-off-by: Fedor Partanskiy --- orderer/consensus/consensus.go | 3 ++ .../consensus/mocks/mock_consenter_support.go | 46 +++++++++++++++++++ .../smartbft/mocks/consenter_support.go | 36 ++++++++++++++- orderer/consensus/smartbft/synchronizer.go | 5 +- .../consensus/smartbft/synchronizer_bft.go | 5 +- .../smartbft/synchronizer_bft_test.go | 8 ++-- .../consensus/smartbft/util_network_test.go | 5 ++ .../mocks/common/multichannel/multichannel.go | 5 ++ 8 files changed, 102 insertions(+), 11 deletions(-) diff --git a/orderer/consensus/consensus.go b/orderer/consensus/consensus.go index 18f214973bd..905a5a2adbb 100644 --- a/orderer/consensus/consensus.go +++ b/orderer/consensus/consensus.go @@ -120,6 +120,9 @@ type ConsenterSupport interface { // WriteBlock commits a block to the ledger. WriteBlock(block *cb.Block, encodedMetadataValue []byte) + // WriteBlockSync commits a block to the ledger. + WriteBlockSync(block *cb.Block, encodedMetadataValue []byte) + // WriteConfigBlock commits a block to the ledger, and applies the config update inside. WriteConfigBlock(block *cb.Block, encodedMetadataValue []byte) diff --git a/orderer/consensus/mocks/mock_consenter_support.go b/orderer/consensus/mocks/mock_consenter_support.go index 3463f09383d..03c7fad799d 100644 --- a/orderer/consensus/mocks/mock_consenter_support.go +++ b/orderer/consensus/mocks/mock_consenter_support.go @@ -201,6 +201,12 @@ type FakeConsenterSupport struct { arg1 *common.Block arg2 []byte } + WriteBlockSyncStub func(*common.Block, []byte) + writeBlockSyncMutex sync.RWMutex + writeBlockSyncArgsForCall []struct { + arg1 *common.Block + arg2 []byte + } WriteConfigBlockStub func(*common.Block, []byte) writeConfigBlockMutex sync.RWMutex writeConfigBlockArgsForCall []struct { @@ -1192,6 +1198,44 @@ func (fake *FakeConsenterSupport) WriteBlockArgsForCall(i int) (*common.Block, [ return argsForCall.arg1, argsForCall.arg2 } +func (fake *FakeConsenterSupport) WriteBlockSync(arg1 *common.Block, arg2 []byte) { + var arg2Copy []byte + if arg2 != nil { + arg2Copy = make([]byte, len(arg2)) + copy(arg2Copy, arg2) + } + fake.writeBlockSyncMutex.Lock() + fake.writeBlockSyncArgsForCall = append(fake.writeBlockSyncArgsForCall, struct { + arg1 *common.Block + arg2 []byte + }{arg1, arg2Copy}) + stub := fake.WriteBlockSyncStub + fake.recordInvocation("WriteBlockSync", []interface{}{arg1, arg2Copy}) + fake.writeBlockSyncMutex.Unlock() + if stub != nil { + fake.WriteBlockSyncStub(arg1, arg2) + } +} + +func (fake *FakeConsenterSupport) WriteBlockSyncCallCount() int { + fake.writeBlockSyncMutex.RLock() + defer fake.writeBlockSyncMutex.RUnlock() + return len(fake.writeBlockSyncArgsForCall) +} + +func (fake *FakeConsenterSupport) WriteBlockSyncCalls(stub func(*common.Block, []byte)) { + fake.writeBlockSyncMutex.Lock() + defer fake.writeBlockSyncMutex.Unlock() + fake.WriteBlockSyncStub = stub +} + +func (fake *FakeConsenterSupport) WriteBlockSyncArgsForCall(i int) (*common.Block, []byte) { + fake.writeBlockSyncMutex.RLock() + defer fake.writeBlockSyncMutex.RUnlock() + argsForCall := fake.writeBlockSyncArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeConsenterSupport) WriteConfigBlock(arg1 *common.Block, arg2 []byte) { var arg2Copy []byte if arg2 != nil { @@ -1267,6 +1311,8 @@ func (fake *FakeConsenterSupport) Invocations() map[string][][]interface{} { defer fake.signatureVerifierMutex.RUnlock() fake.writeBlockMutex.RLock() defer fake.writeBlockMutex.RUnlock() + fake.writeBlockSyncMutex.RLock() + defer fake.writeBlockSyncMutex.RUnlock() fake.writeConfigBlockMutex.RLock() defer fake.writeConfigBlockMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/orderer/consensus/smartbft/mocks/consenter_support.go b/orderer/consensus/smartbft/mocks/consenter_support.go index ed2d970e6a0..6433e2f34d2 100644 --- a/orderer/consensus/smartbft/mocks/consenter_support.go +++ b/orderer/consensus/smartbft/mocks/consenter_support.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.40.1. DO NOT EDIT. +// Code generated by mockery v2.45.0. DO NOT EDIT. package mocks @@ -874,6 +874,40 @@ func (_c *ConsenterSupport_WriteBlock_Call) RunAndReturn(run func(*common.Block, return _c } +// WriteBlockSync provides a mock function with given fields: block, encodedMetadataValue +func (_m *ConsenterSupport) WriteBlockSync(block *common.Block, encodedMetadataValue []byte) { + _m.Called(block, encodedMetadataValue) +} + +// ConsenterSupport_WriteBlockSync_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'WriteBlockSync' +type ConsenterSupport_WriteBlockSync_Call struct { + *mock.Call +} + +// WriteBlockSync is a helper method to define mock.On call +// - block *common.Block +// - encodedMetadataValue []byte +func (_e *ConsenterSupport_Expecter) WriteBlockSync(block interface{}, encodedMetadataValue interface{}) *ConsenterSupport_WriteBlockSync_Call { + return &ConsenterSupport_WriteBlockSync_Call{Call: _e.mock.On("WriteBlockSync", block, encodedMetadataValue)} +} + +func (_c *ConsenterSupport_WriteBlockSync_Call) Run(run func(block *common.Block, encodedMetadataValue []byte)) *ConsenterSupport_WriteBlockSync_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*common.Block), args[1].([]byte)) + }) + return _c +} + +func (_c *ConsenterSupport_WriteBlockSync_Call) Return() *ConsenterSupport_WriteBlockSync_Call { + _c.Call.Return() + return _c +} + +func (_c *ConsenterSupport_WriteBlockSync_Call) RunAndReturn(run func(*common.Block, []byte)) *ConsenterSupport_WriteBlockSync_Call { + _c.Call.Return(run) + return _c +} + // WriteConfigBlock provides a mock function with given fields: block, encodedMetadataValue func (_m *ConsenterSupport) WriteConfigBlock(block *common.Block, encodedMetadataValue []byte) { _m.Called(block, encodedMetadataValue) diff --git a/orderer/consensus/smartbft/synchronizer.go b/orderer/consensus/smartbft/synchronizer.go index 4775be13928..31079c175cd 100644 --- a/orderer/consensus/smartbft/synchronizer.go +++ b/orderer/consensus/smartbft/synchronizer.go @@ -125,7 +125,7 @@ func (s *Synchronizer) synchronize() (*types.Decision, error) { if protoutil.IsConfigBlock(block) { s.Support.WriteConfigBlock(block, nil) } else { - s.Support.WriteBlock(block, nil) + s.Support.WriteBlockSync(block, nil) } s.Logger.Debugf("Fetched and committed block [%d] from cluster", seq) lastPulledBlock = block @@ -141,9 +141,8 @@ func (s *Synchronizer) synchronize() (*types.Decision, error) { return nil, errors.Errorf("failed pulling block %d", seq) } - startSeq := startHeight s.Logger.Infof("Finished synchronizing with cluster, fetched %d blocks, starting from block [%d], up until and including block [%d]", - blocksFetched, startSeq, lastPulledBlock.Header.Number) + blocksFetched, startHeight, lastPulledBlock.Header.Number) viewMetadata, lastConfigSqn := s.getViewMetadataLastConfigSqnFromBlock(lastPulledBlock) diff --git a/orderer/consensus/smartbft/synchronizer_bft.go b/orderer/consensus/smartbft/synchronizer_bft.go index b0476e20d9c..a9291c9fe42 100644 --- a/orderer/consensus/smartbft/synchronizer_bft.go +++ b/orderer/consensus/smartbft/synchronizer_bft.go @@ -258,7 +258,7 @@ func (s *BFTSynchronizer) getBlocksFromSyncBuffer(startHeight, targetHeight uint s.Support.WriteConfigBlock(block, nil) s.Logger.Debugf("Fetched and committed config block [%d] from cluster", seq) } else { - s.Support.WriteBlock(block, nil) + s.Support.WriteBlockSync(block, nil) s.Logger.Debugf("Fetched and committed block [%d] from cluster", seq) } lastPulledBlock = block @@ -277,9 +277,8 @@ func (s *BFTSynchronizer) getBlocksFromSyncBuffer(startHeight, targetHeight uint return nil, errors.Errorf("failed pulling block %d", seq) } - startSeq := startHeight s.Logger.Infof("Finished synchronizing with cluster, fetched %d blocks, starting from block [%d], up until and including block [%d]", - blocksFetched, startSeq, lastPulledBlock.Header.Number) + blocksFetched, startHeight, lastPulledBlock.Header.Number) return lastPulledBlock, nil } diff --git a/orderer/consensus/smartbft/synchronizer_bft_test.go b/orderer/consensus/smartbft/synchronizer_bft_test.go index 93585ce5980..e3011132915 100644 --- a/orderer/consensus/smartbft/synchronizer_bft_test.go +++ b/orderer/consensus/smartbft/synchronizer_bft_test.go @@ -306,7 +306,7 @@ func TestBFTSynchronizer(t *testing.T) { fakeCS.WriteConfigBlockCalls(func(b *cb.Block, m []byte) { ledger = append(ledger, b) }) - fakeCS.WriteBlockCalls(func(b *cb.Block, m []byte) { + fakeCS.WriteBlockSyncCalls(func(b *cb.Block, m []byte) { ledger = append(ledger, b) }) @@ -395,7 +395,7 @@ func TestBFTSynchronizer(t *testing.T) { require.NotNil(t, resp) require.Equal(t, *decision, resp) require.Equal(t, 102, len(ledger)) - require.Equal(t, 1, fakeCS.WriteBlockCallCount()) + require.Equal(t, 1, fakeCS.WriteBlockSyncCallCount()) require.Equal(t, 1, fakeCS.WriteConfigBlockCallCount()) wg.Wait() }) @@ -436,7 +436,7 @@ func TestBFTSynchronizer(t *testing.T) { fakeCS.WriteConfigBlockCalls(func(b *cb.Block, m []byte) { ledger = append(ledger, b) }) - fakeCS.WriteBlockCalls(func(b *cb.Block, m []byte) { + fakeCS.WriteBlockSyncCalls(func(b *cb.Block, m []byte) { ledger = append(ledger, b) }) @@ -533,7 +533,7 @@ func TestBFTSynchronizer(t *testing.T) { require.NotNil(t, resp) require.Equal(t, *decision, resp) require.Equal(t, 103, len(ledger)) - require.Equal(t, 2, fakeCS.WriteBlockCallCount()) + require.Equal(t, 2, fakeCS.WriteBlockSyncCallCount()) require.Equal(t, 1, fakeCS.WriteConfigBlockCallCount()) wg.Wait() }) diff --git a/orderer/consensus/smartbft/util_network_test.go b/orderer/consensus/smartbft/util_network_test.go index 80175038320..79aceb8c5ef 100644 --- a/orderer/consensus/smartbft/util_network_test.go +++ b/orderer/consensus/smartbft/util_network_test.go @@ -515,6 +515,11 @@ func createBFTChainUsingMocks(t *testing.T, node *Node, configInfo *ConfigInfo) node.State.AddBlock(block) node.Logger.Infof("Node %d appended block number %v to ledger", node.NodeId, block.Header.Number) }).Maybe() + supportMock.EXPECT().WriteBlockSync(mock.Anything, mock.Anything).Run( + func(block *cb.Block, encodedMetadataValue []byte) { + node.State.AddBlock(block) + node.Logger.Infof("Node %d appended block number %v to ledger", node.NodeId, block.Header.Number) + }).Maybe() supportMock.EXPECT().WriteConfigBlock(mock.Anything, mock.Anything).Run( func(block *cb.Block, encodedMetadataValue []byte) { diff --git a/orderer/mocks/common/multichannel/multichannel.go b/orderer/mocks/common/multichannel/multichannel.go index 27a5cb904be..2a202e51736 100644 --- a/orderer/mocks/common/multichannel/multichannel.go +++ b/orderer/mocks/common/multichannel/multichannel.go @@ -110,6 +110,11 @@ func (mcs *ConsenterSupport) WriteBlock(block *cb.Block, encodedMetadataValue [] mcs.Append(block) } +// WriteBlock writes data to the Blocks channel +func (mcs *ConsenterSupport) WriteBlockSync(block *cb.Block, encodedMetadataValue []byte) { + mcs.WriteBlock(block, encodedMetadataValue) +} + // WriteConfigBlock calls WriteBlock func (mcs *ConsenterSupport) WriteConfigBlock(block *cb.Block, encodedMetadataValue []byte) { mcs.WriteBlock(block, encodedMetadataValue)