-
Notifications
You must be signed in to change notification settings - Fork 465
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feature/verify trie on state finish (#7691)
Co-authored-by: Lukasz Rozmej <[email protected]> Co-authored-by: Szymon Kulec <[email protected]> Co-authored-by: Kamil Chodoła <[email protected]>
- Loading branch information
4 people
authored
Oct 31, 2024
1 parent
159c2a4
commit 4eb7f09
Showing
14 changed files
with
284 additions
and
63 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
100 changes: 100 additions & 0 deletions
100
src/Nethermind/Nethermind.Synchronization.Test/SynchronizerModuleTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited | ||
| // SPDX-License-Identifier: LGPL-3.0-only | ||
|
|
||
| using System; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Autofac; | ||
| using FluentAssertions; | ||
| using Nethermind.Blockchain.Synchronization; | ||
| using Nethermind.Config; | ||
| using Nethermind.Consensus.Processing; | ||
| using Nethermind.Core; | ||
| using Nethermind.Core.Test.Builders; | ||
| using Nethermind.Db; | ||
| using Nethermind.Logging; | ||
| using Nethermind.State; | ||
| using Nethermind.Synchronization.FastSync; | ||
| using Nethermind.Trie; | ||
| using NSubstitute; | ||
| using NUnit.Framework; | ||
|
|
||
| namespace Nethermind.Synchronization.Test; | ||
|
|
||
| public class SynchronizerModuleTests | ||
| { | ||
| public IContainer CreateTestContainer() | ||
| { | ||
| ITreeSync treeSync = Substitute.For<ITreeSync>(); | ||
| IStateReader stateReader = Substitute.For<IStateReader>(); | ||
| IBlockProcessingQueue blockQueue = Substitute.For<IBlockProcessingQueue>(); | ||
|
|
||
| return new ContainerBuilder() | ||
| .AddModule(new SynchronizerModule(new SyncConfig() | ||
| { | ||
| FastSync = true, | ||
| VerifyTrieOnStateSyncFinished = true | ||
| })) | ||
| .AddKeyedSingleton(DbNames.Code, Substitute.For<IDb>()) | ||
| .AddSingleton(stateReader) | ||
| .AddSingleton(treeSync) | ||
| .AddSingleton(blockQueue) | ||
| .AddSingleton(Substitute.For<IProcessExitSource>()) | ||
| .AddSingleton<ILogManager>(LimboLogs.Instance) | ||
| .Build(); | ||
| } | ||
|
|
||
| [Test] | ||
| public void TestOnTreeSyncFinish_CallVisit() | ||
| { | ||
| IContainer ctx = CreateTestContainer(); | ||
| ITreeSync treeSync = ctx.Resolve<ITreeSync>(); | ||
| IStateReader stateReader = ctx.Resolve<IStateReader>(); | ||
|
|
||
| treeSync.SyncCompleted += Raise.EventWith(null, new ITreeSync.SyncCompletedEventArgs(TestItem.KeccakA)); | ||
|
|
||
| stateReader | ||
| .Received() | ||
| .RunTreeVisitor(Arg.Any<ITreeVisitor>(), Arg.Is(TestItem.KeccakA), Arg.Any<VisitingOptions>()); | ||
| } | ||
|
|
||
| [Test] | ||
| public async Task TestOnTreeSyncFinish_BlockProcessingQueue_UntilFinished() | ||
| { | ||
| IContainer ctx = CreateTestContainer(); | ||
| ITreeSync treeSync = ctx.Resolve<ITreeSync>(); | ||
| IStateReader stateReader = ctx.Resolve<IStateReader>(); | ||
| IBlockProcessingQueue blockQueue = ctx.Resolve<IBlockProcessingQueue>(); | ||
|
|
||
| ManualResetEvent treeVisitorBlocker = new ManualResetEvent(false); | ||
|
|
||
| stateReader | ||
| .When(sr => sr.RunTreeVisitor(Arg.Any<ITreeVisitor>(), Arg.Is(TestItem.KeccakA), Arg.Any<VisitingOptions>())) | ||
| .Do((ci) => | ||
| { | ||
| treeVisitorBlocker.WaitOne(); | ||
| }); | ||
|
|
||
| Task triggerTask = Task.Run(() => | ||
| { | ||
| treeSync.SyncCompleted += Raise.EventWith(null, new ITreeSync.SyncCompletedEventArgs(TestItem.KeccakA)); | ||
| }); | ||
|
|
||
| await Task.Delay(100); | ||
|
|
||
| Task blockQueueTask = Task.Run(() => | ||
| { | ||
| blockQueue.BlockRemoved += | ||
| Raise.EventWith(null, new BlockRemovedEventArgs(null!, ProcessingResult.Success)); | ||
| }); | ||
|
|
||
| await Task.Delay(100); | ||
|
|
||
| blockQueueTask.IsCompleted.Should().BeFalse(); | ||
| treeVisitorBlocker.Set(); | ||
|
|
||
| await triggerTask; | ||
| await blockQueueTask; | ||
| blockQueue.BlockRemoved += Raise.EventWith(null, new BlockRemovedEventArgs(null!, ProcessingResult.Success)); | ||
| } | ||
| } |
17 changes: 17 additions & 0 deletions
17
src/Nethermind/Nethermind.Synchronization/FastSync/ITreeSync.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited | ||
| // SPDX-License-Identifier: LGPL-3.0-only | ||
|
|
||
| using System; | ||
| using Nethermind.Core.Crypto; | ||
|
|
||
| namespace Nethermind.Synchronization.FastSync; | ||
|
|
||
| public interface ITreeSync | ||
| { | ||
| public event EventHandler<SyncCompletedEventArgs> SyncCompleted; | ||
|
|
||
| public class SyncCompletedEventArgs(Hash256 root) : EventArgs | ||
| { | ||
| public Hash256 Root => root; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
67 changes: 67 additions & 0 deletions
67
src/Nethermind/Nethermind.Synchronization/FastSync/VerifyStateOnStateSyncFinished.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited | ||
| // SPDX-License-Identifier: LGPL-3.0-only | ||
|
|
||
| using System; | ||
| using System.Threading; | ||
| using Autofac; | ||
| using Autofac.Features.AttributeFilters; | ||
| using Nethermind.Config; | ||
| using Nethermind.Consensus.Processing; | ||
| using Nethermind.Core.Crypto; | ||
| using Nethermind.Db; | ||
| using Nethermind.Logging; | ||
| using Nethermind.State; | ||
| using Nethermind.Trie; | ||
|
|
||
| namespace Nethermind.Synchronization.FastSync; | ||
|
|
||
| public class VerifyStateOnStateSyncFinished( | ||
| IBlockProcessingQueue processingQueue, | ||
| ITreeSync treeSync, | ||
| IStateReader stateReader, | ||
| [KeyFilter(DbNames.Code)] IDb codeDb, | ||
| IProcessExitSource exitSource, | ||
| ILogManager logManager) : IStartable | ||
| { | ||
| private readonly ILogger _logger = logManager.GetClassLogger<VerifyStateOnStateSyncFinished>(); | ||
|
|
||
| public void Start() | ||
| { | ||
| treeSync.SyncCompleted += TreeSyncOnOnVerifyPostSyncCleanup; | ||
| } | ||
|
|
||
| private void TreeSyncOnOnVerifyPostSyncCleanup(object? sender, ITreeSync.SyncCompletedEventArgs evt) | ||
| { | ||
| ManualResetEvent processingBlocker = new ManualResetEvent(false); | ||
|
|
||
| processingQueue.BlockRemoved += ProcessingQueueOnBlockRemoved; | ||
|
|
||
| try | ||
| { | ||
| Hash256 rootNode = evt.Root; | ||
| _logger!.Info("Collecting trie stats and verifying that no nodes are missing..."); | ||
| TrieStats stats = stateReader.CollectStats(rootNode, codeDb, logManager, exitSource.Token); | ||
| if (stats.MissingNodes > 0) | ||
| { | ||
| _logger.Error($"Missing node found!"); | ||
| } | ||
| _logger.Info($"Stats after finishing state \n" + stats); | ||
| } | ||
| catch (Exception e) | ||
| { | ||
| _logger.Error($"Error in verify trie", e); | ||
| } | ||
| finally | ||
| { | ||
| processingBlocker.Set(); | ||
| processingQueue.BlockRemoved -= ProcessingQueueOnBlockRemoved; | ||
| } | ||
|
|
||
| return; | ||
|
|
||
| void ProcessingQueueOnBlockRemoved(object? o, BlockRemovedEventArgs blockRemovedEventArgs) | ||
| { | ||
| processingBlocker.WaitOne(); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.