diff --git a/src/Nethermind/Nethermind.Api/IBasicApi.cs b/src/Nethermind/Nethermind.Api/IBasicApi.cs index 1ba0adb91f9..d6d8d320ef3 100644 --- a/src/Nethermind/Nethermind.Api/IBasicApi.cs +++ b/src/Nethermind/Nethermind.Api/IBasicApi.cs @@ -63,9 +63,8 @@ public ContainerBuilder ConfigureContainerBuilderFromBasicApi(ContainerBuilder b { builder .AddPropertiesFrom(this) - .AddSingleton(ConfigProvider.GetConfig()); - - DbProvider!.ConfigureServiceCollection(builder); + .AddSingleton(ConfigProvider.GetConfig()) + .AddModule(new DbModule()); return builder; } diff --git a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs index a64ba5d833a..d7f25a723ff 100644 --- a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs +++ b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs @@ -123,6 +123,15 @@ public static ContainerBuilder AddModule(this ContainerBuilder builder, IModule return builder; } + + public static ContainerBuilder Map(this ContainerBuilder builder, Func mapper) where TFrom : notnull where TTo : notnull + { + builder.Register(mapper) + .As() + .ExternallyOwned(); + + return builder; + } } /// diff --git a/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs b/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs index cc317af86fb..cdaa6200985 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs @@ -9,6 +9,7 @@ using System.Text.Json.Serialization; using Nethermind.Core.Extensions; +using Nethermind.Int256; using Nethermind.Serialization.Json; namespace Nethermind.Core.Crypto @@ -58,6 +59,10 @@ public ValueHash256(ReadOnlySpan bytes) _bytes = Unsafe.As>(ref MemoryMarshal.GetReference(bytes)); } + public ValueHash256(UInt256 uint256, bool isBigEndian = true) : this(isBigEndian ? uint256.ToBigEndian() : uint256.ToLittleEndian()) + { + } + public override bool Equals(object? obj) => obj is ValueHash256 keccak && Equals(keccak); public bool Equals(ValueHash256 other) => _bytes.Equals(other._bytes); @@ -112,6 +117,11 @@ public Hash256 ToCommitment() public static bool operator >=(in ValueHash256 left, in ValueHash256 right) => left.CompareTo(in right) >= 0; public static bool operator <=(in ValueHash256 left, in ValueHash256 right) => left.CompareTo(in right) <= 0; public static implicit operator Hash256(in ValueHash256 keccak) => new(keccak); + + public UInt256 ToUInt256(bool isBigEndian = true) + { + return new UInt256(Bytes, isBigEndian: isBigEndian); + } } public readonly struct Hash256AsKey(Hash256 key) : IEquatable diff --git a/src/Nethermind/Nethermind.Db/DbModule.cs b/src/Nethermind/Nethermind.Db/DbModule.cs new file mode 100644 index 00000000000..59fefea8b93 --- /dev/null +++ b/src/Nethermind/Nethermind.Db/DbModule.cs @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Autofac; +using Nethermind.Core; + +namespace Nethermind.Db; + +public class DbModule : Module +{ + protected override void Load(ContainerBuilder builder) + { + base.Load(builder); + builder.AddSingleton(this); + + // TODO: Have hooks that automatically get these + string[] dbNames = [ + DbNames.State, + DbNames.Code, + DbNames.Metadata, + DbNames.Blocks, + DbNames.Headers, + DbNames.BlockInfos, + DbNames.BadBlocks, + DbNames.Bloom, + DbNames.Metadata, + ]; + foreach (string dbName in dbNames) + { + ConfigureDb(builder, dbName); + } + + ConfigureColumnDb(builder, DbNames.Receipts); + } + + private static void ConfigureDb(ContainerBuilder builder, string dbName) + { + builder.Register((ctx) => + { + IDbProvider dbProvider = ctx.Resolve(); + IDb db = dbProvider.GetDb(dbName); + return db; + }) + .ExternallyOwned() + .Named(dbName) + .Named(dbName); + + builder.Register((ctx) => + { + IDbProvider dbProvider = ctx.Resolve(); + IDb db = dbProvider.GetDb(dbName); + return db as ITunableDb ?? new NoopTunableDb(); + }) + .ExternallyOwned() + .Named(dbName); + } + + + private static void ConfigureColumnDb(ContainerBuilder builder, string dbName) + { + builder.Register((ctx) => + { + IDbProvider dbProvider = ctx.Resolve(); + IColumnsDb db = dbProvider.GetColumnDb(dbName); + return db; + }) + .ExternallyOwned() + .As>(); + + builder.Register((ctx) => + { + IDbProvider dbProvider = ctx.Resolve(); + IColumnsDb db = dbProvider.GetColumnDb(dbName); + return db as ITunableDb ?? new NoopTunableDb(); + }) + .ExternallyOwned() + .Named(dbName); + } +} diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index 6a0380b0446..d3676082f49 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -33,34 +33,5 @@ public interface IDbProvider : IDisposable void RegisterDb(string dbName, T db) where T : class, IDb; void RegisterColumnDb(string dbName, IColumnsDb db); IEnumerable> GetAllDbMeta(); - - void ConfigureServiceCollection(ContainerBuilder sc) - { - sc.AddSingleton(this); - - // TODO: Have hooks that automatically get these - string[] dbNames = [ - DbNames.State, - DbNames.Code, - DbNames.Metadata, - DbNames.Blocks, - DbNames.Headers, - DbNames.BlockInfos, - DbNames.BadBlocks, - DbNames.Bloom, - DbNames.Metadata, - ]; - foreach (string dbName in dbNames) - { - var db = GetDb(dbName); - sc.AddKeyedSingleton(dbName, db); - sc.AddKeyedSingleton(dbName, db); - sc.AddKeyedSingleton(dbName, db as ITunableDb ?? new NoopTunableDb()); - } - - IColumnsDb receiptColumnDb = GetColumnDb(DbNames.Receipts); - sc.AddSingleton>(receiptColumnDb); - sc.AddKeyedSingleton(DbNames.Receipts, receiptColumnDb as ITunableDb ?? new NoopTunableDb()); - } } } diff --git a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs deleted file mode 100644 index 0b0ce1d8b2b..00000000000 --- a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs +++ /dev/null @@ -1,330 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -#nullable disable - -using System; -using System.Collections.Generic; -using System.Linq; -using FluentAssertions; -using Nethermind.Blockchain; -using Nethermind.Core; -using Nethermind.Core.Crypto; -using Nethermind.Core.Test.Builders; -using Nethermind.Db; -using Nethermind.Logging; -using Nethermind.State; -using Nethermind.State.Proofs; -using Nethermind.State.Snap; -using Nethermind.Synchronization.SnapSync; -using Nethermind.Trie; -using Nethermind.Trie.Pruning; -using NUnit.Framework; - -namespace Nethermind.Store.Test; - -public class RecreateStateFromAccountRangesTests -{ - private StateTree _inputTree; - - [OneTimeSetUp] - public void Setup() - { - _inputTree = TestItem.Tree.GetStateTree(); - } - - //[Test] - public void Test01() - { - Hash256 rootHash = _inputTree.RootHash; // "0x8c81279168edc449089449bc0f2136fc72c9645642845755633cf259cd97988b" - - AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountsWithPaths[0].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - byte[][] firstProof = accountProofCollector.BuildResult().Proof; - - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[5].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - byte[][] lastProof = accountProofCollector.BuildResult().Proof; - - MemDb db = new(); - IScopedTrieStore store = new TrieStore(db, LimboLogs.Instance).GetTrieStore(null); - StateTree tree = new(store, LimboLogs.Instance); - - IList nodes = new List(); - - for (int i = 0; i < (firstProof!).Length; i++) - { - byte[] nodeBytes = (firstProof!)[i]; - var node = new TrieNode(NodeType.Unknown, nodeBytes); - TreePath emptyPath = TreePath.Empty; - node.ResolveKey(store, ref emptyPath, i == 0); - - nodes.Add(node); - if (i < (firstProof!).Length - 1) - { - //IBatch batch = store.GetOrStartNewBatch(); - //batch[node.Keccak!.Bytes] = nodeBytes; - //db.Set(node.Keccak!, nodeBytes); - } - } - - for (int i = 0; i < (lastProof!).Length; i++) - { - byte[] nodeBytes = (lastProof!)[i]; - var node = new TrieNode(NodeType.Unknown, nodeBytes); - TreePath emptyPath = TreePath.Empty; - node.ResolveKey(store, ref emptyPath, i == 0); - - nodes.Add(node); - if (i < (lastProof!).Length - 1) - { - //IBatch batch = store.GetOrStartNewBatch(); - //batch[node.Keccak!.Bytes] = nodeBytes; - //db.Set(node.Keccak!, nodeBytes); - } - } - - tree.RootRef = nodes[0]; - - tree.Set(TestItem.Tree.AccountsWithPaths[0].Path, TestItem.Tree.AccountsWithPaths[0].Account); - tree.Set(TestItem.Tree.AccountsWithPaths[1].Path, TestItem.Tree.AccountsWithPaths[1].Account); - tree.Set(TestItem.Tree.AccountsWithPaths[2].Path, TestItem.Tree.AccountsWithPaths[2].Account); - tree.Set(TestItem.Tree.AccountsWithPaths[3].Path, TestItem.Tree.AccountsWithPaths[3].Account); - tree.Set(TestItem.Tree.AccountsWithPaths[4].Path, TestItem.Tree.AccountsWithPaths[4].Account); - tree.Set(TestItem.Tree.AccountsWithPaths[5].Path, TestItem.Tree.AccountsWithPaths[5].Account); - - tree.Commit(); - - Assert.That(tree.RootHash, Is.EqualTo(_inputTree.RootHash)); - Assert.That(db.Keys.Count, Is.EqualTo(6)); // we don't persist proof nodes (boundary nodes) - Assert.That(db.KeyExists(rootHash), Is.False); // the root node is a part of the proof nodes - } - - [Test] - public void RecreateAccountStateFromOneRangeWithNonExistenceProof() - { - Hash256 rootHash = _inputTree.RootHash; // "0x8c81279168edc449089449bc0f2136fc72c9645642845755633cf259cd97988b" - - AccountProofCollector accountProofCollector = new(Keccak.Zero.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - byte[][] firstProof = accountProofCollector.BuildResult().Proof; - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[5].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - byte[][] lastProof = accountProofCollector.BuildResult().Proof; - - MemDb db = new(); - DbProvider dbProvider = new(); - dbProvider.RegisterDb(DbNames.State, db); - using ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = CreateSnapProvider(progressTracker, dbProvider); - AddRangeResult result = snapProvider.AddAccountRange(1, rootHash, Keccak.Zero, TestItem.Tree.AccountsWithPaths, firstProof!.Concat(lastProof!).ToArray()); - - Assert.That(result, Is.EqualTo(AddRangeResult.OK)); - Assert.That(db.Keys.Count, Is.EqualTo(10)); // we persist proof nodes (boundary nodes) via stitching - Assert.That(db.KeyExists(rootHash), Is.False); - } - - [Test] - public void RecreateAccountStateFromOneRangeWithExistenceProof() - { - Hash256 rootHash = _inputTree.RootHash; // "0x8c81279168edc449089449bc0f2136fc72c9645642845755633cf259cd97988b" - - AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountsWithPaths[0].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - byte[][] firstProof = accountProofCollector.BuildResult().Proof; - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[5].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - byte[][] lastProof = accountProofCollector.BuildResult().Proof; - - MemDb db = new(); - DbProvider dbProvider = new(); - dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = CreateSnapProvider(progressTracker, dbProvider); - var result = snapProvider.AddAccountRange(1, rootHash, TestItem.Tree.AccountsWithPaths[0].Path, TestItem.Tree.AccountsWithPaths, firstProof!.Concat(lastProof!).ToArray()); - - Assert.That(result, Is.EqualTo(AddRangeResult.OK)); - Assert.That(db.Keys.Count, Is.EqualTo(10)); // we persist proof nodes (boundary nodes) via stitching - Assert.That(db.KeyExists(rootHash), Is.False); - } - - [Test] - public void RecreateAccountStateFromOneRangeWithoutProof() - { - Hash256 rootHash = _inputTree.RootHash; // "0x8c81279168edc449089449bc0f2136fc72c9645642845755633cf259cd97988b" - - MemDb db = new(); - DbProvider dbProvider = new(); - dbProvider.RegisterDb(DbNames.State, db); - using ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = CreateSnapProvider(progressTracker, dbProvider); - var result = snapProvider.AddAccountRange(1, rootHash, TestItem.Tree.AccountsWithPaths[0].Path, TestItem.Tree.AccountsWithPaths); - - Assert.That(result, Is.EqualTo(AddRangeResult.OK)); - Assert.That(db.Keys.Count, Is.EqualTo(10)); // we don't have the proofs so we persist all nodes - Assert.That(db.KeyExists(rootHash), Is.False); // the root node is NOT a part of the proof nodes - } - - [Test] - public void RecreateAccountStateFromMultipleRange() - { - Hash256 rootHash = _inputTree.RootHash; // "0x8c81279168edc449089449bc0f2136fc72c9645642845755633cf259cd97988b" - - // output state - MemDb db = new(); - DbProvider dbProvider = new(); - dbProvider.RegisterDb(DbNames.State, db); - using ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = CreateSnapProvider(progressTracker, dbProvider); - - AccountProofCollector accountProofCollector = new(Keccak.Zero.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - byte[][] firstProof = accountProofCollector.BuildResult().Proof; - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[1].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - byte[][] lastProof = accountProofCollector.BuildResult().Proof; - - var result1 = snapProvider.AddAccountRange(1, rootHash, Keccak.Zero, TestItem.Tree.AccountsWithPaths[0..2], firstProof!.Concat(lastProof!).ToArray()); - - Assert.That(db.Keys.Count, Is.EqualTo(2)); - - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[2].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - firstProof = accountProofCollector.BuildResult().Proof; - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[3].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - lastProof = accountProofCollector.BuildResult().Proof; - - var result2 = snapProvider.AddAccountRange(1, rootHash, TestItem.Tree.AccountsWithPaths[2].Path, TestItem.Tree.AccountsWithPaths[2..4], firstProof!.Concat(lastProof!).ToArray()); - - Assert.That(db.Keys.Count, Is.EqualTo(5)); // we don't persist proof nodes (boundary nodes) - - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[4].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - firstProof = accountProofCollector.BuildResult().Proof; - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[5].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - lastProof = accountProofCollector.BuildResult().Proof; - - var result3 = snapProvider.AddAccountRange(1, rootHash, TestItem.Tree.AccountsWithPaths[4].Path, TestItem.Tree.AccountsWithPaths[4..6], firstProof!.Concat(lastProof!).ToArray()); - - Assert.That(result1, Is.EqualTo(AddRangeResult.OK)); - Assert.That(result2, Is.EqualTo(AddRangeResult.OK)); - Assert.That(result3, Is.EqualTo(AddRangeResult.OK)); - Assert.That(db.Keys.Count, Is.EqualTo(10)); // we persist proof nodes (boundary nodes) via stitching - Assert.That(db.KeyExists(rootHash), Is.False); - } - - [Test] - public void MissingAccountFromRange() - { - Hash256 rootHash = _inputTree.RootHash; // "0x8c81279168edc449089449bc0f2136fc72c9645642845755633cf259cd97988b" - - // output state - MemDb db = new(); - DbProvider dbProvider = new(); - dbProvider.RegisterDb(DbNames.State, db); - using ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = CreateSnapProvider(progressTracker, dbProvider); - - AccountProofCollector accountProofCollector = new(Keccak.Zero.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - byte[][] firstProof = accountProofCollector.BuildResult().Proof; - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[1].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - byte[][] lastProof = accountProofCollector.BuildResult().Proof; - - var result1 = snapProvider.AddAccountRange(1, rootHash, Keccak.Zero, TestItem.Tree.AccountsWithPaths[0..2], firstProof!.Concat(lastProof!).ToArray()); - - Assert.That(db.Keys.Count, Is.EqualTo(2)); - - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[2].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - firstProof = accountProofCollector.BuildResult().Proof; - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[3].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - lastProof = accountProofCollector.BuildResult().Proof; - - // missing TestItem.Tree.AccountsWithHashes[2] - var result2 = snapProvider.AddAccountRange(1, rootHash, TestItem.Tree.AccountsWithPaths[2].Path, TestItem.Tree.AccountsWithPaths[3..4], firstProof!.Concat(lastProof!).ToArray()); - - Assert.That(db.Keys.Count, Is.EqualTo(2)); - - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[4].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - firstProof = accountProofCollector.BuildResult().Proof; - accountProofCollector = new(TestItem.Tree.AccountsWithPaths[5].Path.Bytes); - _inputTree.Accept(accountProofCollector, _inputTree.RootHash); - lastProof = accountProofCollector.BuildResult().Proof; - - var result3 = snapProvider.AddAccountRange(1, rootHash, TestItem.Tree.AccountsWithPaths[4].Path, TestItem.Tree.AccountsWithPaths[4..6], firstProof!.Concat(lastProof!).ToArray()); - - Assert.That(result1, Is.EqualTo(AddRangeResult.OK)); - Assert.That(result2, Is.EqualTo(AddRangeResult.DifferentRootHash)); - Assert.That(result3, Is.EqualTo(AddRangeResult.OK)); - Assert.That(db.Keys.Count, Is.EqualTo(6)); - Assert.That(db.KeyExists(rootHash), Is.False); - } - - [Test] - public void Will_not_redownload_persisted_code() - { - MemDb db = new(); - MemDb codeDb = new(); - DbProvider dbProvider = new(); - dbProvider.RegisterDb(DbNames.State, db); - dbProvider.RegisterDb(DbNames.Code, codeDb); - - BlockTree tree = Build.A.BlockTree().OfChainLength(5).TestObject; - using ProgressTracker progressTracker = new(tree, dbProvider.GetDb(DbNames.State), LimboLogs.Instance, - accountRangePartitionCount: 1); - SnapProvider snapProvider = CreateSnapProvider(progressTracker, dbProvider); - - PathWithAccount[] accountsWithPath = - [ - new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001112345"), - new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[0])), - new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001113456"), - new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[1])), - new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001114567"), - new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[2])), - new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001123456"), - new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[3])), - new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001123457"), - new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[4])) - ]; - - codeDb[TestItem.Keccaks[1].Bytes] = [1]; - codeDb[TestItem.Keccaks[2].Bytes] = [1]; - - StateTree stateTree = new StateTree(); - foreach (PathWithAccount pathWithAccount in accountsWithPath) - { - stateTree.Set(pathWithAccount.Path, pathWithAccount.Account); - } - - stateTree.UpdateRootHash(); - - snapProvider.AddAccountRange(1, - stateTree.RootHash, - accountsWithPath[0].Path, - accountsWithPath); - - progressTracker.IsFinished(out SnapSyncBatch nextRequest).Should().BeFalse(); - progressTracker.IsFinished(out nextRequest).Should().BeFalse(); - nextRequest.CodesRequest.Count.Should().Be(3); - } - - private SnapProvider CreateSnapProvider(ProgressTracker progressTracker, IDbProvider dbProvider) - { - try - { - IDb _ = dbProvider.CodeDb; - } - catch (ArgumentException) - { - dbProvider.RegisterDb(DbNames.Code, new MemDb()); - } - return new(progressTracker, dbProvider.CodeDb, new NodeStorage(dbProvider.StateDb), LimboLogs.Instance); - } -} diff --git a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromStorageRangesTests.cs b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromStorageRangesTests.cs deleted file mode 100644 index a4dc0f7ef38..00000000000 --- a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromStorageRangesTests.cs +++ /dev/null @@ -1,179 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -#nullable disable - -using System; -using System.Linq; -using Nethermind.Core; -using Nethermind.Core.Crypto; -using Nethermind.Core.Test.Builders; -using Nethermind.Db; -using Nethermind.Int256; -using Nethermind.Logging; -using Nethermind.State; -using Nethermind.State.Proofs; -using Nethermind.State.Snap; -using Nethermind.Synchronization.SnapSync; -using Nethermind.Trie; -using Nethermind.Trie.Pruning; -using NUnit.Framework; - -namespace Nethermind.Store.Test -{ - [TestFixture] - public class RecreateStateFromStorageRangesTests - { - - private TrieStore _store; - private StateTree _inputStateTree; - private StorageTree _inputStorageTree; - private readonly PathWithAccount _pathWithAccount = new PathWithAccount(TestItem.Tree.AccountAddress0.ValueHash256, new Account(UInt256.Zero)); - - [OneTimeSetUp] - public void Setup() - { - _store = new TrieStore(new MemDb(), LimboLogs.Instance); - (_inputStateTree, _inputStorageTree, _) = TestItem.Tree.GetTrees(_store); - } - - [OneTimeTearDown] - public void TearDown() => _store?.Dispose(); - - [Test] - public void RecreateStorageStateFromOneRangeWithNonExistenceProof() - { - Hash256 rootHash = _inputStorageTree!.RootHash; // "..." - - AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new ValueHash256[] { Keccak.Zero, TestItem.Tree.SlotsWithPaths[5].Path }); - _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); - var proof = accountProofCollector.BuildResult(); - - MemDb db = new(); - DbProvider dbProvider = new(); - dbProvider.RegisterDb(DbNames.State, db); - using ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = CreateSnapProvider(progressTracker, dbProvider); - var result = snapProvider.AddStorageRange(1, _pathWithAccount, rootHash, Keccak.Zero, TestItem.Tree.SlotsWithPaths, proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); - - Assert.That(result, Is.EqualTo(AddRangeResult.OK)); - } - - [Test] - public void RecreateAccountStateFromOneRangeWithExistenceProof() - { - Hash256 rootHash = _inputStorageTree!.RootHash; // "..." - - AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new ValueHash256[] { TestItem.Tree.SlotsWithPaths[0].Path, TestItem.Tree.SlotsWithPaths[5].Path }); - _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); - var proof = accountProofCollector.BuildResult(); - - MemDb db = new(); - DbProvider dbProvider = new(); - dbProvider.RegisterDb(DbNames.State, db); - using ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = CreateSnapProvider(progressTracker, dbProvider); - var result = snapProvider.AddStorageRange(1, _pathWithAccount, rootHash, Keccak.Zero, TestItem.Tree.SlotsWithPaths, proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); - - Assert.That(result, Is.EqualTo(AddRangeResult.OK)); - } - - [Test] - public void RecreateStorageStateFromOneRangeWithoutProof() - { - Hash256 rootHash = _inputStorageTree!.RootHash; // "..." - - MemDb db = new MemDb(); - DbProvider dbProvider = new(); - dbProvider.RegisterDb(DbNames.State, db); - using ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = CreateSnapProvider(progressTracker, dbProvider); - var result = snapProvider.AddStorageRange(1, _pathWithAccount, rootHash, TestItem.Tree.SlotsWithPaths[0].Path, TestItem.Tree.SlotsWithPaths); - - Assert.That(result, Is.EqualTo(AddRangeResult.OK)); - } - - [Test] - public void RecreateAccountStateFromMultipleRange() - { - Hash256 rootHash = _inputStorageTree!.RootHash; // "..." - - // output state - MemDb db = new MemDb(); - DbProvider dbProvider = new(); - dbProvider.RegisterDb(DbNames.State, db); - using ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = CreateSnapProvider(progressTracker, dbProvider); - - AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new ValueHash256[] { Keccak.Zero, TestItem.Tree.SlotsWithPaths[1].Path }); - _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); - var proof = accountProofCollector.BuildResult(); - - var result1 = snapProvider.AddStorageRange(1, _pathWithAccount, rootHash, Keccak.Zero, TestItem.Tree.SlotsWithPaths[0..2], proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); - - accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new ValueHash256[] { TestItem.Tree.SlotsWithPaths[2].Path, TestItem.Tree.SlotsWithPaths[3].Path }); - _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); - proof = accountProofCollector.BuildResult(); - - var result2 = snapProvider.AddStorageRange(1, _pathWithAccount, rootHash, TestItem.Tree.SlotsWithPaths[2].Path, TestItem.Tree.SlotsWithPaths[2..4], proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); - - accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new ValueHash256[] { TestItem.Tree.SlotsWithPaths[4].Path, TestItem.Tree.SlotsWithPaths[5].Path }); - _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); - proof = accountProofCollector.BuildResult(); - - var result3 = snapProvider.AddStorageRange(1, _pathWithAccount, rootHash, TestItem.Tree.SlotsWithPaths[4].Path, TestItem.Tree.SlotsWithPaths[4..6], proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); - - Assert.That(result1, Is.EqualTo(AddRangeResult.OK)); - Assert.That(result2, Is.EqualTo(AddRangeResult.OK)); - Assert.That(result3, Is.EqualTo(AddRangeResult.OK)); - } - - [Test] - public void MissingAccountFromRange() - { - Hash256 rootHash = _inputStorageTree!.RootHash; // "..." - - // output state - MemDb db = new MemDb(); - DbProvider dbProvider = new(); - dbProvider.RegisterDb(DbNames.State, db); - using ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = CreateSnapProvider(progressTracker, dbProvider); - - AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new ValueHash256[] { Keccak.Zero, TestItem.Tree.SlotsWithPaths[1].Path }); - _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); - var proof = accountProofCollector.BuildResult(); - - var result1 = snapProvider.AddStorageRange(1, _pathWithAccount, rootHash, Keccak.Zero, TestItem.Tree.SlotsWithPaths[0..2], proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); - - accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new ValueHash256[] { TestItem.Tree.SlotsWithPaths[2].Path, TestItem.Tree.SlotsWithPaths[3].Path }); - _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); - proof = accountProofCollector.BuildResult(); - - var result2 = snapProvider.AddStorageRange(1, _pathWithAccount, rootHash, TestItem.Tree.SlotsWithPaths[2].Path, TestItem.Tree.SlotsWithPaths[3..4], proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); - - accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new ValueHash256[] { TestItem.Tree.SlotsWithPaths[4].Path, TestItem.Tree.SlotsWithPaths[5].Path }); - _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); - proof = accountProofCollector.BuildResult(); - - var result3 = snapProvider.AddStorageRange(1, _pathWithAccount, rootHash, TestItem.Tree.SlotsWithPaths[4].Path, TestItem.Tree.SlotsWithPaths[4..6], proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); - - Assert.That(result1, Is.EqualTo(AddRangeResult.OK)); - Assert.That(result2, Is.EqualTo(AddRangeResult.DifferentRootHash)); - Assert.That(result3, Is.EqualTo(AddRangeResult.OK)); - } - - private SnapProvider CreateSnapProvider(ProgressTracker progressTracker, IDbProvider dbProvider) - { - try - { - IDb _ = dbProvider.CodeDb; - } - catch (ArgumentException) - { - dbProvider.RegisterDb(DbNames.Code, new MemDb()); - } - return new(progressTracker, dbProvider.CodeDb, new NodeStorage(dbProvider.StateDb), LimboLogs.Instance); - } - } -} diff --git a/src/Nethermind/Nethermind.Synchronization.Test/IContainerSynchronizerTestExtensions.cs b/src/Nethermind/Nethermind.Synchronization.Test/IContainerSynchronizerTestExtensions.cs new file mode 100644 index 00000000000..5622a96675f --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/IContainerSynchronizerTestExtensions.cs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Autofac; +using Nethermind.Blockchain; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Test.Builders; +using NSubstitute; + +namespace Nethermind.Synchronization.Test; + +public static class IContainerSynchronizerTestExtensions +{ + public static ContainerBuilder WithSuggestedHeaderOfStateRoot(this ContainerBuilder builder, Hash256 stateRoot) + { + IBlockTree blockTree = Substitute.For(); + BlockHeader header = Build.A.BlockHeader.WithStateRoot(stateRoot).TestObject; + blockTree.FindHeader(Arg.Any()).Returns(header); + blockTree.BestSuggestedHeader.Returns(header); + + builder.AddSingleton(blockTree); + + return builder; + } +} diff --git a/src/Nethermind/Nethermind.Synchronization.Test/Nethermind.Synchronization.Test.csproj b/src/Nethermind/Nethermind.Synchronization.Test/Nethermind.Synchronization.Test.csproj index adc3e797f41..abcafea8c9f 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/Nethermind.Synchronization.Test.csproj +++ b/src/Nethermind/Nethermind.Synchronization.Test/Nethermind.Synchronization.Test.csproj @@ -27,4 +27,23 @@ + + + + PreserveNewest + + + + PreserveNewest + + + + PreserveNewest + + + + PreserveNewest + + + diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index 50b6b6a54eb..3a6d64f36a7 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -68,6 +68,9 @@ public async Task Setup() IStateReader stateReader = new StateReader(trieStore, _codeDb, LimboLogs.Instance); ContainerBuilder builder = new ContainerBuilder() + .AddModule(new SynchronizerModule(syncConfig)) + .AddModule(new DbModule()) + .AddSingleton(dbProvider) .AddSingleton(nodeStorage) .AddSingleton(MainnetSpecProvider.Instance) .AddSingleton(_blockTree) @@ -84,9 +87,6 @@ public async Task Setup() .AddSingleton(stateReader) .AddSingleton(No.BeaconSync) .AddSingleton(LimboLogs.Instance); - dbProvider.ConfigureServiceCollection(builder); - - builder.RegisterModule(new SynchronizerModule(syncConfig)); IContainer container = builder.Build(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapProviderTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapProviderTests.cs index 58e0a808b06..1529dc46714 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapProviderTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapProviderTests.cs @@ -5,7 +5,6 @@ using Nethermind.Blockchain; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; -using Nethermind.Db; using Nethermind.Logging; using Nethermind.State.Snap; using Nethermind.Synchronization.SnapSync; @@ -14,25 +13,38 @@ using NUnit.Framework; using System; using System.Collections.Generic; +using System.IO.Compression; +using System.Linq; +using System.Text.Json; +using Autofac; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Blockchain.Utils; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Test; +using Nethermind.Core.Test.Builders; +using Nethermind.Db; +using Nethermind.Serialization.Rlp; +using Nethermind.State; +using Nethermind.Trie.Pruning; +using AccountRange = Nethermind.State.Snap.AccountRange; namespace Nethermind.Synchronization.Test.SnapSync; [TestFixture] public class SnapProviderTests { - [Test] public void AddAccountRange_AccountListIsEmpty_ThrowArgumentException() { - MemDb db = new(); - IDbProvider dbProvider = new DbProvider(); - dbProvider.RegisterDb(DbNames.State, db); - using ProgressTracker progressTracker = new(Substitute.For(), dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - dbProvider.RegisterDb(DbNames.Code, new MemDb()); - SnapProvider sut = new(progressTracker, dbProvider.CodeDb, new NodeStorage(dbProvider.StateDb), LimboLogs.Instance); + using IContainer container = new ContainerBuilder() + .AddModule(new TestSynchronizerModule(new SyncConfig())) + .Build(); + + SnapProvider snapProvider = container.Resolve(); Assert.That( - () => sut.AddAccountRange( + () => snapProvider.AddAccountRange( 0, Keccak.Zero, Keccak.Zero, @@ -40,23 +52,158 @@ public void AddAccountRange_AccountListIsEmpty_ThrowArgumentException() Array.Empty().AsReadOnly()), Throws.ArgumentException); } - [Test] public void AddAccountRange_ResponseHasEmptyListOfAccountsAndOneProof_ReturnsExpiredRootHash() { - MemDb db = new(); - IDbProvider dbProvider = new DbProvider(); - dbProvider.RegisterDb(DbNames.State, db); - using ProgressTracker progressTracker = new(Substitute.For(), dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - dbProvider.RegisterDb(DbNames.Code, new MemDb()); - AccountRange accountRange = new(Keccak.Zero, Keccak.Zero, Keccak.MaxValue); + using IContainer container = new ContainerBuilder() + .AddModule(new TestSynchronizerModule(new SyncConfig())) + .Build(); + + SnapProvider snapProvider = container.Resolve(); + using AccountsAndProofs accountsAndProofs = new(); + AccountRange accountRange = new(Keccak.Zero, Keccak.Zero, Keccak.MaxValue); accountsAndProofs.PathAndAccounts = new List().ToPooledList(); accountsAndProofs.Proofs = new List { new byte[] { 0x0 } }.ToPooledList(); - SnapProvider sut = new(progressTracker, dbProvider.CodeDb, new NodeStorage(dbProvider.StateDb), LimboLogs.Instance); + snapProvider.AddAccountRange(accountRange, accountsAndProofs).Should().Be(AddRangeResult.ExpiredRootHash); + } + + [Test] + public void AddAccountRange_SetStartRange_ToAfterLastPath() + { + (Hash256, Account)[] entries = + [ + (TestItem.KeccakA, TestItem.GenerateRandomAccount()), + (TestItem.KeccakB, TestItem.GenerateRandomAccount()), + (TestItem.KeccakC, TestItem.GenerateRandomAccount()), + (TestItem.KeccakD, TestItem.GenerateRandomAccount()), + (TestItem.KeccakE, TestItem.GenerateRandomAccount()), + (TestItem.KeccakF, TestItem.GenerateRandomAccount()), + ]; + Array.Sort(entries, (e1, e2) => e1.Item1.CompareTo(e2.Item1)); + + (SnapServer ss, Hash256 root) = BuildSnapServerFromEntries(entries); + + using IContainer container = new ContainerBuilder() + .AddModule(new TestSynchronizerModule(new SyncConfig() + { + SnapSyncAccountRangePartitionCount = 1 + })) + .WithSuggestedHeaderOfStateRoot(root) + .Build(); + + SnapProvider snapProvider = container.Resolve(); + ProgressTracker progressTracker = container.Resolve(); + + (IOwnedReadOnlyList accounts, IOwnedReadOnlyList proofs) res = ss.GetAccountRanges( + root, Keccak.Zero, entries[3].Item1, 1.MB(), default); + + progressTracker.IsFinished(out SnapSyncBatch? batch).Should().Be(false); + + using AccountsAndProofs accountsAndProofs = new(); + accountsAndProofs.PathAndAccounts = res.accounts; + accountsAndProofs.Proofs = res.proofs; + + snapProvider.AddAccountRange(batch?.AccountRangeRequest!, accountsAndProofs).Should().Be(AddRangeResult.OK); + progressTracker.IsFinished(out batch).Should().Be(false); + batch?.AccountRangeRequest?.StartingHash.Should().BeGreaterThan(entries[3].Item1); + batch?.AccountRangeRequest?.StartingHash.Should().BeLessThan(entries[4].Item1); + } + + [Test] + public void AddAccountRange_ShouldNotStoreStorageAfterLimit() + { + (Hash256, Account)[] entries = + [ + (TestItem.KeccakA, TestItem.GenerateRandomAccount().WithChangedStorageRoot(TestItem.GetRandomKeccak())), + (TestItem.KeccakB, TestItem.GenerateRandomAccount().WithChangedStorageRoot(TestItem.GetRandomKeccak())), + (TestItem.KeccakC, TestItem.GenerateRandomAccount().WithChangedStorageRoot(TestItem.GetRandomKeccak())), + (TestItem.KeccakD, TestItem.GenerateRandomAccount().WithChangedStorageRoot(TestItem.GetRandomKeccak())), + (TestItem.KeccakE, TestItem.GenerateRandomAccount().WithChangedStorageRoot(TestItem.GetRandomKeccak())), + (TestItem.KeccakF, TestItem.GenerateRandomAccount().WithChangedStorageRoot(TestItem.GetRandomKeccak())), + ]; + Array.Sort(entries, (e1, e2) => e1.Item1.CompareTo(e2.Item1)); + + (SnapServer ss, Hash256 root) = BuildSnapServerFromEntries(entries); + + using IContainer container = new ContainerBuilder() + .AddModule(new TestSynchronizerModule(new SyncConfig() + { + SnapSyncAccountRangePartitionCount = 2 + })) + .WithSuggestedHeaderOfStateRoot(root) + .Build(); + + SnapProvider snapProvider = container.Resolve(); + ProgressTracker progressTracker = container.Resolve(); + + (IOwnedReadOnlyList accounts, IOwnedReadOnlyList proofs) res = ss.GetAccountRanges( + root, Keccak.Zero, Keccak.MaxValue, 1.MB(), default); + + progressTracker.IsFinished(out SnapSyncBatch? batch).Should().Be(false); + + using AccountsAndProofs accountsAndProofs = new(); + accountsAndProofs.PathAndAccounts = res.accounts; + accountsAndProofs.Proofs = res.proofs; - sut.AddAccountRange(accountRange, accountsAndProofs).Should().Be(AddRangeResult.ExpiredRootHash); + snapProvider.AddAccountRange(batch?.AccountRangeRequest!, accountsAndProofs).Should().Be(AddRangeResult.OK); + + container.ResolveNamed(DbNames.State).GetAllKeys().Count().Should().Be(6); + } + + [TestCase("badreq-roothash.zip")] + [TestCase("badreq-roothash-2.zip")] + [TestCase("badreq-roothash-3.zip")] + [TestCase("badreq-trieexception.zip")] + public void Test_EdgeCases(string testFileName) + { + using DeflateStream decompressor = + new DeflateStream( + GetType().Assembly + .GetManifestResourceStream($"Nethermind.Synchronization.Test.SnapSync.TestFixtures.{testFileName}")!, + CompressionMode.Decompress); + BadReq asReq = JsonSerializer.Deserialize(decompressor)!; + AccountDecoder acd = new AccountDecoder(); + Account[] accounts = asReq.Accounts.Select((bt) => acd.Decode(new RlpStream(Bytes.FromHexString(bt)))!).ToArray(); + ValueHash256[] paths = asReq.Paths.Select((bt) => new ValueHash256(Bytes.FromHexString(bt))).ToArray(); + List pathWithAccounts = accounts.Select((acc, idx) => new PathWithAccount(paths[idx], acc)).ToList(); + List proofs = asReq.Proofs.Select((str) => Bytes.FromHexString(str)).ToList(); + + StateTree stree = new StateTree(new TrieStore(new TestMemDb(), LimboLogs.Instance), LimboLogs.Instance); + SnapProviderHelper.AddAccountRange( + stree, + 0, + new ValueHash256(asReq.Root), + new ValueHash256(asReq.StartingHash), + new ValueHash256(asReq.LimitHash), + pathWithAccounts, + proofs).result.Should().Be(AddRangeResult.OK); } + private record BadReq( + string Root, + string StartingHash, + string LimitHash, + List Proofs, + List Paths, + List Accounts + ); + + private static (SnapServer, Hash256) BuildSnapServerFromEntries((Hash256, Account)[] entries) + { + TestMemDb stateDb = new TestMemDb(); + TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); + StateTree st = new StateTree(trieStore, LimboLogs.Instance); + foreach (var entry in entries) + { + st.Set(entry.Item1, entry.Item2); + } + st.Commit(); + + ILastNStateRootTracker stateRootTracker = Substitute.For(); + stateRootTracker.HasStateRoot(st.RootHash).Returns(true); + var ss = new SnapServer(trieStore.AsReadOnly(), new TestMemDb(), stateRootTracker, LimboLogs.Instance); + return (ss, st.RootHash); + } } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestFixtures/badreq-roothash-2.zip b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestFixtures/badreq-roothash-2.zip new file mode 100644 index 00000000000..3912b587269 Binary files /dev/null and b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestFixtures/badreq-roothash-2.zip differ diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestFixtures/badreq-roothash-3.zip b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestFixtures/badreq-roothash-3.zip new file mode 100644 index 00000000000..1e4fd13e123 Binary files /dev/null and b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestFixtures/badreq-roothash-3.zip differ diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestFixtures/badreq-roothash.zip b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestFixtures/badreq-roothash.zip new file mode 100644 index 00000000000..9eefc5312bd Binary files /dev/null and b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestFixtures/badreq-roothash.zip differ diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestFixtures/badreq-trieexception.zip b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestFixtures/badreq-trieexception.zip new file mode 100644 index 00000000000..9d626d0bb5e Binary files /dev/null and b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestFixtures/badreq-trieexception.zip differ diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index 6e57e836982..bb6cc747ee5 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -361,6 +361,8 @@ private SyncTestContext CreateSyncManager(int index) ContainerBuilder builder = new ContainerBuilder(); builder + .AddModule(new DbModule()) + .AddModule(new SynchronizerModule(syncConfig)) .AddSingleton(dbProvider) .AddSingleton(new NodeStorage(dbProvider.StateDb)) .AddSingleton(MainnetSpecProvider.Instance) @@ -379,8 +381,6 @@ private SyncTestContext CreateSyncManager(int index) .AddSingleton(receiptStorage) .AddSingleton(No.BeaconSync) .AddSingleton(logManager); - dbProvider.ConfigureServiceCollection(builder); - builder.RegisterModule(new SynchronizerModule(syncConfig)); IContainer container = builder.Build(); Synchronizer synchronizer = container.Resolve(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index 47c721d274c..f0d71662b6b 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -345,9 +345,10 @@ ISyncConfig GetSyncConfig() => IInvalidChainTracker invalidChainTracker = new NoopInvalidChainTracker(); ContainerBuilder builder = new ContainerBuilder(); - dbProvider.ConfigureServiceCollection(builder); builder + .AddModule(new DbModule()) + .AddModule(new SynchronizerModule(syncConfig)) .AddSingleton(dbProvider) .AddSingleton(nodeStorage) .AddSingleton(MainnetSpecProvider.Instance) @@ -370,8 +371,6 @@ ISyncConfig GetSyncConfig() => .AddSingleton(beaconPivot) .AddSingleton(_logManager); - builder.RegisterModule(new SynchronizerModule(syncConfig)); - if (IsMerge(synchronizerType)) { builder.RegisterModule(new MergeSynchronizerModule()); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/TestSynchronizerModule.cs b/src/Nethermind/Nethermind.Synchronization.Test/TestSynchronizerModule.cs new file mode 100644 index 00000000000..d6868ba4f09 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/TestSynchronizerModule.cs @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Autofac; +using Nethermind.Blockchain; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Core; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.Trie; +using NSubstitute; + +namespace Nethermind.Synchronization.Test; + +public class TestSynchronizerModule(ISyncConfig syncConfig) : Module +{ + protected override void Load(ContainerBuilder builder) + { + base.Load(builder); + + builder + .AddModule(new SynchronizerModule(syncConfig)) + .AddModule(new DbModule()) + .AddSingleton(TestMemDbProvider.Init()) + .Map(dbProvider => new NodeStorage(dbProvider.StateDb)) + .AddSingleton(Substitute.For()) + .AddSingleton(syncConfig) + .AddSingleton(LimboLogs.Instance); + + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs index 132ff77c0da..4b97e4c6fe3 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers.Binary; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; @@ -72,8 +73,8 @@ public ProgressTracker(IBlockTree blockTree, IDb db, ILogManager logManager, int _pivot = new Pivot(blockTree, logManager); - if (accountRangePartitionCount < 1 || accountRangePartitionCount > 256) - throw new ArgumentException("Account range partition must be between 1 to 256."); + if (accountRangePartitionCount < 1 || accountRangePartitionCount > int.MaxValue) + throw new ArgumentException($"Account range partition must be between 1 to {int.MaxValue}."); _accountRangePartitionCount = accountRangePartitionCount; SetupAccountRangePartition(); @@ -84,23 +85,20 @@ public ProgressTracker(IBlockTree blockTree, IDb db, ILogManager logManager, int private void SetupAccountRangePartition() { - // Confusingly dividing the range evenly via UInt256 for example, consistently cause root hash mismatch. - // The mismatch happens on exactly the same partition every time, suggesting tome kind of boundary issues - // either on proof generation or validation. - byte curStartingPath = 0; - int partitionSize = (256 / _accountRangePartitionCount); + uint curStartingPath = 0; + uint partitionSize = (uint)(((ulong)uint.MaxValue + 1) / (ulong)_accountRangePartitionCount); for (var i = 0; i < _accountRangePartitionCount; i++) { AccountRangePartition partition = new AccountRangePartition(); Hash256 startingPath = new Hash256(Keccak.Zero.Bytes); - startingPath.Bytes[0] = curStartingPath; + BinaryPrimitives.WriteUInt32BigEndian(startingPath.Bytes, curStartingPath); partition.NextAccountPath = startingPath; partition.AccountPathStart = startingPath; - curStartingPath += (byte)partitionSize; + curStartingPath += partitionSize; Hash256 limitPath; @@ -112,7 +110,7 @@ private void SetupAccountRangePartition() else { limitPath = new Hash256(Keccak.Zero.Bytes); - limitPath.Bytes[0] = curStartingPath; + BinaryPrimitives.WriteUInt32BigEndian(limitPath.Bytes, curStartingPath); } partition.AccountPathLimit = limitPath; diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index 51548bfa9f6..99e598804c8 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -15,6 +15,7 @@ using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Db; +using Nethermind.Int256; using Nethermind.Logging; using Nethermind.State; using Nethermind.State.Snap; @@ -120,7 +121,9 @@ public AddRangeResult AddAccountRange( _progressTracker.EnqueueCodeHashes(filteredCodeHashes.AsSpan()); - _progressTracker.UpdateAccountRangePartitionProgress(effectiveHashLimit, accounts[^1].Path, moreChildrenToRight); + UInt256 nextPath = accounts[^1].Path.ToUInt256(); + nextPath += UInt256.One; + _progressTracker.UpdateAccountRangePartitionProgress(effectiveHashLimit, new ValueHash256(nextPath), moreChildrenToRight); } else if (result == AddRangeResult.MissingRootHashInProofs) { diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs index 967e10cad47..2bf517043f2 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs @@ -44,13 +44,21 @@ public static (AddRangeResult result, bool moreChildrenToRight, List accountsWithStorage = new(); List codeHashes = new(); + bool hasExtraStorage = false; for (var index = 0; index < accounts.Count; index++) { PathWithAccount account = accounts[index]; if (account.Account.HasStorage) { - accountsWithStorage.Add(account); + if (account.Path >= limitHash || account.Path < startingHash) + { + hasExtraStorage = true; + } + else + { + accountsWithStorage.Add(account); + } } if (account.Account.HasCode) @@ -72,7 +80,30 @@ public static (AddRangeResult result, bool moreChildrenToRight, List= limitHash || account.Path < startingHash) continue; + _ = tree.Set(account.Path, account.Account); + } + } + else + { + StitchBoundaries(sortedBoundaryList, tree.TrieStore); + } tree.Commit(skipRoot: true, writeFlags: WriteFlags.DisableWAL); @@ -148,30 +179,28 @@ private static (AddRangeResult result, List<(TrieNode, TreePath)> sortedBoundary return (AddRangeResult.MissingRootHashInProofs, null, true); } - // BytesToNibbleBytes will throw if the input is not 32 bytes long, so we can use stackalloc+SkipLocalsInit - Span leftBoundary = stackalloc byte[64]; - Nibbles.BytesToNibbleBytes(effectiveStartingHAsh.Bytes, leftBoundary); - Span rightBoundary = stackalloc byte[64]; - Nibbles.BytesToNibbleBytes(endHash.Bytes, rightBoundary); - Span rightLimit = stackalloc byte[64]; - Nibbles.BytesToNibbleBytes(limitHash.Bytes, rightLimit); + TreePath leftBoundaryPath = TreePath.FromPath(effectiveStartingHAsh.Bytes); + TreePath rightBoundaryPath = TreePath.FromPath(endHash.Bytes); + TreePath rightLimitPath = TreePath.FromPath(limitHash.Bytes); // For when in very-very unlikely case where the last remaining address is Keccak.MaxValue, (who knows why, // the chain have special handling for it maybe) and it is not included the returned account range, (again, // very-very unlikely), we want `moreChildrenToRight` to return true. bool noLimit = limitHash == ValueKeccak.MaxValue; - Stack<(TrieNode parent, TrieNode node, int pathIndex, List path)> proofNodesToProcess = new(); + // Connect the proof nodes starting from state root. + // It also remove child path which is within the start/end range. If key are missing, the resolved + // hash will not match. + Stack<(TrieNode node, TreePath path)> proofNodesToProcess = new(); tree.RootRef = root; - proofNodesToProcess.Push((null, root, -1, new List())); + proofNodesToProcess.Push((root, TreePath.Empty)); sortedBoundaryList.Add((root, TreePath.Empty)); bool moreChildrenToRight = false; - while (proofNodesToProcess.Count > 0) { - (TrieNode parent, TrieNode node, int pathIndex, List path) = proofNodesToProcess.Pop(); + (TrieNode node, TreePath path) = proofNodesToProcess.Pop(); if (node.IsExtension) { @@ -181,39 +210,19 @@ private static (AddRangeResult result, List<(TrieNode, TreePath)> sortedBoundary { node.SetChild(0, child); - pathIndex += node.Key.Length; - path.AddRange(node.Key); - proofNodesToProcess.Push((node, child, pathIndex, path)); - sortedBoundaryList.Add((child, TreePath.FromNibble(CollectionsMarshal.AsSpan(path)))); - } - else - { - Span pathSpan = CollectionsMarshal.AsSpan(path); - if (Bytes.BytesComparer.Compare(pathSpan, leftBoundary[0..path.Count]) >= 0 - && parent is not null - && parent.IsBranch) - { - for (int i = 0; i < 15; i++) - { - if (parent.GetChildHashAsValueKeccak(i, out ValueHash256 kec) && kec == node.Keccak) - { - parent.SetChild(i, null); - break; - } - } - } + TreePath childPath = path.Append(node.Key); + + proofNodesToProcess.Push((child, childPath)); + sortedBoundaryList.Add((child, childPath)); } } } if (node.IsBranch) { - pathIndex++; - - Span pathSpan = CollectionsMarshal.AsSpan(path); - int left = Bytes.BytesComparer.Compare(pathSpan, leftBoundary[0..path.Count]) == 0 ? leftBoundary[pathIndex] : 0; - int right = Bytes.BytesComparer.Compare(pathSpan, rightBoundary[0..path.Count]) == 0 ? rightBoundary[pathIndex] : 15; - int limit = Bytes.BytesComparer.Compare(pathSpan, rightLimit[0..path.Count]) == 0 ? rightLimit[pathIndex] : 15; + int left = leftBoundaryPath.CompareToTruncated(path, path.Length) == 0 ? leftBoundaryPath[path.Length] : 0; + int right = rightBoundaryPath.CompareToTruncated(path, path.Length) == 0 ? rightBoundaryPath[path.Length] : 15; + int limit = rightLimitPath.CompareToTruncated(path, path.Length) == 0 ? rightLimitPath[path.Length] : 15; int maxIndex = moreChildrenToRight ? right : 15; @@ -225,23 +234,42 @@ private static (AddRangeResult result, List<(TrieNode, TreePath)> sortedBoundary if (ci >= left && ci <= right) { + // Clear child within boundary node.SetChild(ci, null); } if (hasKeccak && (ci == left || ci == right) && dict.TryGetValue(childKeccak, out TrieNode child)) { - if (!child.IsLeaf) + TreePath childPath = path.Append(ci); + + if (child.IsBranch) { node.SetChild(ci, child); - // TODO: we should optimize it - copy only if there are two boundary children - List newPath = new(path) + proofNodesToProcess.Push((child, childPath)); + sortedBoundaryList.Add((child, childPath)); + } + else if (child.IsExtension) + { + // If its an extension, its path + key must be outside or equal to the boundary. + TreePath wholePath = childPath.Append(child.Key); + if (leftBoundaryPath.CompareToTruncated(wholePath, wholePath.Length) >= 0 || rightBoundaryPath.CompareToTruncated(wholePath, wholePath.Length) <= 0) { - (byte)ci - }; - - proofNodesToProcess.Push((node, child, pathIndex, newPath)); - sortedBoundaryList.Add((child, TreePath.FromNibble(CollectionsMarshal.AsSpan(newPath)))); + node.SetChild(ci, child); + proofNodesToProcess.Push((child, childPath)); + sortedBoundaryList.Add((child, childPath)); + } + } + else + { + // If its a leaf, its path + key must be outside the boundary. + TreePath wholePath = childPath.Append(child.Key); + if (leftBoundaryPath.CompareToTruncated(wholePath, wholePath.Length) > 0 || rightBoundaryPath.CompareToTruncated(wholePath, wholePath.Length) < 0) + { + node.SetChild(ci, child); + proofNodesToProcess.Push((child, childPath)); + sortedBoundaryList.Add((child, childPath)); + } } } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TreePath.cs b/src/Nethermind/Nethermind.Trie/Pruning/TreePath.cs index 36306f16241..16f36170263 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TreePath.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TreePath.cs @@ -136,7 +136,7 @@ internal void AppendMut(ReadOnlySpan nibbles) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void AppendMut(int nib) + public void AppendMut(int nib) { this[Length] = nib; Length++; @@ -309,6 +309,12 @@ public readonly int CompareTo(in TreePath otherTree) return Length.CompareTo(otherTree.Length); } + /// + /// Compare with otherTree, as if this TreePath was truncated to `length`. + /// + /// + /// + /// public readonly int CompareToTruncated(in TreePath otherTree, int length) { int minLength = Math.Min(length, otherTree.Length);