Merge remote-tracking branch 'origin/master' into feature/eip-4844-ga…

…s-calculations

.github/workflows/hive-tests.yml

@@ -83,7 +83,7 @@ jobs:
           ref: ${{ env.HIVE_BRANCH }}
           path: hive
       - name: Patch Hive Dockerfile
-        run: sed -i 's#FROM $user/$repo:$branch#FROM nethermind:test-${{ github.sha }}#g' hive/clients/nethermind/Dockerfile
+        run: sed -i 's#FROM $baseimage:$tag#FROM nethermind:test-${{ github.sha }}#g' hive/clients/nethermind/Dockerfile
       - name: Build Hive
         working-directory: hive
         run: go build .

src/Nethermind/Ethereum.Blockchain.Test/Eip3855Tests.cs → src/Nethermind/Ethereum.Blockchain.Test/Eip3651WarmCoinbaseTests.cs

@@ -9,7 +9,7 @@ namespace Ethereum.Blockchain.Test;
 
 [TestFixture]
 [Parallelizable(ParallelScope.All)]
-public class Eip3855Tests : GeneralStateTestBase
+public class Eip3651WarmCoinbaseTests : GeneralStateTestBase
 {
     [TestCaseSource(nameof(LoadTests))]
     public void Test(GeneralStateTest test)
@@ -19,7 +19,7 @@ public void Test(GeneralStateTest test)
 
     public static IEnumerable<GeneralStateTest> LoadTests()
     {
-        var loader = new TestsSourceLoader(new LoadGeneralStateTestsStrategy(), "stEIP3855");
+        var loader = new TestsSourceLoader(new LoadGeneralStateTestsStrategy(), "stEIP3651-warmcoinbase");
         return (IEnumerable<GeneralStateTest>)loader.LoadTests();
     }
 }

src/Nethermind/Ethereum.Blockchain.Test/Eip3651Tests.cs → src/Nethermind/Ethereum.Blockchain.Test/Eip3855Push0Tests.cs

@@ -9,7 +9,7 @@ namespace Ethereum.Blockchain.Test;
 
 [TestFixture]
 [Parallelizable(ParallelScope.All)]
-public class Eip3651Tests : GeneralStateTestBase
+public class Eip3855Push0Tests : GeneralStateTestBase
 {
     [TestCaseSource(nameof(LoadTests))]
     public void Test(GeneralStateTest test)
@@ -19,7 +19,7 @@ public void Test(GeneralStateTest test)
 
     public static IEnumerable<GeneralStateTest> LoadTests()
     {
-        var loader = new TestsSourceLoader(new LoadGeneralStateTestsStrategy(), "stEIP3651");
+        var loader = new TestsSourceLoader(new LoadGeneralStateTestsStrategy(), "stEIP3855-push0");
         return (IEnumerable<GeneralStateTest>)loader.LoadTests();
     }
 }

src/Nethermind/Ethereum.Blockchain.Test/Eip3860Tests.cs → src/Nethermind/Ethereum.Blockchain.Test/Eip3860LimitmeterInitCodeTests.cs

@@ -9,7 +9,7 @@ namespace Ethereum.Blockchain.Test;
 
 [TestFixture]
 [Parallelizable(ParallelScope.All)]
-public class Eip3860Tests : GeneralStateTestBase
+public class Eip3860LimitmeterInitCodeTests : GeneralStateTestBase
 {
     [TestCaseSource(nameof(LoadTests))]
     public void Test(GeneralStateTest test)
@@ -19,7 +19,7 @@ public void Test(GeneralStateTest test)
 
     public static IEnumerable<GeneralStateTest> LoadTests()
     {
-        var loader = new TestsSourceLoader(new LoadGeneralStateTestsStrategy(), "stEIP3860");
+        var loader = new TestsSourceLoader(new LoadGeneralStateTestsStrategy(), "stEIP3860-limitmeterinitcode");
         return (IEnumerable<GeneralStateTest>)loader.LoadTests();
     }
 }

src/Nethermind/Ethereum.Blockchain.Test/Ethereum.Blockchain.Test.csproj

@@ -35,7 +35,7 @@
       <Link>%(RecursiveDir)%(FileName)%(Extension)</Link>
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </EmbeddedResource>
-    <EmbeddedResource Include="..\..\tests\GeneralStateTests\EIPTests\stEIP*\**\*.*">
+    <EmbeddedResource Include="..\..\tests\GeneralStateTests\Shanghai\st*\**\*.*">
       <Link>%(RecursiveDir)%(FileName)%(Extension)</Link>
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </EmbeddedResource>

src/Nethermind/Ethereum.Blockchain.Test/MetaTests.cs

@@ -34,7 +34,7 @@ public void All_categories_are_tested()
             List<string> missingCategories = new List<string>();
             foreach (string directory in directories)
             {
-                string expectedTypeName = ExpectedTypeName(directory);
+                string expectedTypeName = ExpectedTypeName(directory).Replace("-", "");
                 Type type = types.SingleOrDefault(t => string.Equals(t.Name, expectedTypeName, StringComparison.InvariantCultureIgnoreCase));
                 if (type == null && !excludesDirectories.Contains(directory))
                 {

src/Nethermind/Nethermind.Api/IInitConfig.cs

@@ -35,9 +35,12 @@ public interface IInitConfig : IConfig
         [ConfigItem(Description = "Path to the chain definition file created by Hive for test purpouse", DefaultValue = "chainspec/test.json")]
         string HiveChainSpecPath { get; set; }
 
-        [ConfigItem(Description = "Base directoy path for all the nethermind databases.", DefaultValue = "\"db\"")]
+        [ConfigItem(Description = "Base directory path for all the nethermind databases.", DefaultValue = "\"db\"")]
         string BaseDbPath { get; set; }
 
+        [ConfigItem(Description = "Kzg trusted setup file path", DefaultValue = "null")]
+        string? KzgSetupPath { get; set; }
+
         [ConfigItem(Description = "Hash of the genesis block - if the default null value is left then the genesis block validity will not be checked which is useful for ad hoc test/private networks.", DefaultValue = "null")]
         string? GenesisHash { get; set; }
 

src/Nethermind/Nethermind.Api/InitConfig.cs

@@ -21,6 +21,7 @@ public class InitConfig : IInitConfig
         public string LogFileName { get; set; } = "log.txt";
         public string? GenesisHash { get; set; }
         public string StaticNodesPath { get; set; } = "Data/static-nodes.json";
+        public string? KzgSetupPath { get; set; } = null;
         public string LogDirectory { get; set; } = "logs";
         public string? LogRules { get; set; } = null;
         public bool StoreReceipts { get; set; } = true;

src/Nethermind/Nethermind.Blockchain.Test/BlockTreeTests.cs

@@ -316,8 +316,8 @@ public void Add_and_find_branch()
             BlockTree blockTree = BuildBlockTree();
             Block block = Build.A.Block.TestObject;
             blockTree.SuggestBlock(block);
-            Block found = blockTree.FindBlock(block.Hash, BlockTreeLookupOptions.None);
-            Assert.That(found.Header.CalculateHash(), Is.EqualTo(block.Hash));
+            Block? found = blockTree.FindBlock(block.Hash, BlockTreeLookupOptions.None);
+            Assert.That(found?.Header.CalculateHash(), Is.EqualTo(block.Hash));
         }
 
         [Test, Timeout(Timeout.MaxTestTime)]
@@ -326,8 +326,8 @@ public void Add_on_branch_move_find()
             BlockTree blockTree = BuildBlockTree();
             Block block = Build.A.Block.TestObject;
             AddToMain(blockTree, block);
-            Block found = blockTree.FindBlock(block.Hash, BlockTreeLookupOptions.RequireCanonical);
-            Assert.That(found.Header.CalculateHash(), Is.EqualTo(block.Hash));
+            Block? found = blockTree.FindBlock(block.Hash, BlockTreeLookupOptions.RequireCanonical);
+            Assert.That(found?.Header.CalculateHash(), Is.EqualTo(block.Hash));
         }
 
         [Test, Timeout(Timeout.MaxTestTime)]
@@ -336,8 +336,8 @@ public void Add_on_branch_move_find_via_block_finder_interface()
             BlockTree blockTree = BuildBlockTree();
             Block block = Build.A.Block.TestObject;
             AddToMain(blockTree, block);
-            Block found = ((IBlockFinder)blockTree).FindBlock(new BlockParameter(block.Hash, true));
-            Assert.That(found.Header.CalculateHash(), Is.EqualTo(block.Hash));
+            Block? found = ((IBlockFinder)blockTree).FindBlock(new BlockParameter(block.Hash, true));
+            Assert.That(found?.Header.CalculateHash(), Is.EqualTo(block.Hash));
         }
 
         [Test, Timeout(Timeout.MaxTestTime)]
@@ -346,7 +346,7 @@ public void Add_on_branch_and_not_find_on_main()
             BlockTree blockTree = BuildBlockTree();
             Block block = Build.A.Block.TestObject;
             blockTree.SuggestBlock(block);
-            Block found = blockTree.FindBlock(block.Hash, BlockTreeLookupOptions.RequireCanonical);
+            Block? found = blockTree.FindBlock(block.Hash, BlockTreeLookupOptions.RequireCanonical);
             Assert.IsNull(found);
         }
 
@@ -356,7 +356,7 @@ public void Add_on_branch_and_not_find_on_main_via_block_finder_interface()
             BlockTree blockTree = BuildBlockTree();
             Block block = Build.A.Block.TestObject;
             blockTree.SuggestBlock(block);
-            Block found = ((IBlockFinder)blockTree).FindBlock(new BlockParameter(block.Hash, true));
+            Block? found = ((IBlockFinder)blockTree).FindBlock(new BlockParameter(block.Hash, true));
             Assert.IsNull(found);
         }
 
@@ -371,8 +371,8 @@ public void Find_by_number_basic()
             AddToMain(blockTree, block1);
             AddToMain(blockTree, block2);
 
-            Block found = blockTree.FindBlock(2, BlockTreeLookupOptions.None);
-            Assert.That(found.Header.CalculateHash(), Is.EqualTo(block2.Hash));
+            Block? found = blockTree.FindBlock(2, BlockTreeLookupOptions.None);
+            Assert.That(found?.Header.CalculateHash(), Is.EqualTo(block2.Hash));
         }
 
         [Test, Timeout(Timeout.MaxTestTime)]
@@ -386,7 +386,7 @@ public void Find_by_number_beyond_what_is_known_returns_null()
             AddToMain(blockTree, block1);
             AddToMain(blockTree, block2);
 
-            Block found = blockTree.FindBlock(1920000, BlockTreeLookupOptions.None);
+            Block? found = blockTree.FindBlock(1920000, BlockTreeLookupOptions.None);
             Assert.Null(found);
         }
 
@@ -399,7 +399,7 @@ public void Find_by_number_returns_null_when_block_is_missing()
             AddToMain(blockTree, block0);
             AddToMain(blockTree, block1);
 
-            Block found = blockTree.FindBlock(5, BlockTreeLookupOptions.None);
+            Block? found = blockTree.FindBlock(5, BlockTreeLookupOptions.None);
             Assert.IsNull(found);
         }
 
@@ -751,9 +751,9 @@ public void Stores_multiple_blocks_per_level()
             AddToMain(blockTree, block1);
             blockTree.SuggestBlock(block1B);
 
-            Block found = blockTree.FindBlock(block1B.Hash, BlockTreeLookupOptions.None);
+            Block? found = blockTree.FindBlock(block1B.Hash, BlockTreeLookupOptions.None);
 
-            Assert.That(found.Header.CalculateHash(), Is.EqualTo(block1B.Hash));
+            Assert.That(found?.Header.CalculateHash(), Is.EqualTo(block1B.Hash));
         }
 
         [Test, Timeout(Timeout.MaxTestTime)]
@@ -1101,7 +1101,7 @@ public void When_deleting_invalid_block_does_not_delete_blocks_that_are_not_its_
             Assert.That(tree.BestSuggestedHeader!.Hash, Is.EqualTo(block5.Hash), "suggested");
         }
 
-        [Test, Timeout(Timeout.MaxTestTime), TestCaseSource("SourceOfBSearchTestCases")]
+        [Test, Timeout(Timeout.MaxTestTime), TestCaseSource(nameof(SourceOfBSearchTestCases))]
         public void Loads_lowest_inserted_header_correctly(long beginIndex, long insertedBlocks)
         {
             long? expectedResult = insertedBlocks == 0L ? (long?)null : beginIndex - insertedBlocks + 1L;
@@ -1128,7 +1128,7 @@ public void Loads_lowest_inserted_header_correctly(long beginIndex, long inserte
             Assert.That(loadedTree.LowestInsertedHeader?.Number, Is.EqualTo(expectedResult), "loaded tree");
         }
 
-        [Test, Timeout(Timeout.MaxTestTime), TestCaseSource("SourceOfBSearchTestCases")]
+        [Test, Timeout(Timeout.MaxTestTime), TestCaseSource(nameof(SourceOfBSearchTestCases))]
         public void Loads_lowest_inserted_body_correctly(long beginIndex, long insertedBlocks)
         {
             // left old code to prove that it does not matter for the result nowadays

src/Nethermind/Nethermind.Blockchain.Test/BlockhashProviderTests.cs

@@ -22,10 +22,10 @@ public void Can_get_parent_only_headers()
             BlockTree tree = Build.A.BlockTree(genesis).OfHeadersOnly.OfChainLength(chainLength).TestObject;
 
             BlockhashProvider provider = new(tree, LimboLogs.Instance);
-            BlockHeader head = tree.FindHeader(chainLength - 1, BlockTreeLookupOptions.None);
-            Block current = Build.A.Block.WithParent(head).TestObject;
+            BlockHeader? head = tree.FindHeader(chainLength - 1, BlockTreeLookupOptions.None);
+            Block current = Build.A.Block.WithParent(head!).TestObject;
             Keccak result = provider.GetBlockhash(current.Header, chainLength - 1);
-            Assert.That(result, Is.EqualTo(head.Hash));
+            Assert.That(result, Is.EqualTo(head?.Hash));
         }
 
         [Test, Timeout(Timeout.MaxTestTime)]

src/Nethermind/Nethermind.Blockchain.Test/Consensus/CompositeTxSourceTests.cs

@@ -27,7 +27,7 @@ public void To_string_does_not_throw()
         [Test, Timeout(Timeout.MaxTestTime)]
         public void Throws_on_null_argument()
         {
-            Assert.Throws<ArgumentNullException>(() => new CompositeTxSource(null));
+            Assert.Throws<ArgumentNullException>(() => new CompositeTxSource(null!));
         }
 
         [Test, Timeout(Timeout.MaxTestTime)]

src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO.Abstractions;
 using System.Threading;
 using System.Threading.Tasks;
 using FluentAssertions;
@@ -174,6 +175,8 @@ private class TestContext
             public FullPruner Pruner { get; }
             public MemDb TrieDb { get; }
             public TestMemDb CopyDb { get; }
+            public IDriveInfo DriveInfo { get; set; } = Substitute.For<IDriveInfo>();
+            public IChainEstimations _chainEstimations = ChainSizes.UnknownChain.Instance;
 
             public IProcessExitSource ProcessExitSource { get; } = Substitute.For<IProcessExitSource>();
 
@@ -201,7 +204,7 @@ public TestContext(
                     FullPruningMaxDegreeOfParallelism = degreeOfParallelism,
                     FullPruningMemoryBudgetMb = fullScanMemoryBudgetMb,
                     FullPruningCompletionBehavior = completionBehavior
-                }, BlockTree, StateReader, ProcessExitSource, LimboLogs.Instance);
+                }, BlockTree, StateReader, ProcessExitSource, _chainEstimations, DriveInfo, LimboLogs.Instance);
             }
 
             public async Task<bool> WaitForPruning()

src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs

@@ -39,6 +39,8 @@ public class PruningTestBlockchain : TestBlockchain
             public IPruningTrigger PruningTrigger { get; } = Substitute.For<IPruningTrigger>();
             public FullTestPruner FullPruner { get; private set; }
             public IPruningConfig PruningConfig { get; set; } = new PruningConfig();
+            public IDriveInfo DriveInfo { get; set; } = Substitute.For<IDriveInfo>();
+            public IChainEstimations _chainEstimations = Substitute.For<IChainEstimations>();
             public IProcessExitSource ProcessExitSource { get; } = Substitute.For<IProcessExitSource>();
 
             public PruningTestBlockchain()
@@ -50,7 +52,9 @@ protected override async Task<TestBlockchain> Build(ISpecProvider? specProvider
             {
                 TestBlockchain chain = await base.Build(specProvider, initialValues);
                 PruningDb = (IFullPruningDb)DbProvider.StateDb;
-                FullPruner = new FullTestPruner(PruningDb, PruningTrigger, PruningConfig, BlockTree, StateReader, ProcessExitSource, LogManager);
+                DriveInfo.AvailableFreeSpace.Returns(long.MaxValue);
+                _chainEstimations.StateSize.Returns((long?)null);
+                FullPruner = new FullTestPruner(PruningDb, PruningTrigger, PruningConfig, BlockTree, StateReader, ProcessExitSource, DriveInfo, _chainEstimations, LogManager);
                 return chain;
             }
 
@@ -89,8 +93,10 @@ public FullTestPruner(
                     IBlockTree blockTree,
                     IStateReader stateReader,
                     IProcessExitSource processExitSource,
+                    IDriveInfo driveInfo,
+                    IChainEstimations chainEstimations,
                     ILogManager logManager)
-                    : base(pruningDb, pruningTrigger, pruningConfig, blockTree, stateReader, processExitSource, logManager)
+                    : base(pruningDb, pruningTrigger, pruningConfig, blockTree, stateReader, processExitSource, chainEstimations, driveInfo, logManager)
                 {
                 }
 
@@ -122,6 +128,20 @@ public async Task prune_on_disk_only_once()
             }
         }
 
+        [TestCase(100, 150, false)]
+        [TestCase(200, 100, true)]
+        [TestCase(130, 100, true)]
+        [TestCase(130, 101, false)]
+        public async Task should_check_available_space_before_running(long availableSpace, long requiredSpace, bool isEnoughSpace)
+        {
+            using PruningTestBlockchain chain = await PruningTestBlockchain.Create();
+            chain._chainEstimations.PruningSize.Returns(requiredSpace);
+            chain.DriveInfo.AvailableFreeSpace.Returns(availableSpace);
+            PruningTriggerEventArgs args = new();
+            chain.PruningTrigger.Prune += Raise.Event<EventHandler<PruningTriggerEventArgs>>(args);
+            args.Status.Should().Be(isEnoughSpace ? PruningStatus.Starting : PruningStatus.NotEnoughDiskSpace);
+        }
+
         private static async Task RunPruning(PruningTestBlockchain chain, int time, bool onlyFirstRuns)
         {
             chain.FullPruner.WaitHandle.Reset();

src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs

@@ -0,0 +1,27 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using FluentAssertions;
+using Nethermind.Core;
+using Nethermind.Core.Extensions;
+using NUnit.Framework;
+
+namespace Nethermind.Blockchain.Test;
+
+public class KnownChainSizesTests
+{
+    [Test]
+    public void Update_known_chain_sizes()
+    {
+        // Pruning size have to be updated frequently
+        ChainSizes.CreateChainSizeInfo(BlockchainIds.Mainnet).PruningSize.Should().BeLessThan(200.GB());
+        ChainSizes.CreateChainSizeInfo(BlockchainIds.Goerli).PruningSize.Should().BeLessThan(55.GB());
+        ChainSizes.CreateChainSizeInfo(BlockchainIds.Sepolia).PruningSize.Should().BeLessThan(8.GB());
+
+        ChainSizes.CreateChainSizeInfo(BlockchainIds.Chiado).PruningSize.Should().Be(null);
+        ChainSizes.CreateChainSizeInfo(BlockchainIds.Gnosis).PruningSize.Should().Be(null);
+
+        ChainSizes.CreateChainSizeInfo(BlockchainIds.EnergyWeb).PruningSize.Should().Be(null);
+        ChainSizes.CreateChainSizeInfo(BlockchainIds.Volta).PruningSize.Should().Be(null);
+    }
+}

src/Nethermind/Nethermind.Blockchain.Test/Producers/BuildBlocksOnlyWhenNotProcessingTests.cs

@@ -33,13 +33,13 @@ public async Task should_trigger_block_production_when_queue_empties()
         {
             Context context = new();
             context.BlockProcessingQueue.IsEmpty.Returns(false);
-            Task<Block> buildTask = context.MainBlockProductionTrigger.BuildBlock();
+            Task<Block?> buildTask = context.MainBlockProductionTrigger.BuildBlock();
 
             await Task.Delay(BuildBlocksOnlyWhenNotProcessing.ChainNotYetProcessedMillisecondsDelay * 2);
             buildTask.IsCanceled.Should().BeFalse();
 
             context.BlockProcessingQueue.IsEmpty.Returns(true);
-            Block block = await buildTask;
+            Block? block = await buildTask;
             block.Should().Be(context.DefaultBlock);
             context.TriggeredCount.Should().Be(1);
         }
@@ -76,10 +76,10 @@ public Context()
             public Block DefaultBlock { get; }
             public int TriggeredCount { get; private set; }
 
-            private void OnTriggerBlockProduction(object sender, BlockProductionEventArgs e)
+            private void OnTriggerBlockProduction(object? sender, BlockProductionEventArgs e)
             {
                 TriggeredCount++;
-                e.BlockProductionTask = Task.FromResult(DefaultBlock);
+                e.BlockProductionTask = Task.FromResult<Block?>(DefaultBlock);
             }
 
             public BuildBlocksOnlyWhenNotProcessing Trigger { get; }

src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs

@@ -227,7 +227,7 @@ void SetAccountStates(IEnumerable<Address> missingAddresses)
                 .Where(t => t?.SenderAddress is not null)
                 .GroupBy(t => t.SenderAddress)
                 .ToDictionary(
-                    g => g.Key,
+                    g => g.Key!,
                     g => g.OrderBy(t => t, comparer).ToArray());
             transactionPool.GetPendingTransactionsBySender().Returns(transactions);
             BlocksConfig blocksConfig = new() { MinGasPrice = testCase.MinGasPriceForMining };