Skip to content

Commit

Permalink
Merge branch 'perf/somepath-hashdb' into perf/improved-inmemory-pruning
Browse files Browse the repository at this point in the history
  • Loading branch information
asdacap committed Jan 15, 2024
2 parents e0cf1a6 + 75ef588 commit 3eba584
Show file tree
Hide file tree
Showing 205 changed files with 1,094 additions and 7,887 deletions.
6,756 changes: 116 additions & 6,640 deletions .github/workflows/hive-consensus-tests.yml

Large diffs are not rendered by default.

35 changes: 10 additions & 25 deletions src/Nethermind/Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>
<PackageVersion Include="Antlr4.Runtime.Standard" Version="4.13.1" />
<PackageVersion Include="AspNetCore.HealthChecks.UI" Version="8.0.0" />
<PackageVersion Include="AspNetCore.HealthChecks.UI.Client" Version="8.0.0" />
<PackageVersion Include="AspNetCore.HealthChecks.UI.InMemory.Storage" Version="8.0.0" />
Expand All @@ -14,39 +13,34 @@
<PackageVersion Include="CommandLineParser" Version="2.9.1" />
<PackageVersion Include="ConcurrentHashSet" Version="1.3.0" />
<PackageVersion Include="coverlet.collector" Version="6.0.0" />
<PackageVersion Include="coverlet.msbuild" Version="6.0.0" />
<PackageVersion Include="Crc32.NET" Version="1.2.0" />
<PackageVersion Include="DnsClient" Version="1.7.0" />
<PackageVersion Include="FastEnum" Version="1.8.0" />
<PackageVersion Include="FluentAssertions" Version="6.12.0" />
<PackageVersion Include="FluentAssertions.Json" Version="6.1.0" />
<PackageVersion Include="Google.Protobuf" Version="3.25.1" />
<PackageVersion Include="Google.Protobuf.Tools" Version="3.25.1" />
<PackageVersion Include="Google.Protobuf" Version="3.25.2" />
<PackageVersion Include="Google.Protobuf.Tools" Version="3.25.2" />
<PackageVersion Include="Grpc" Version="2.46.6" />
<PackageVersion Include="Grpc.Tools" Version="2.60.0" />
<PackageVersion Include="HexMate" Version="0.0.3" />
<PackageVersion Include="Jint" Version="2.11.58" />
<PackageVersion Include="MathNet.Numerics.FSharp" Version="5.0.0" />
<PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="8.0.0" />
<PackageVersion Include="Microsoft.AspNetCore.DataProtection.Abstractions" Version="8.0.0" />
<PackageVersion Include="Microsoft.AspNetCore.DataProtection.Extensions" Version="8.0.0" />
<PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
<PackageVersion Include="Microsoft.AspNetCore.WebSockets" Version="2.2.1" />
<PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="8.0.1" />
<PackageVersion Include="Microsoft.AspNetCore.DataProtection.Extensions" Version="8.0.1" />
<PackageVersion Include="Microsoft.ClearScript.V8" Version="7.4.4" />
<PackageVersion Include="Microsoft.ClearScript.V8.Native.linux-arm64" Version="7.4.4" />
<PackageVersion Include="Microsoft.ClearScript.V8.Native.linux-x64" Version="7.4.4" />
<PackageVersion Include="Microsoft.ClearScript.V8.Native.osx-arm64" Version="7.4.4" />
<PackageVersion Include="Microsoft.ClearScript.V8.Native.osx-x64" Version="7.4.4" />
<PackageVersion Include="Microsoft.ClearScript.V8.Native.win-x64" Version="7.4.4" />
<PackageVersion Include="Microsoft.Extensions.CommandLineUtils" Version="1.1.1" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.ObjectPool" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.ObjectPool" Version="8.0.1" />
<PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.0" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
<PackageVersion Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.5" />
<PackageVersion Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.6" />
<PackageVersion Include="Nethermind.Crypto.Pairings" Version="1.0.1" />
<PackageVersion Include="Nethermind.Crypto.SecP256k1" Version="1.1.1" />
<PackageVersion Include="Nethermind.DotNetty.Buffers" Version="1.0.1" />
Expand All @@ -56,38 +50,29 @@
<PackageVersion Include="Nethermind.Numerics.Int256" Version="1.1.1" />
<PackageVersion Include="Nito.Collections.Deque" Version="1.2.1" />
<PackageVersion Include="NLog" Version="5.2.8" />
<PackageVersion Include="NLog.Targets.Seq" Version="3.0.0" />
<PackageVersion Include="NLog.Targets.Seq" Version="3.1.0" />
<PackageVersion Include="NSubstitute" Version="5.1.0" />
<PackageVersion Include="NUnit" Version="3.14.0" />
<PackageVersion Include="NUnit.Analyzers" Version="3.9.0" />
<PackageVersion Include="NUnit3TestAdapter" Version="4.5.0" />
<PackageVersion Include="Open.NAT.Core" Version="2.1.0.5" />
<PackageVersion Include="Polly" Version="8.2.0" />
<PackageVersion Include="Polly" Version="8.2.1" />
<PackageVersion Include="Portable.BouncyCastle" Version="1.9.0" />
<PackageVersion Include="prometheus-net" Version="8.2.0" />
<PackageVersion Include="prometheus-net.AspNetCore" Version="8.2.0" />
<PackageVersion Include="prometheus-net.AspNetCore" Version="8.2.1" />
<PackageVersion Include="Pyroscope" Version="0.8.14" />
<PackageVersion Include="ReadLine" Version="2.0.1" />
<PackageVersion Include="RichardSzalay.MockHttp" Version="7.0.0" />
<PackageVersion Include="RocksDB" Version="8.3.2.39829" />
<PackageVersion Include="SCrypt" Version="2.0.0.2" />
<PackageVersion Include="Seq.Api" Version="2023.4.0" />
<PackageVersion Include="SharpCompress" Version="0.35.0" />
<PackageVersion Include="Shouldly" Version="4.2.1" />
<PackageVersion Include="Snappy.Standard" Version="0.2.0" />
<PackageVersion Include="System.Configuration.ConfigurationManager" Version="8.0.0" />
<PackageVersion Include="System.Diagnostics.TextWriterTraceListener" Version="4.3.0" />
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="7.1.2" />
<PackageVersion Include="System.IO.Compression.ZipFile" Version="4.3.0" />
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="7.2.0" />
<PackageVersion Include="System.IO.Pipelines" Version="8.0.0" />
<PackageVersion Include="System.Linq.Async" Version="6.0.1" />
<PackageVersion Include="System.Net.Http" Version="4.3.4" />
<PackageVersion Include="System.Runtime.Caching" Version="8.0.0" />
<PackageVersion Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
<PackageVersion Include="System.Text.Encodings.Web" Version="8.0.0" />
<PackageVersion Include="TestableIO.System.IO.Abstractions.TestingHelpers" Version="20.0.4" />
<PackageVersion Include="TestableIO.System.IO.Abstractions.Wrappers" Version="20.0.4" />
<PackageVersion Include="Websocket.Client" Version="5.0.0" />
<PackageVersion Include="YamlDotNet" Version="13.7.1" />
</ItemGroup>
</Project>
2 changes: 1 addition & 1 deletion src/Nethermind/Ethereum.Abi.Test/Ethereum.Abi.Test.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
</EmbeddedResource>
</ItemGroup>
<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@

using System;
using System.Collections.Generic;
using System.Formats.Tar;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net.Http;
using Ethereum.Test.Base;
using Ethereum.Test.Base.Interfaces;
using SharpCompress.Archives;
using SharpCompress.Archives.Tar;

namespace Ethereum.Blockchain.Pyspec.Test;

Expand All @@ -35,8 +35,12 @@ private void DownloadAndExtract(string archiveVersion, string archiveName, strin
HttpResponseMessage response = httpClient.GetAsync(string.Format(Constants.ARCHIVE_URL_TEMPLATE, archiveVersion, archiveName)).GetAwaiter().GetResult();
response.EnsureSuccessStatusCode();
using Stream contentStream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult();
using TarArchive archive = TarArchive.Open(contentStream);
archive.ExtractToDirectory(testsDirectoryName);
using GZipStream gzStream = new(contentStream, CompressionMode.Decompress);

if (!Directory.Exists(testsDirectoryName))
Directory.CreateDirectory(testsDirectoryName);

TarFile.ExtractToDirectory(gzStream, testsDirectoryName, true);
}

private IEnumerable<BlockchainTest> LoadTestsFromDirectory(string testDir, string wildcard)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,10 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.DataProtection" />
<PackageReference Include="Microsoft.AspNetCore.DataProtection.Abstractions" />
<PackageReference Include="Microsoft.AspNetCore.DataProtection.Extensions" />
<PackageReference Include="Microsoft.NET.Test.Sdk" />
<PackageReference Include="NUnit" />
Expand All @@ -18,8 +16,6 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="NUnit3TestAdapter" />
<PackageReference Include="System.Diagnostics.TextWriterTraceListener" />
<PackageReference Include="System.Security.Cryptography.ProtectedData" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="..\..\tests\GeneralStateTests\st*\**\*.*">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
2 changes: 1 addition & 1 deletion src/Nethermind/Ethereum.PoW.Test/Ethereum.PoW.Test.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
2 changes: 1 addition & 1 deletion src/Nethermind/Ethereum.Rlp.Test/Ethereum.Rlp.Test.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="NUnit3TestAdapter" />
<PackageReference Include="SharpCompress" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Nethermind.Blockchain\Nethermind.Blockchain.csproj" />
Expand Down
2 changes: 1 addition & 1 deletion src/Nethermind/Ethereum.Test.Base/EthereumTestResult.cs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public EthereumTestResult(string? name, string? loadFailure)
public bool Pass { get; set; }
public string Fork { get; set; }

[JsonIgnore] public int TimeInMs { get; set; }
public double TimeInMs { get; set; }

public Hash256 StateRoot { get; set; } = Keccak.EmptyTreeHash;
}
Expand Down
45 changes: 45 additions & 0 deletions src/Nethermind/Ethereum.Test.Base/GeneralStateTestInfoJson.cs
Original file line number Diff line number Diff line change
@@ -1,12 +1,57 @@
// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
// SPDX-License-Identifier: LGPL-3.0-only

using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;

namespace Ethereum.Test.Base
{
[JsonConverter(typeof(GeneralStateTestInfoConverter))]
public class GeneralStateTestInfoJson
{
public Dictionary<string, string>? Labels { get; set; }
}

public class GeneralStateTestInfoConverter : JsonConverter<GeneralStateTestInfoJson>
{
public override GeneralStateTestInfoJson? Read(
ref Utf8JsonReader reader,
Type typeToConvert,
JsonSerializerOptions options)
{
Dictionary<string, string>? labels = null;
if (reader.TokenType == JsonTokenType.StartObject)
{
var depth = reader.CurrentDepth;
while (reader.Read())
{
if (reader.TokenType == JsonTokenType.EndObject && reader.CurrentDepth == depth)
{
break;
}
if (reader.TokenType == JsonTokenType.PropertyName && reader.ValueTextEquals("labels"u8))
{
reader.Read();
labels = JsonSerializer.Deserialize<Dictionary<string, string>>(ref reader, options);
}
else
{
reader.Skip();
}
}
}

return new GeneralStateTestInfoJson { Labels = labels };
}

public override void Write(
Utf8JsonWriter writer,
GeneralStateTestInfoJson info,
JsonSerializerOptions options)
{
JsonSerializer.Serialize(writer, info, options);
}
}
}
2 changes: 1 addition & 1 deletion src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer)

List<string> differences = RunAssertions(test, stateProvider);
EthereumTestResult testResult = new(test.Name, test.ForkName, differences.Count == 0);
testResult.TimeInMs = (int)stopwatch.Elapsed.TotalMilliseconds;
testResult.TimeInMs = stopwatch.Elapsed.TotalMilliseconds;
testResult.StateRoot = stateProvider.StateRoot;

// Assert.Zero(differences.Count, "differences");
Expand Down
5 changes: 0 additions & 5 deletions src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,6 @@ public static IEnumerable<GeneralStateTest> Convert(string name, GeneralStateTes
foreach (KeyValuePair<string, PostStateJson[]> postStateBySpec in testJson.Post)
{
int iterationNumber = 0;
int testIndex = testJson.Info?.Labels?.Select(x => System.Convert.ToInt32(x.Key)).FirstOrDefault() ?? 0;
foreach (PostStateJson stateJson in postStateBySpec.Value)
{
GeneralStateTest test = new();
Expand All @@ -219,10 +218,6 @@ public static IEnumerable<GeneralStateTest> Convert(string name, GeneralStateTes
{
test.Name += testJson.Info?.Labels?[iterationNumber.ToString()]?.Replace(":label ", string.Empty);
}
else
{
test.Name += string.Empty;
}

test.ForkName = postStateBySpec.Key;
test.Fork = ParseSpec(postStateBySpec.Key);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
2 changes: 1 addition & 1 deletion src/Nethermind/Ethereum.VM.Test/Ethereum.VM.Test.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.msbuild">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
<ProjectReference Include="..\Nethermind.Consensus.Test\Nethermind.Consensus.Test.csproj" />
<ProjectReference Include="..\Nethermind.Evm.Test\Nethermind.Evm.Test.csproj" />
<ProjectReference Include="..\Nethermind.JsonRpc.Test\Nethermind.JsonRpc.Test.csproj" />
<ProjectReference Include="..\Nethermind.Merge.Plugin.Test\Nethermind.Merge.Plugin.Test.csproj" />
<ProjectReference Include="..\Nethermind.Merge.Plugin\Nethermind.Merge.Plugin.csproj" />
<ProjectReference Include="..\Nethermind.Monitoring.Test\Nethermind.Monitoring.Test.csproj" />
<ProjectReference Include="..\Nethermind.Network.Test\Nethermind.Network.Test.csproj" />
</ItemGroup>

Expand Down
Loading

0 comments on commit 3eba584

Please sign in to comment.