use Xxhash3 for StorageKey.GetHashCode (#3559)

* fea: StorageKey.GetHashCode uses XxHash3 for better performance * fea: StorageKey.GetHashCode uses XxHash3 for better performance * fea: StorageKey.GetHashCode uses XxHash3 for better performance * use HashCode.Combine * Unify seed * fix ut * remove unused --------- Co-authored-by: Shargon <[email protected]>
neo-project · Nov 6, 2024 · 25554df · 25554df
1 parent a021ebe
commit 25554df
Show file tree

Hide file tree

Showing 10 changed files with 120 additions and 11 deletions.
diff --git a/.gitignore b/.gitignore
@@ -257,3 +257,6 @@ paket-files/
 PublishProfiles
 /.vscode
 launchSettings.json
+
+# Benchmarks
+**/BenchmarkDotNet.Artifacts/
diff --git a/benchmarks/Neo.Benchmarks/Benchmarks.Hash.cs b/benchmarks/Neo.Benchmarks/Benchmarks.Hash.cs
@@ -0,0 +1,56 @@
+// Copyright (C) 2015-2024 The Neo Project.
+//
+// Benchmarks.Hash.cs file belongs to the neo project and is free
+// software distributed under the MIT software license, see the
+// accompanying file LICENSE in the main directory of the
+// repository or http://www.opensource.org/licenses/mit-license.php
+// for more details.
+//
+// Redistribution and use in source and binary forms with or without
+// modifications are permitted.
+
+using BenchmarkDotNet.Attributes;
+using Neo.Cryptography;
+using Neo.Extensions;
+using System.Diagnostics;
+using System.IO.Hashing;
+using System.Text;
+
+namespace Neo.Benchmark;
+
+public class Benchmarks_Hash
+{
+    // 256 KiB
+    static readonly byte[] data = Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat("Hello, World!^_^", 16 * 1024)));
+
+    static readonly byte[] hash = "9182abedfbb9b18d81a05d8bcb45489e7daa2858".HexToBytes();
+
+    [Benchmark]
+    public void RIPEMD160_ComputeHash()
+    {
+        using var ripemd160 = new RIPEMD160Managed();
+        var result = ripemd160.ComputeHash(data);
+        Debug.Assert(result.SequenceEqual(hash));
+    }
+
+    [Benchmark]
+    public void XxHash32_HashToUInt32()
+    {
+        var result = XxHash32.HashToUInt32(data);
+        Debug.Assert(result == 682967318u);
+    }
+
+    [Benchmark]
+    public void XxHash3_HashToUInt64()
+    {
+        var result = (uint)XxHash3.HashToUInt64(data);
+        Debug.Assert(result == 1389469485u);
+    }
+
+    [Benchmark]
+    public void Murmur32_HashToUInt32()
+    {
+        var result = data.Murmur32(0);
+        Debug.Assert(result == 3731881930u);
+    }
+}
diff --git a/benchmarks/Neo.Benchmarks/Neo.Benchmarks.csproj b/benchmarks/Neo.Benchmarks/Neo.Benchmarks.csproj
@@ -9,6 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="System.IO.Hashing" Version="8.0.0" />
     <ProjectReference Include="..\..\src\Neo\Neo.csproj" />
     <PackageReference Include="BenchmarkDotNet" Version="0.13.12" />
   </ItemGroup>

diff --git a/benchmarks/Neo.Benchmarks/Program.cs b/benchmarks/Neo.Benchmarks/Program.cs
@@ -14,3 +14,4 @@
 
 // BenchmarkRunner.Run<Benchmarks_PoCs>();
 BenchmarkRunner.Run<Benchmarks_UInt160>();
+BenchmarkRunner.Run<Benchmarks_Hash>();
diff --git a/src/Neo.VM/Unsafe.cs b/src/Neo.VM/Unsafe.cs
@@ -17,7 +17,7 @@ namespace Neo.VM
 {
     unsafe internal static class Unsafe
     {
-        const long HashMagicNumber = 40343;
+        private const long DefaultXxHash3Seed = 40343;
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static bool NotZero(ReadOnlySpan<byte> x)
@@ -46,11 +46,12 @@ public static bool NotZero(ReadOnlySpan<byte> x)
         /// Get 64-bit hash code for a byte array
         /// </summary>
         /// <param name="span">Span</param>
-        /// <returns></returns>
+        /// <param name="seed">The seed used by the xxhash3 algorithm.</param>
+        /// <returns>The computed hash code.</returns>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static ulong HashBytes(ReadOnlySpan<byte> span)
+        public static ulong HashBytes(ReadOnlySpan<byte> span, long seed = DefaultXxHash3Seed)
         {
-            return XxHash3.HashToUInt64(span, HashMagicNumber);
+            return XxHash3.HashToUInt64(span, seed);
         }
     }
 }
diff --git a/src/Neo/Cryptography/Helper.cs b/src/Neo/Cryptography/Helper.cs
@@ -18,6 +18,7 @@
 using Org.BouncyCastle.Crypto.Parameters;
 using System;
 using System.Buffers.Binary;
+using System.IO.Hashing;
 using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
@@ -31,7 +32,9 @@ namespace Neo.Cryptography
     /// </summary>
     public static class Helper
     {
+        private const int DefaultXxHash3Seed = 40343;
         private static readonly bool IsOSX = RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
+
         /// <summary>
         /// Computes the hash value for the specified byte array using the ripemd160 algorithm.
         /// </summary>
@@ -117,6 +120,30 @@ public static byte[] Sha256(this byte[] value)
             return sha256.ComputeHash(value);
         }
 
+        /// <summary>
+        /// Computes the 32-bit hash value for the specified byte array using the xxhash3 algorithm.
+        /// </summary>
+        /// <param name="value">The input to compute the hash code for.</param>
+        /// <param name="seed">The seed used by the xxhash3 algorithm.</param>
+        /// <returns>The computed hash code.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int XxHash3_32(this ReadOnlySpan<byte> value, long seed = DefaultXxHash3Seed)
+        {
+            return HashCode.Combine(XxHash3.HashToUInt64(value, seed));
+        }
+
+        /// <summary>
+        /// Computes the 32-bit hash value for the specified byte array using the xxhash3 algorithm.
+        /// </summary>
+        /// <param name="value">The input to compute the hash code for.</param>
+        /// <param name="seed">The seed used by the xxhash3 algorithm.</param>
+        /// <returns>The computed hash code.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int XxHash3_32(this byte[] value, long seed = DefaultXxHash3Seed)
+        {
+            return XxHash3_32(value.AsSpan(), seed);
+        }
+
         /// <summary>
         /// Computes the hash value for the specified region of the specified byte array using the sha256 algorithm.
         /// </summary>

diff --git a/src/Neo/Neo.csproj b/src/Neo/Neo.csproj
@@ -16,6 +16,7 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
+    <PackageReference Include="System.IO.Hashing" Version="8.0.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/Neo/SmartContract/StorageKey.cs b/src/Neo/SmartContract/StorageKey.cs
@@ -33,8 +33,15 @@ public sealed record StorageKey
 
         private byte[] cache = null;
 
+        // NOTE: StorageKey is readonly, so we can cache the hash code.
+        private int _hashCode = 0;
+
         public StorageKey() { }
 
+        /// <summary>
+        /// Initializes a new instance of the <see cref="StorageKey"/> class.
+        /// </summary>
+        /// <param name="cache">The cached byte array. NOTE: It must be read-only and can be modified by the caller.</param>
         internal StorageKey(byte[] cache)
         {
             this.cache = cache;
@@ -67,7 +74,9 @@ public bool Equals(StorageKey other)
 
         public override int GetHashCode()
         {
-            return Id + (int)Key.Span.Murmur32(0);
+            if (_hashCode == 0)
+                _hashCode = HashCode.Combine(Id, Key.Span.XxHash3_32());
+            return _hashCode;
         }
 
         public byte[] ToArray()

diff --git a/tests/Neo.UnitTests/Cryptography/UT_Cryptography_Helper.cs b/tests/Neo.UnitTests/Cryptography/UT_Cryptography_Helper.cs
@@ -19,6 +19,8 @@
 using Neo.Wallets;
 using Neo.Wallets.NEP6;
 using System;
+using System.Collections.Generic;
+using System.IO.Hashing;
 using System.Linq;
 using System.Text;
 
@@ -55,13 +57,19 @@ public void TestMurmurReadOnlySpan()
             input.Murmur128(0).Should().Equal(input2.Murmur128(0));
         }
 
+        [TestMethod]
+        public void TestXxHash3()
+        {
+            byte[] data = Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat("Hello, World!^_^", 16 * 1024)));
+            data.XxHash3_32().Should().Be(HashCode.Combine(XxHash3.HashToUInt64(data, 40343)));
+        }
+
         [TestMethod]
         public void TestSha256()
         {
             byte[] value = Encoding.ASCII.GetBytes("hello world");
             byte[] result = value.Sha256(0, value.Length);
-            string resultStr = result.ToHexString();
-            resultStr.Should().Be("b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9");
+            result.ToHexString().Should().Be("b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9");
             value.Sha256().Should().Equal(result);
             ((Span<byte>)value).Sha256().Should().Equal(result);
             ((ReadOnlySpan<byte>)value).Sha256().Should().Equal(result);
@@ -82,8 +90,7 @@ public void TestRIPEMD160()
         {
             ReadOnlySpan<byte> value = Encoding.ASCII.GetBytes("hello world");
             byte[] result = value.RIPEMD160();
-            string resultStr = result.ToHexString();
-            resultStr.Should().Be("98c615784ccb5fe5936fbc0cbe9dfdb408d92f0f");
+            result.ToHexString().Should().Be("98c615784ccb5fe5936fbc0cbe9dfdb408d92f0f");
         }
 
         [TestMethod]

diff --git a/tests/Neo.UnitTests/Ledger/UT_StorageKey.cs b/tests/Neo.UnitTests/Ledger/UT_StorageKey.cs
@@ -11,7 +11,9 @@
 
 using FluentAssertions;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Neo.Cryptography;
 using Neo.SmartContract;
+using System;
 
 namespace Neo.UnitTests.Ledger
 {
@@ -102,8 +104,9 @@ public void Equals_SameHash_DiffKey()
         [TestMethod]
         public void GetHashCode_Get()
         {
-            StorageKey uut = new() { Id = 0x42000000, Key = TestUtils.GetByteArray(10, 0x42) };
-            uut.GetHashCode().Should().Be(1374529787);
+            var data = TestUtils.GetByteArray(10, 0x42);
+            StorageKey uut = new() { Id = 0x42000000, Key = data };
+            uut.GetHashCode().Should().Be(HashCode.Combine(0x42000000, data.XxHash3_32()));
         }
 
         [TestMethod]
Original file line number	Diff line number	Diff line change
Expand Up		@@ -14,3 +14,4 @@

		// BenchmarkRunner.Run<Benchmarks_PoCs>();
		BenchmarkRunner.Run<Benchmarks_UInt160>();
		BenchmarkRunner.Run<Benchmarks_Hash>();