Handle PR feedback and switch to xxHash3

Nexus-Mods · Oct 22, 2024 · 065ee2c · 065ee2c
1 parent fed4b46
commit 065ee2c
Show file tree

Hide file tree

Showing 18 changed files with 52 additions and 104 deletions.
diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadTests.cs b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadTests.cs
@@ -4,7 +4,7 @@
 using System.Threading.Tasks;
 using BenchmarkDotNet.Attributes;
 using NexusMods.MnemonicDB.Abstractions;
-using NexusMods.Hashing.xxHash64;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.MnemonicDB.TestModel;
 using NexusMods.Paths;
 

diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadThenWriteBenchmarks.cs b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadThenWriteBenchmarks.cs
@@ -1,7 +1,7 @@
 using System.Linq;
 using System.Threading.Tasks;
 using BenchmarkDotNet.Attributes;
-using NexusMods.Hashing.xxHash64;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.MnemonicDB.TestModel;
 using NexusMods.Paths;

diff --git a/benchmarks/OneBillionDatomsTest/Program.cs b/benchmarks/OneBillionDatomsTest/Program.cs
@@ -1,14 +1,12 @@
 using System;
 using System.Diagnostics;
-using System.Threading;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using NexusMods.MnemonicDB;
 using NexusMods.MnemonicDB.Abstractions;
-using NexusMods.MnemonicDB.Storage;
 using NexusMods.MnemonicDB.TestModel;
-using NexusMods.Hashing.xxHash64;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.Paths;
 
 var host = Host.CreateDefaultBuilder()

diff --git a/src/NexusMods.MnemonicDB.Abstractions/NexusMods.MnemonicDB.Abstractions.csproj b/src/NexusMods.MnemonicDB.Abstractions/NexusMods.MnemonicDB.Abstractions.csproj
@@ -7,7 +7,6 @@
     <ItemGroup>
         <PackageReference Include="DynamicData" Version="8.4.1"/>
         <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1"/>
-        <PackageReference Include="NexusMods.Hashing.xxHash64" Version="2.0.2" />
         <PackageReference Include="NexusMods.Paths" Version="0.9.5" />
         <PackageReference Include="R3" Version="1.2.8" />
         <PackageReference Include="System.IO.Hashing" Version="8.0.0" />

diff --git a/src/NexusMods.MnemonicDB.Abstractions/Serializer.cs b/src/NexusMods.MnemonicDB.Abstractions/Serializer.cs
@@ -1,15 +1,9 @@
 using System;
 using System.Buffers;
-using System.Collections.Generic;
-using System.IO.Hashing;
 using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using System.Text;
-using NexusMods.Hashing.xxHash64;
 using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 using NexusMods.MnemonicDB.Abstractions.Internals;
 using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
-using Reloaded.Memory.Extensions;
 
 namespace NexusMods.MnemonicDB.Abstractions;
 

diff --git a/src/NexusMods.MnemonicDB.Abstractions/ValueSerializers/IUnmanagedValueSerializer.cs b/src/NexusMods.MnemonicDB.Abstractions/ValueSerializers/IUnmanagedValueSerializer.cs
diff --git a/src/NexusMods.MnemonicDB.Abstractions/ValueSerializers/TupleSerializers.cs b/src/NexusMods.MnemonicDB.Abstractions/ValueSerializers/TupleSerializers.cs
@@ -16,7 +16,12 @@ public sealed class Tuple2_UShort_Utf8I_Serializer : IValueSerializer<(ushort, s
 
     public static int Compare(ReadOnlySpan<byte> a, ReadOnlySpan<byte> b)
     {
-        throw new NotImplementedException();
+        var aUShort = MemoryMarshal.Read<ushort>(a);
+        var bUShort = MemoryMarshal.Read<ushort>(b);
+        var ushortComparison = aUShort.CompareTo(bUShort);
+        if (ushortComparison != 0) return ushortComparison;
+
+        return Utf8InsensitiveSerializer.Compare(a.SliceFast(sizeof(ushort)), b.SliceFast(sizeof(ushort)));
     }
 
     /// <inheritdoc />
@@ -27,9 +32,7 @@ public static unsafe int Compare(byte* aVal, int aLen, byte* bVal, int bLen)
         var ushortComparison = aUShort.CompareTo(bUShort);
         if (ushortComparison != 0) return ushortComparison;
 
-        var aStr = Encoding.UTF8.GetString(aVal + sizeof(ushort), aLen - sizeof(ushort));
-        var bStr = Encoding.UTF8.GetString(bVal + sizeof(ushort), bLen - sizeof(ushort));
-        return string.Compare(aStr, bStr, StringComparison.Ordinal);    
+        return Utf8InsensitiveSerializer.Compare(aVal + sizeof(ushort), aLen - sizeof(ushort), bVal + sizeof(ushort), bLen - sizeof(ushort));  
     }
 
     /// <inheritdoc />
@@ -65,9 +68,19 @@ public sealed class Tuple3_Ref_UShort_Utf8I_Serializer : IValueSerializer<(Entit
     public static ValueTag ValueTag => ValueTag.Tuple3_Ref_UShort_Utf8I;
 
     /// <inheritdoc />
-    public static int Compare(ReadOnlySpan<byte> a, ReadOnlySpan<byte> b)
+    public static unsafe int Compare(ReadOnlySpan<byte> a, ReadOnlySpan<byte> b)
     {
-        throw new NotImplementedException();
+        var aEntityId = MemoryMarshal.Read<EntityId>(a);
+        var bEntityId = MemoryMarshal.Read<EntityId>(b);
+        var entityIdComparison = aEntityId.CompareTo(bEntityId);
+        if (entityIdComparison != 0) return entityIdComparison;
+
+        var aUShort = MemoryMarshal.Read<ushort>(a.SliceFast(sizeof(EntityId)));
+        var bUShort = MemoryMarshal.Read<ushort>(b.SliceFast(sizeof(EntityId)));
+        var ushortComparison = aUShort.CompareTo(bUShort);
+        if (ushortComparison != 0) return ushortComparison;
+
+        return Utf8InsensitiveSerializer.Compare(a.SliceFast(sizeof(EntityId) + sizeof(ushort)), b.SliceFast(sizeof(EntityId) + sizeof(ushort)));
     }
 
     /// <inheritdoc />
@@ -83,9 +96,8 @@ public static unsafe int Compare(byte* aVal, int aLen, byte* bVal, int bLen)
         var ushortComparison = aUShort.CompareTo(bUShort);
         if (ushortComparison != 0) return ushortComparison;
 
-        var aStr = Encoding.UTF8.GetString(aVal + sizeof(EntityId) + sizeof(ushort), aLen - sizeof(EntityId) - sizeof(ushort));
-        var bStr = Encoding.UTF8.GetString(bVal + sizeof(EntityId) + sizeof(ushort), bLen - sizeof(EntityId) - sizeof(ushort));
-        return string.Compare(aStr, bStr, StringComparison.Ordinal);
+        var offset = sizeof(EntityId) + sizeof(ushort);
+        return Utf8InsensitiveSerializer.Compare(aVal + offset, aLen - offset, bVal + offset, bLen - offset);
     }
 
     /// <inheritdoc />

diff --git a/src/NexusMods.MnemonicDB.Abstractions/ValueSerializers/ValueSerializers.cs b/src/NexusMods.MnemonicDB.Abstractions/ValueSerializers/ValueSerializers.cs
@@ -1,9 +1,10 @@
-    using System;
-    using System.Buffers;
-    using System.Runtime.CompilerServices;
-    using System.Runtime.InteropServices;
-    using NexusMods.MnemonicDB.Abstractions;
-    using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+// AUTOGENERATED CODE - DO NOT MODIFY
+using System;
+using System.Buffers;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 namespace NexusMods.MnemonicDB.Abstractions.ValueSerializers;
 
 

diff --git a/src/NexusMods.MnemonicDB.Abstractions/ValueSerializers/ValueSerializers.tt b/src/NexusMods.MnemonicDB.Abstractions/ValueSerializers/ValueSerializers.tt
@@ -1,12 +1,13 @@
 <#@ template language="C#" #>
 <#@ import namespace="System.Text" #>
 <#@ import namespace="System.Collections.Generic" #>
-    using System;
-    using System.Buffers;
-    using System.Runtime.CompilerServices;
-    using System.Runtime.InteropServices;
-    using NexusMods.MnemonicDB.Abstractions;
-    using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+// AUTOGENERATED CODE - DO NOT MODIFY
+using System;
+using System.Buffers;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 namespace NexusMods.MnemonicDB.Abstractions.ValueSerializers;
 <#
 

diff --git a/src/NexusMods.MnemonicDB/Storage/DatomStore.cs b/src/NexusMods.MnemonicDB/Storage/DatomStore.cs
@@ -101,16 +101,6 @@ public DatomStore(
         Logger = logger;
         _settings = settings;
 
-        Backend.DeclareEAVT(IndexType.EAVTCurrent);
-        Backend.DeclareEAVT(IndexType.EAVTHistory);
-        Backend.DeclareAEVT(IndexType.AEVTCurrent);
-        Backend.DeclareAEVT(IndexType.AEVTHistory);
-        Backend.DeclareVAET(IndexType.VAETCurrent);
-        Backend.DeclareVAET(IndexType.VAETHistory);
-        Backend.DeclareAVET(IndexType.AVETCurrent);
-        Backend.DeclareAVET(IndexType.AVETHistory);
-        Backend.DeclareTxLog(IndexType.TxLog);
-
         Backend.Init(settings.Path);
 
         if (bootstrap) Bootstrap();

diff --git a/tests/NexusMods.MnemonicDB.Storage.Tests/ABackendTest.cs b/tests/NexusMods.MnemonicDB.Storage.Tests/ABackendTest.cs
@@ -1,7 +1,7 @@
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.MnemonicDB.Storage.Abstractions;
 using NexusMods.MnemonicDB.TestModel.Helpers;
-using NexusMods.Hashing.xxHash64;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
 using NexusMods.MnemonicDB.Abstractions.DatomComparators;
 using NexusMods.MnemonicDB.Abstractions.DatomIterators;

diff --git a/tests/NexusMods.MnemonicDB.TestModel/Attributes/HashAttribute.cs b/tests/NexusMods.MnemonicDB.TestModel/Attributes/HashAttribute.cs
@@ -1,7 +1,6 @@
-using NexusMods.Hashing.xxHash64;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.MnemonicDB.Abstractions.Attributes;
-using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
 
 namespace NexusMods.MnemonicDB.TestModel.Attributes;

diff --git a/tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj b/tests/NexusMods.MnemonicDB.TestModel/NexusMods.MnemonicDB.TestModel.csproj
@@ -40,7 +40,7 @@
         <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       </PackageReference>
       <PackageReference Include="Argon" Version="0.17.0" />
-      <PackageReference Include="NexusMods.Hashing.xxHash64" Version="2.0.2" />
+      <PackageReference Include="NexusMods.Hashing.xxHash3" Version="3.0.2" />
       <PackageReference Include="Verify.Xunit" Version="25.2.0" />
     </ItemGroup>
 

diff --git a/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs b/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs
@@ -4,16 +4,12 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using NexusMods.MnemonicDB.Abstractions;
-using NexusMods.MnemonicDB.Abstractions.Models;
 using NexusMods.MnemonicDB.Storage;
 using NexusMods.MnemonicDB.Storage.RocksDbBackend;
-using NexusMods.MnemonicDB.TestModel.Helpers;
-using NexusMods.Hashing.xxHash64;
-using NexusMods.MnemonicDB.Abstractions.DatomIterators;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.MnemonicDB.TestModel;
 using NexusMods.MnemonicDB.TestModel.Analyzers;
 using NexusMods.Paths;
-using Xunit.Sdk;
 using File = NexusMods.MnemonicDB.TestModel.File;
 
 namespace NexusMods.MnemonicDB.Tests;

diff --git a/tests/NexusMods.MnemonicDB.Tests/ComplexModelTests.cs b/tests/NexusMods.MnemonicDB.Tests/ComplexModelTests.cs
@@ -1,7 +1,7 @@
 using System.Diagnostics;
+using System.Text;
 using Microsoft.Extensions.Logging;
-using NexusMods.Hashing.xxHash64;
-using NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.MnemonicDB.TestModel;
 using NexusMods.Paths;
 using File = NexusMods.MnemonicDB.TestModel.File;
@@ -52,7 +52,7 @@ public async Task CanCreateLoadout(int modCount, int filesPerMod)
                     Path = name,
                     ModId = mod,
                     Size = Size.FromLong(name.Length),
-                    Hash = Hash.FromLong(name.XxHash64AsUtf8())
+                    Hash = HashAsUtf8(name)
                 };
 
                 files.Add(file);
@@ -142,7 +142,7 @@ public async Task CanRestartStorage(int modCount, int filesPerMod, int extraFile
                     Path = name,
                     ModId = mod,
                     Size = Size.FromLong(name.Length),
-                    Hash = Hash.FromLong(name.XxHash64AsUtf8())
+                    Hash = HashAsUtf8(name)
                 };
 
                 files.Add(file);
@@ -164,7 +164,7 @@ public async Task CanRestartStorage(int modCount, int filesPerMod, int extraFile
                 Path = name,
                 ModId = firstMod,
                 Size = Size.FromLong(name.Length),
-                Hash = Hash.FromLong(name.XxHash64AsUtf8())
+                Hash = HashAsUtf8(name)
             };
 
             files.Add(file);
@@ -231,4 +231,6 @@ public async Task CanGetFromTransaction()
         var remap = result.Remap(archiveFile);
         remap.AsFile().Path.Should().Be("bar");
     }
+
+    private static Hash HashAsUtf8(string value) => Hash.FromLong(Encoding.UTF8.GetBytes(value).xxHash3());
 }
diff --git a/tests/NexusMods.MnemonicDB.Tests/DbTests.cs b/tests/NexusMods.MnemonicDB.Tests/DbTests.cs
@@ -1,11 +1,10 @@
 using System.Diagnostics;
 using System.Reactive.Linq;
-using System.Security.Cryptography.X509Certificates;
 using DynamicData;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using NexusMods.MnemonicDB.Abstractions;
-using NexusMods.Hashing.xxHash64;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.MnemonicDB.Abstractions.Attributes;
 using NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
 using NexusMods.MnemonicDB.Abstractions.DatomIterators;

diff --git a/tests/NexusMods.MnemonicDB.Tests/ImportExportTests.cs b/tests/NexusMods.MnemonicDB.Tests/ImportExportTests.cs
@@ -1,6 +1,6 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
-using NexusMods.Hashing.xxHash64;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.MnemonicDB.Abstractions.Query;
 using NexusMods.MnemonicDB.Storage;

diff --git a/tests/NexusMods.MnemonicDB.Tests/MigrationTests.cs b/tests/NexusMods.MnemonicDB.Tests/MigrationTests.cs
@@ -1,9 +1,5 @@
-using System.Net;
-using NexusMods.Hashing.xxHash64;
-using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.Hashing.xxHash3;
 using NexusMods.MnemonicDB.Abstractions.Attributes;
-using NexusMods.MnemonicDB.Abstractions.IndexSegments;
-using NexusMods.MnemonicDB.Abstractions.Query;
 using NexusMods.MnemonicDB.TestModel;
 using NexusMods.MnemonicDB.TestModel.Attributes;
 using NexusMods.Paths;