diff --git a/src/NexusMods.MnemonicDB.Abstractions/Datom.cs b/src/NexusMods.MnemonicDB.Abstractions/Datom.cs index 1aab945..a27f11a 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/Datom.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/Datom.cs @@ -9,7 +9,7 @@ namespace NexusMods.MnemonicDB.Abstractions.DatomIterators; /// Represents a raw (unparsed) datom from an index. Most of the time this datom is only valid for the /// lifetime of the current iteration. It is not safe to store this datom for later use. /// -public readonly struct Datom +public readonly struct Datom : IEquatable { private readonly KeyPrefix _prefix; private readonly ReadOnlyMemory _valueBlob; @@ -42,6 +42,7 @@ public byte[] ToArray() _valueBlob.Span.CopyTo(array.AsSpan(KeyPrefix.Size)); return array; } + /// /// The KeyPrefix of the datom @@ -143,4 +144,22 @@ public Datom Retract() { return new Datom(_prefix with {IsRetract = true, T = TxId.Tmp}, _valueBlob); } + + /// + public bool Equals(Datom other) + { + return _prefix.Equals(other._prefix) && _valueBlob.Span.SequenceEqual(other._valueBlob.Span); + } + + /// + public override bool Equals(object? obj) + { + return obj is Datom other && Equals(other); + } + + /// + public override int GetHashCode() + { + return HashCode.Combine(_prefix.GetHashCode()); + } } diff --git a/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs b/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs index e73345f..bf6990c 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs @@ -55,6 +55,11 @@ public interface IConnection /// A service provider that entities can use to resolve their values /// public IServiceProvider ServiceProvider { get; } + + /// + /// Gets the datom store that this connection uses to store data. + /// + public IDatomStore DatomStore { get; } /// /// Returns a snapshot of the database as of the given transaction id. diff --git a/src/NexusMods.MnemonicDB.Abstractions/IDatomStore.cs b/src/NexusMods.MnemonicDB.Abstractions/IDatomStore.cs index 4caeff1..ac62073 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IDatomStore.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IDatomStore.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Threading.Tasks; using NexusMods.MnemonicDB.Abstractions.IndexSegments; using NexusMods.MnemonicDB.Abstractions.TxFunctions; @@ -27,6 +28,17 @@ public interface IDatomStore : IDisposable /// AttributeCache AttributeCache { get; } + /// + /// Exports the database (including all indexes) to the given stream + /// + public Task ExportAsync(Stream stream); + + /// + /// Imports the database (including all indexes) from the given stream. + /// Any existing data will be deleted before importing. + /// + public Task ImportAsync(Stream stream); + /// /// Transacts (adds) the given datoms into the store. /// diff --git a/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegment.cs b/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegment.cs index ccef8c5..7ad99d2 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegment.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegment.cs @@ -47,7 +47,22 @@ public IndexSegment(ReadOnlySpan data, ReadOnlySpan offsets, Attribut ReprocessData(_rowCount, data, offsets, memory.Span); } + + /// + /// Create an index segment from raw data + /// + public IndexSegment(int rowCount, ReadOnlyMemory data, AttributeCache attributeCache) + { + _attributeCache = attributeCache; + _data = data; + _rowCount = rowCount; + } + /// + /// Gets read-only access to the data in this segment + /// + public ReadOnlyMemory Data => _data; + /// /// All the upper values /// diff --git a/src/NexusMods.MnemonicDB.Abstractions/Query/SliceDescriptor.cs b/src/NexusMods.MnemonicDB.Abstractions/Query/SliceDescriptor.cs index 6f979d2..038852b 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/Query/SliceDescriptor.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/Query/SliceDescriptor.cs @@ -35,11 +35,25 @@ public readonly struct SliceDescriptor /// public bool IsReverse => From.Compare(To, Index) > 0; + /// + /// Returns this descriptor with a reversed iteration order. + /// + public SliceDescriptor Reversed() + { + return new SliceDescriptor + { + Index = Index, + From = To, + To = From + }; + } + /// /// Returns true if the datom is within the slice, false otherwise. /// public bool Includes(in Datom datom) { + return Index switch { IndexType.TxLog => DatomComparators.TxLogComparator.Compare(From, datom) <= 0 && @@ -50,6 +64,9 @@ public bool Includes(in Datom datom) IndexType.AEVTCurrent or IndexType.AEVTHistory => DatomComparators.AEVTComparator.Compare(From, datom) <= 0 && DatomComparators.AEVTComparator.Compare(datom, To) < 0, + IndexType.AVETCurrent or IndexType.AVETHistory => + DatomComparators.AVETComparator.Compare(From, datom) <= 0 && + DatomComparators.AVETComparator.Compare(datom, To) < 0, IndexType.VAETCurrent or IndexType.VAETHistory => DatomComparators.VAETComparator.Compare(From, datom) <= 0 && DatomComparators.VAETComparator.Compare(datom, To) < 0, diff --git a/src/NexusMods.MnemonicDB/Connection.cs b/src/NexusMods.MnemonicDB/Connection.cs index 0c7b65f..cb39928 100644 --- a/src/NexusMods.MnemonicDB/Connection.cs +++ b/src/NexusMods.MnemonicDB/Connection.cs @@ -74,6 +74,9 @@ private IObservable ProcessUpdates(IObservable dbStream) /// public IServiceProvider ServiceProvider { get; set; } + /// + public IDatomStore DatomStore => _store; + /// public IDb Db { diff --git a/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj b/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj index 40e2afb..54e06b4 100644 --- a/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj +++ b/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj @@ -18,6 +18,9 @@ + + DatomStore.cs + diff --git a/src/NexusMods.MnemonicDB/Storage/DatomStore.cs b/src/NexusMods.MnemonicDB/Storage/DatomStore.cs index b51bbed..f4ca6ce 100644 --- a/src/NexusMods.MnemonicDB/Storage/DatomStore.cs +++ b/src/NexusMods.MnemonicDB/Storage/DatomStore.cs @@ -3,6 +3,7 @@ using System.Collections.Frozen; using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Threading; @@ -22,7 +23,7 @@ namespace NexusMods.MnemonicDB.Storage; -public class DatomStore : IDatomStore +public partial class DatomStore : IDatomStore { private readonly IIndex _aevtCurrent; private readonly IIndex _aevtHistory; @@ -78,7 +79,7 @@ public class DatomStore : IDatomStore /// /// DI constructor /// - public DatomStore(ILogger logger, DatomStoreSettings settings, IStoreBackend backend) + public DatomStore(ILogger logger, DatomStoreSettings settings, IStoreBackend backend, bool bootstrap = true) { _remapFunc = Remap; _dbStream = new DbStream(); @@ -116,7 +117,8 @@ public DatomStore(ILogger logger, DatomStoreSettings settings, IStor _avetCurrent = _backend.GetIndex(IndexType.AVETCurrent); _avetHistory = _backend.GetIndex(IndexType.AVETHistory); - Bootstrap(); + if (bootstrap) + Bootstrap(); } /// @@ -620,4 +622,5 @@ private unsafe PrevState GetPreviousState(bool isRemapped, AttributeId attrId, I #endregion + } diff --git a/src/NexusMods.MnemonicDB/Storage/ImportExport.cs b/src/NexusMods.MnemonicDB/Storage/ImportExport.cs new file mode 100644 index 0000000..e0c0d5d --- /dev/null +++ b/src/NexusMods.MnemonicDB/Storage/ImportExport.cs @@ -0,0 +1,112 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.IndexSegments; +using NexusMods.MnemonicDB.Abstractions.Query; + +namespace NexusMods.MnemonicDB.Storage; + +public partial class DatomStore +{ + // File format: + // FOURCC: "MDBX" + // ushort: version + // one or more chunks + // + // chunk: + // byte: IndexType + // uint: datomCount (number of datoms in the chunk) + // uint: chunkSize (in bytes) + // datomBlob + + private static readonly byte[] FourCC = "MDBX"u8.ToArray(); + private const int ChunkSize = 1024 * 16; + + /// + /// Exports the database to the given stream + /// + public async Task ExportAsync(Stream stream) + { + var exportedDatoms = 0; + var binaryWriter = new BinaryWriter(stream); + binaryWriter.Write(FourCC); + binaryWriter.Write((ushort)1); + + var snapshot = _backend.GetSnapshot(); + + foreach (var indexType in Enum.GetValues()) + { + var slice = SliceDescriptor.Create(indexType); + var chunks = snapshot.DatomsChunked(slice, ChunkSize); + + foreach (var chunk in chunks) + { + var data = chunk.Data; + binaryWriter.Write((byte)indexType); + binaryWriter.Write((uint)chunk.Count); + binaryWriter.Write((uint)data.Length); + binaryWriter.Write(data.Span); + exportedDatoms += chunk.Count; + } + } + _logger.LogInformation("Exported {0} datoms", exportedDatoms); + } + + public async Task ImportAsync(Stream stream) + { + CleanStore(); + var importedCount = 0; + var binaryReader = new BinaryReader(stream); + var fourCC = binaryReader.ReadBytes(4); + if (!fourCC.SequenceEqual(FourCC)) + throw new InvalidDataException("Invalid file format"); + + var version = binaryReader.ReadUInt16(); + if (version != 1) + throw new InvalidDataException("Invalid file version"); + + while (stream.Position < stream.Length) + { + var indexType = (IndexType)binaryReader.ReadByte(); + var datomCount = binaryReader.ReadUInt32(); + var chunkSize = binaryReader.ReadUInt32(); + var data = binaryReader.ReadBytes((int)chunkSize); + var segment = new IndexSegment((int)datomCount, data.AsMemory(), _backend.AttributeCache); + + using var batch = _backend.CreateBatch(); + var index = _backend.GetIndex(indexType); + + foreach (var datom in segment) + index.Put(batch, datom); + + batch.Commit(); + importedCount += (int)datomCount; + } + + _logger.LogInformation("Imported {0} datoms", importedCount); + _nextIdCache.ResetCaches(); + Bootstrap(); + } + + private void CleanStore() + { + int datomCount = 0; + var snapshot = _backend.GetSnapshot(); + using var batch = _backend.CreateBatch(); + foreach (var index in Enum.GetValues()) + { + var slice = SliceDescriptor.Create(index); + var datoms = snapshot.Datoms(slice); + foreach (var datom in datoms) + { + _backend.GetIndex(index).Delete(batch, datom); + datomCount++; + } + } + batch.Commit(); + _logger.LogInformation("Cleaned {0} datoms", datomCount); + } +} diff --git a/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Snapshot.cs b/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Snapshot.cs index 0ab686b..ef347c1 100644 --- a/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Snapshot.cs +++ b/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Snapshot.cs @@ -2,6 +2,7 @@ using System.Collections.Immutable; using System.Linq; using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; using NexusMods.MnemonicDB.Abstractions.IndexSegments; using NexusMods.MnemonicDB.Abstractions.Query; @@ -23,111 +24,98 @@ public void Dispose() { } /// public IndexSegment Datoms(SliceDescriptor descriptor) { - var thisIndex = _indexes[(int)descriptor.Index]; - if (thisIndex.Count == 0) - return new IndexSegment(); - - var idxLower = thisIndex.IndexOf(descriptor.From.ToArray()); - var idxUpper = thisIndex.IndexOf(descriptor.To.ToArray()); - bool upperExact = true; - bool lowerExact = true; - - if (idxLower < 0) - { - idxLower = ~idxLower; - lowerExact = false; - } + var index = _indexes[(int)descriptor.Index]; + var isReverse = descriptor.IsReverse; + int increment = 1; + int startIndex; - if (idxUpper < 0) + if (!isReverse) { - idxUpper = ~idxUpper; - upperExact = false; + var indexOf = index.IndexOf(descriptor.From.ToArray()); + if (indexOf >= 0) + startIndex = indexOf; + else + startIndex = ~indexOf; } - - var lower = idxLower; - var upper = idxUpper; - - if (descriptor.IsReverse) + else { - lower = idxUpper; - upper = idxLower; - (lowerExact, upperExact) = (upperExact, lowerExact); + increment = -1; + var indexOf = index.IndexOf(descriptor.From.ToArray()); + if (indexOf >= 0) + startIndex = indexOf; + else + startIndex = (~indexOf) - 1; } - + using var segmentBuilder = new IndexSegmentBuilder(_attributeCache); - if (descriptor.IsReverse) - { - if (!lowerExact) - lower++; - for (var i = upper; i >= lower; i--) - { - segmentBuilder.Add(thisIndex.ElementAt(i)); - } - } - else + while (true) { - if (!upperExact) - upper--; - for (var i = lower; i <= upper; i++) - { - segmentBuilder.Add(thisIndex.ElementAt(i)); - } - } - + if (startIndex < 0 || startIndex >= index.Count) + break; + + var current = index.ElementAt(startIndex); + var datom = new Datom(current); + if (!descriptor.Includes(in datom)) + break; + + segmentBuilder.Add(current); + startIndex += increment; + + } return segmentBuilder.Build(); } /// public IEnumerable DatomsChunked(SliceDescriptor descriptor, int chunkSize) { - var idxLower = _indexes[(int)descriptor.Index].IndexOf(descriptor.From.ToArray()); - var idxUpper = _indexes[(int)descriptor.Index].IndexOf(descriptor.To.ToArray()); - - if (idxLower < 0) - idxLower = ~idxLower; - - if (idxUpper < 0) - idxUpper = ~idxUpper; - - var lower = idxLower; - var upper = idxUpper; - var reverse = false; - - if (idxLower > idxUpper) - { - lower = idxUpper; - upper = idxLower; - reverse = true; - } - - using var segmentBuilder = new IndexSegmentBuilder(_attributeCache); var index = _indexes[(int)descriptor.Index]; + var isReverse = descriptor.IsReverse; + var includesDescriptor = descriptor; + int increment = 1; + int startIndex; - if (!reverse) + if (!isReverse) { - for (var i = lower; i < upper; i++) - { - segmentBuilder.Add(index.ElementAt(i)); - if (segmentBuilder.Count == chunkSize) - { - yield return segmentBuilder.Build(); - segmentBuilder.Reset(); - } - } + var indexOf = index.IndexOf(descriptor.From.ToArray()); + if (indexOf >= 0) + startIndex = indexOf; + else + startIndex = ~indexOf; } else { - for (var i = upper; i > lower; i--) + includesDescriptor = descriptor.Reversed(); + increment = -1; + var indexOf = index.IndexOf(descriptor.From.ToArray()); + if (indexOf >= 0) + startIndex = indexOf; + else + startIndex = (~indexOf) - 1; + } + + using var segmentBuilder = new IndexSegmentBuilder(_attributeCache); + + while (true) + { + if (startIndex < 0 || startIndex >= index.Count) + break; + + var current = index.ElementAt(startIndex); + var datom = new Datom(current); + if (!includesDescriptor.Includes(in datom)) + break; + + segmentBuilder.Add(current); + startIndex += increment; + + if (segmentBuilder.Count == chunkSize) { - segmentBuilder.Add(index.ElementAt(i)); - if (segmentBuilder.Count == chunkSize) - { - yield return segmentBuilder.Build(); - segmentBuilder.Reset(); - } + yield return segmentBuilder.Build(); + segmentBuilder.Reset(); } } - yield return segmentBuilder.Build(); + if (segmentBuilder.Count > 0) + yield return segmentBuilder.Build(); } } diff --git a/src/NexusMods.MnemonicDB/Storage/NextIdCache.cs b/src/NexusMods.MnemonicDB/Storage/NextIdCache.cs index fb21f4e..ce0628c 100644 --- a/src/NexusMods.MnemonicDB/Storage/NextIdCache.cs +++ b/src/NexusMods.MnemonicDB/Storage/NextIdCache.cs @@ -35,6 +35,17 @@ public EntityId NextId(ISnapshot snapshot, PartitionId partitionId) return partitionId.MakeEntityId(newId); } + /// + /// Resets all the caches + /// + public void ResetCaches() + { + for (var i = 0; i < 256; i++) + { + this[i] = 0; + } + } + /// /// Gets the last recorded entity in the partition in the snapshot /// @@ -46,7 +57,7 @@ public EntityId LastEntityInPartition(ISnapshot snapshot, PartitionId partitionI return partitionId.MakeEntityId(this[partition]); } - var descriptor = SliceDescriptor.Create(partitionId.MakeEntityId(ulong.MaxValue), partitionId.MakeEntityId(0)); + var descriptor = SliceDescriptor.Create(partitionId.MakeEntityId(0), partitionId.MakeEntityId(ulong.MaxValue)).Reversed(); var lastEnt = snapshot.DatomsChunked(descriptor, 1) .SelectMany(c => c) diff --git a/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Snapshot.cs b/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Snapshot.cs index 88f50ef..2b8d629 100644 --- a/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Snapshot.cs +++ b/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Snapshot.cs @@ -105,6 +105,7 @@ public IEnumerable DatomsChunked(SliceDescriptor descriptor, int c else iterator.Next(); } - yield return builder.Build(); + if (builder.Count > 0) + yield return builder.Build(); } } diff --git a/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs b/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs index 6b6abc7..3128b15 100644 --- a/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs +++ b/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs @@ -11,6 +11,8 @@ public class NullConnection : IConnection public TxId TxId => throw new NotSupportedException(); public IObservable Revisions => throw new NotSupportedException(); public IServiceProvider ServiceProvider => throw new NotSupportedException(); + public IDatomStore DatomStore => throw new NotSupportedException(); + public IDb AsOf(TxId txId) { throw new NotSupportedException(); diff --git a/tests/NexusMods.MnemonicDB.Tests/ImportExportTests.cs b/tests/NexusMods.MnemonicDB.Tests/ImportExportTests.cs new file mode 100644 index 0000000..c18dfcd --- /dev/null +++ b/tests/NexusMods.MnemonicDB.Tests/ImportExportTests.cs @@ -0,0 +1,80 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NexusMods.Hashing.xxHash64; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.Query; +using NexusMods.MnemonicDB.Storage; +using NexusMods.MnemonicDB.Storage.InMemoryBackend; +using NexusMods.MnemonicDB.TestModel; +using NexusMods.Paths; + +namespace NexusMods.MnemonicDB.Tests; + +public class ImportExportTests(IServiceProvider provider) : AMnemonicDBTest(provider) +{ + [Fact] + public async Task CanExportAndImportData() + { + await InsertData(); + + var ms = new MemoryStream(); + await Connection.DatomStore.ExportAsync(ms); + + Logger.LogInformation("Exported {0} bytes", ms.Length); + + var datomStore = new DatomStore(provider.GetRequiredService>()!, + Config, new Backend(), bootstrap: false); + + ms.Position = 0; + await datomStore.ImportAsync(ms); + + foreach (var index in Enum.GetValues()) + { + var slice = SliceDescriptor.Create(index); + var setA = Connection.DatomStore.GetSnapshot().Datoms(slice); + var setB = datomStore.GetSnapshot().Datoms(slice); + + var setDiff = setB.Except(setA).ToArray(); + + setB.Count.Should().Be(setA.Count); + foreach (var (a, b) in setA.Zip(setB)) + { + a.Should().BeEquivalentTo(b); + } + } + + } + + private async Task InsertData() + { + using var tx = Connection.BeginTransaction(); + + var loadout = new Loadout.New(tx) + { + Name = "Test Loadout", + }; + + foreach (var modIdx in Enumerable.Range(0, 10)) + { + var mod = new Mod.New(tx) + { + Name = $"Mod{modIdx}", + Source = new Uri($"http://somesite.com/Mod{modIdx}"), + LoadoutId = loadout, + }; + + foreach (var fileIdx in Enumerable.Range(0, 10)) + { + _ = new TestModel.File.New(tx) + { + Path = $"File{fileIdx}", + ModId = mod, + Size = Size.From((ulong)fileIdx), + Hash = Hash.From((ulong)(0xDEADBEEF + fileIdx)), + }; + } + } + + var txResult = await tx.Commit(); + } +}