From cc2ad03eb30002c9161139a1156a52004377f2a9 Mon Sep 17 00:00:00 2001 From: Christopher Schuchardt Date: Wed, 4 Dec 2024 21:05:56 -0500 Subject: [PATCH] Improve `LevelDBStore` (#3607) * Improve `LevelDBStore` * Added `DebuggerTypeProxy` * Fixed up `null` for debuggerview * revert `null` check * Fixed test * Reverted `DebuggerTypeProxy` * Added caution comment to dataset classes --- .../LevelDBStore/IO/Data/LevelDB/DB.cs | 41 +++++++++---------- .../LevelDBStore/IO/Data/LevelDB/Helper.cs | 2 +- .../LevelDBStore/IO/Data/LevelDB/Iterator.cs | 10 ++--- .../LevelDBStore/IO/Data/LevelDB/Native.cs | 4 +- .../LevelDBStore/IO/Data/LevelDB/Options.cs | 4 +- .../LevelDBStore/IO/Data/LevelDB/Snapshot.cs | 8 ++-- .../LevelDBStore/Plugins/Storage/Snapshot.cs | 18 +++++++- .../LevelDBStore/Plugins/Storage/Store.cs | 19 ++++++++- tests/Neo.Plugins.Storage.Tests/StoreTest.cs | 15 +++++++ 9 files changed, 79 insertions(+), 42 deletions(-) diff --git a/src/Plugins/LevelDBStore/IO/Data/LevelDB/DB.cs b/src/Plugins/LevelDBStore/IO/Data/LevelDB/DB.cs index 583385d34a..d1fd548bcf 100644 --- a/src/Plugins/LevelDBStore/IO/Data/LevelDB/DB.cs +++ b/src/Plugins/LevelDBStore/IO/Data/LevelDB/DB.cs @@ -11,7 +11,7 @@ #nullable enable -using System; +using System.Collections; using System.Collections.Generic; using System.IO; @@ -20,14 +20,15 @@ namespace Neo.IO.Storage.LevelDB /// /// A DB is a persistent ordered map from keys to values. /// A DB is safe for concurrent access from multiple threads without any external synchronization. + /// Iterating over the whole dataset can be time-consuming. Depending upon how large the dataset is. /// - public class DB : LevelDBHandle + public class DB : LevelDBHandle, IEnumerable> { - private DB(IntPtr handle) : base(handle) { } + private DB(nint handle) : base(handle) { } protected override void FreeUnManagedObjects() { - if (Handle != IntPtr.Zero) + if (Handle != nint.Zero) { Native.leveldb_close(Handle); } @@ -40,7 +41,7 @@ protected override void FreeUnManagedObjects() /// public void Delete(WriteOptions options, byte[] key) { - Native.leveldb_delete(Handle, options.Handle, key, (UIntPtr)key.Length, out var error); + Native.leveldb_delete(Handle, options.Handle, key, (nuint)key.Length, out var error); NativeHelper.CheckError(error); } @@ -50,7 +51,7 @@ public void Delete(WriteOptions options, byte[] key) /// public byte[] Get(ReadOptions options, byte[] key) { - var value = Native.leveldb_get(Handle, options.Handle, key, (UIntPtr)key.Length, out var length, out var error); + var value = Native.leveldb_get(Handle, options.Handle, key, (nuint)key.Length, out var length, out var error); try { NativeHelper.CheckError(error); @@ -58,16 +59,16 @@ public byte[] Get(ReadOptions options, byte[] key) } finally { - if (value != IntPtr.Zero) Native.leveldb_free(value); + if (value != nint.Zero) Native.leveldb_free(value); } } public bool Contains(ReadOptions options, byte[] key) { - var value = Native.leveldb_get(Handle, options.Handle, key, (UIntPtr)key.Length, out _, out var error); + var value = Native.leveldb_get(Handle, options.Handle, key, (nuint)key.Length, out _, out var error); NativeHelper.CheckError(error); - if (value != IntPtr.Zero) + if (value != nint.Zero) { Native.leveldb_free(value); return true; @@ -76,12 +77,12 @@ public bool Contains(ReadOptions options, byte[] key) return false; } - public Snapshot GetSnapshot() + public Snapshot CreateSnapshot() { return new Snapshot(Handle); } - public Iterator NewIterator(ReadOptions options) + public Iterator CreateIterator(ReadOptions options) { return new Iterator(Native.leveldb_create_iterator(Handle, options.Handle)); } @@ -104,7 +105,7 @@ public static DB Open(string name, Options options) /// public void Put(WriteOptions options, byte[] key, byte[] value) { - Native.leveldb_put(Handle, options.Handle, key, (UIntPtr)key.Length, value, (UIntPtr)value.Length, out var error); + Native.leveldb_put(Handle, options.Handle, key, (nuint)key.Length, value, (nuint)value.Length, out var error); NativeHelper.CheckError(error); } @@ -126,18 +127,14 @@ public void Write(WriteOptions options, WriteBatch write_batch) NativeHelper.CheckError(error); } - public IEnumerable> GetAll(Snapshot? snapshot = null) + public IEnumerator> GetEnumerator() { - using var options = new ReadOptions(); - if (snapshot != null) options.Snapshot = snapshot; - - using var iterator = NewIterator(options); - iterator.SeekToFirst(); - while (iterator.Valid()) - { + using var iterator = CreateIterator(ReadOptions.Default); + for (iterator.SeekToFirst(); iterator.Valid(); iterator.Next()) yield return new KeyValuePair(iterator.Key(), iterator.Value()); - iterator.Next(); - } } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); } } diff --git a/src/Plugins/LevelDBStore/IO/Data/LevelDB/Helper.cs b/src/Plugins/LevelDBStore/IO/Data/LevelDB/Helper.cs index 907452353a..c224d23c6d 100644 --- a/src/Plugins/LevelDBStore/IO/Data/LevelDB/Helper.cs +++ b/src/Plugins/LevelDBStore/IO/Data/LevelDB/Helper.cs @@ -20,7 +20,7 @@ public static class Helper { public static IEnumerable<(byte[], byte[])> Seek(this DB db, ReadOptions options, byte[] prefix, SeekDirection direction) { - using Iterator it = db.NewIterator(options); + using Iterator it = db.CreateIterator(options); if (direction == SeekDirection.Forward) { for (it.Seek(prefix); it.Valid(); it.Next()) diff --git a/src/Plugins/LevelDBStore/IO/Data/LevelDB/Iterator.cs b/src/Plugins/LevelDBStore/IO/Data/LevelDB/Iterator.cs index 2fa007af7c..60c1af71db 100644 --- a/src/Plugins/LevelDBStore/IO/Data/LevelDB/Iterator.cs +++ b/src/Plugins/LevelDBStore/IO/Data/LevelDB/Iterator.cs @@ -9,8 +9,6 @@ // Redistribution and use in source and binary forms with or without // modifications are permitted. -using System; - namespace Neo.IO.Storage.LevelDB { /// @@ -18,7 +16,7 @@ namespace Neo.IO.Storage.LevelDB /// public class Iterator : LevelDBHandle { - internal Iterator(IntPtr handle) : base(handle) { } + internal Iterator(nint handle) : base(handle) { } private void CheckError() { @@ -28,7 +26,7 @@ private void CheckError() protected override void FreeUnManagedObjects() { - if (Handle != IntPtr.Zero) + if (Handle != nint.Zero) { Native.leveldb_iter_destroy(Handle); } @@ -67,9 +65,9 @@ public void Prev() /// The iterator is Valid() after this call if the source contains /// an entry that comes at or past target. /// - public void Seek(byte[] target) + public void Seek(byte[] key) { - Native.leveldb_iter_seek(Handle, target, (UIntPtr)target.Length); + Native.leveldb_iter_seek(Handle, key, (nuint)key.Length); } public void SeekToFirst() diff --git a/src/Plugins/LevelDBStore/IO/Data/LevelDB/Native.cs b/src/Plugins/LevelDBStore/IO/Data/LevelDB/Native.cs index bf91524fae..04bffbd997 100644 --- a/src/Plugins/LevelDBStore/IO/Data/LevelDB/Native.cs +++ b/src/Plugins/LevelDBStore/IO/Data/LevelDB/Native.cs @@ -17,8 +17,8 @@ namespace Neo.IO.Storage.LevelDB { public enum CompressionType : byte { - kNoCompression = 0x0, - kSnappyCompression = 0x1 + NoCompression = 0x0, + SnappyCompression = 0x1 } public static class Native diff --git a/src/Plugins/LevelDBStore/IO/Data/LevelDB/Options.cs b/src/Plugins/LevelDBStore/IO/Data/LevelDB/Options.cs index 2c91d2f9b8..b6c80422a3 100644 --- a/src/Plugins/LevelDBStore/IO/Data/LevelDB/Options.cs +++ b/src/Plugins/LevelDBStore/IO/Data/LevelDB/Options.cs @@ -126,12 +126,12 @@ public int BlockRestartInterval /// incompressible, the kSnappyCompression implementation will /// efficiently detect that and will switch to uncompressed mode. /// - public CompressionType Compression + public CompressionType CompressionLevel { set { Native.leveldb_options_set_compression(Handle, value); } } - public IntPtr FilterPolicy + public nint FilterPolicy { set { Native.leveldb_options_set_filter_policy(Handle, value); } } diff --git a/src/Plugins/LevelDBStore/IO/Data/LevelDB/Snapshot.cs b/src/Plugins/LevelDBStore/IO/Data/LevelDB/Snapshot.cs index 668f06718b..400535e971 100644 --- a/src/Plugins/LevelDBStore/IO/Data/LevelDB/Snapshot.cs +++ b/src/Plugins/LevelDBStore/IO/Data/LevelDB/Snapshot.cs @@ -9,8 +9,6 @@ // Redistribution and use in source and binary forms with or without // modifications are permitted. -using System; - namespace Neo.IO.Storage.LevelDB { /// @@ -19,16 +17,16 @@ namespace Neo.IO.Storage.LevelDB /// public class Snapshot : LevelDBHandle { - internal IntPtr _db; + internal nint _db; - internal Snapshot(IntPtr db) : base(Native.leveldb_create_snapshot(db)) + internal Snapshot(nint db) : base(Native.leveldb_create_snapshot(db)) { _db = db; } protected override void FreeUnManagedObjects() { - if (Handle != IntPtr.Zero) + if (Handle != nint.Zero) { Native.leveldb_release_snapshot(_db, Handle); } diff --git a/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs b/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs index c4a2709b6d..d9f6772a0b 100644 --- a/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs +++ b/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs @@ -11,12 +11,16 @@ using Neo.IO.Storage.LevelDB; using Neo.Persistence; +using System.Collections; using System.Collections.Generic; using LSnapshot = Neo.IO.Storage.LevelDB.Snapshot; namespace Neo.Plugins.Storage { - internal class Snapshot : ISnapshot + /// + /// Iterating over the whole dataset can be time-consuming. Depending upon how large the dataset is. + /// + internal class Snapshot : ISnapshot, IEnumerable> { private readonly DB _db; private readonly LSnapshot _snapshot; @@ -27,7 +31,7 @@ internal class Snapshot : ISnapshot public Snapshot(DB db) { _db = db; - _snapshot = db.GetSnapshot(); + _snapshot = db.CreateSnapshot(); _readOptions = new ReadOptions { FillCache = false, Snapshot = _snapshot }; _batch = new WriteBatch(); } @@ -76,5 +80,15 @@ public bool TryGet(byte[] key, out byte[] value) value = _db.Get(_readOptions, key); return value != null; } + + public IEnumerator> GetEnumerator() + { + using var iterator = _db.CreateIterator(_readOptions); + for (iterator.SeekToFirst(); iterator.Valid(); iterator.Next()) + yield return new KeyValuePair(iterator.Key(), iterator.Value()); + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); } } diff --git a/src/Plugins/LevelDBStore/Plugins/Storage/Store.cs b/src/Plugins/LevelDBStore/Plugins/Storage/Store.cs index c77a6beeb1..0a0f630613 100644 --- a/src/Plugins/LevelDBStore/Plugins/Storage/Store.cs +++ b/src/Plugins/LevelDBStore/Plugins/Storage/Store.cs @@ -11,18 +11,27 @@ using Neo.IO.Storage.LevelDB; using Neo.Persistence; +using System.Collections; using System.Collections.Generic; namespace Neo.Plugins.Storage { - internal class Store : IStore + /// + /// Iterating over the whole dataset can be time-consuming. Depending upon how large the dataset is. + /// + internal class Store : IStore, IEnumerable> { private readonly DB _db; private readonly Options _options; public Store(string path) { - _options = new Options { CreateIfMissing = true, FilterPolicy = Native.leveldb_filterpolicy_create_bloom(15) }; + _options = new Options + { + CreateIfMissing = true, + FilterPolicy = Native.leveldb_filterpolicy_create_bloom(15), + CompressionLevel = CompressionType.SnappyCompression, + }; _db = DB.Open(path, _options); } @@ -60,5 +69,11 @@ public bool TryGet(byte[] key, out byte[] value) public IEnumerable<(byte[], byte[])> Seek(byte[] prefix, SeekDirection direction = SeekDirection.Forward) => _db.Seek(ReadOptions.Default, prefix, direction); + + public IEnumerator> GetEnumerator() => + _db.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); } } diff --git a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs index ed44faab53..7717c29538 100644 --- a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs +++ b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs @@ -10,6 +10,7 @@ // modifications are permitted. using Microsoft.VisualStudio.TestTools.UnitTesting; +using Neo.IO.Storage.LevelDB; using Neo.Persistence; using System.IO; using System.Linq; @@ -72,6 +73,20 @@ public void TestLevelDb() TestPersistenceRead(levelDbStore.GetStore(path_leveldb), false); } + [TestMethod] + public void TestLevelDbDatabase() + { + using var db = DB.Open(Path.GetRandomFileName(), new() { CreateIfMissing = true }); + + db.Put(WriteOptions.Default, [0x00, 0x00, 0x01], [0x01]); + db.Put(WriteOptions.Default, [0x00, 0x00, 0x02], [0x02]); + db.Put(WriteOptions.Default, [0x00, 0x00, 0x03], [0x03]); + + CollectionAssert.AreEqual(new byte[] { 0x01, }, db.Get(ReadOptions.Default, [0x00, 0x00, 0x01])); + CollectionAssert.AreEqual(new byte[] { 0x02, }, db.Get(ReadOptions.Default, [0x00, 0x00, 0x02])); + CollectionAssert.AreEqual(new byte[] { 0x03, }, db.Get(ReadOptions.Default, [0x00, 0x00, 0x03])); + } + [TestMethod] public void TestLevelDbSnapshot() {