-
Notifications
You must be signed in to change notification settings - Fork 1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Neo Core MemoryStore] MemoryStore Unit Tests. #3404
Merged
Merged
Changes from all commits
Commits
Show all changes
17 commits
Select commit
Hold shift + click to select a range
771acc9
test
Jim8y 1f2a8af
Merge branch 'master' into snapshot-tests
cschuchardt88 2e6979c
fix snapshot issue and add tests
Jim8y 88301ba
Merge branch 'snapshot-tests' of github.com:Jim8y/neo into snapshot-t…
Jim8y 8f37558
fix test
Jim8y 8416d06
Merge branch 'master' into snapshot-tests
Jim8y b928241
Merge branch 'master' into snapshot-tests
NGDAdmin a9a651c
Merge branch 'master' into snapshot-tests
NGDAdmin cbbad90
Merge branch 'master' into snapshot-tests
NGDAdmin 20453f6
apply old snapshot
Jim8y e131adf
memory snapshot tests
Jim8y c09637e
memory test
Jim8y 6cfc25e
add more tests
Jim8y 6d045bc
Merge branch 'master' into memory-snapshot
shargon 38a915c
make it more clear
Jim8y ffae0bf
revert storetest
Jim8y 19cf34a
Merge branch 'master' into memory-snapshot
Jim8y File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
127 changes: 127 additions & 0 deletions
127
tests/Neo.UnitTests/Persistence/UT_MemoryClonedCache.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,127 @@ | ||
// Copyright (C) 2015-2024 The Neo Project. | ||
// | ||
// UT_MemoryClonedCache.cs file belongs to the neo project and is free | ||
// software distributed under the MIT software license, see the | ||
// accompanying file LICENSE in the main directory of the | ||
// repository or http://www.opensource.org/licenses/mit-license.php | ||
// for more details. | ||
// | ||
// Redistribution and use in source and binary forms with or without | ||
// modifications are permitted. | ||
|
||
using Microsoft.VisualStudio.TestTools.UnitTesting; | ||
using Neo.Persistence; | ||
using Neo.SmartContract; | ||
|
||
namespace Neo.UnitTests.Persistence; | ||
|
||
/// <summary> | ||
/// When adding data to `datacache` <see cref="DataCache"/>, | ||
/// it gets passed to `snapshotcache` <see cref="SnapshotCache"/> during commit. | ||
/// If `snapshotcache` <see cref="SnapshotCache"/>commits, the data is then passed | ||
/// to the underlying store <see cref="IStore"/>. | ||
/// However, because snapshots <see cref="ISnapshot"/> are immutable, the new data | ||
/// cannot be retrieved from the snapshot <see cref="ISnapshot"/>. | ||
/// | ||
/// When deleting data from `datacache` <see cref="DataCache"/>, | ||
/// it won't exist in `datacache` upon commit, and therefore will be removed from `snapshotcache` <see cref="SnapshotCache"/>. | ||
/// Upon `snapshotcache` <see cref="SnapshotCache"/>commit, the data is deleted from the store <see cref="IStore"/>. | ||
/// However, since the snapshot <see cref="ISnapshot"/> remains unchanged, the data still exists in the snapshot. | ||
/// If you attempt to read this data from `datacache` <see cref="DataCache"/> or `snapshotcache` <see cref="SnapshotCache"/>, | ||
/// which do not have the data, they will retrieve it from the snapshot instead of the store. | ||
/// Thus, they can still access data that has been deleted. | ||
/// </summary> | ||
[TestClass] | ||
public class UT_MemoryClonedCache | ||
{ | ||
private MemoryStore _memoryStore; | ||
private MemorySnapshot _snapshot; | ||
private SnapshotCache _snapshotCache; | ||
private DataCache _dataCache; | ||
|
||
[TestInitialize] | ||
public void Setup() | ||
{ | ||
_memoryStore = new MemoryStore(); | ||
_snapshot = _memoryStore.GetSnapshot() as MemorySnapshot; | ||
_snapshotCache = new SnapshotCache(_snapshot); | ||
_dataCache = _snapshotCache.CreateSnapshot(); | ||
} | ||
|
||
[TestCleanup] | ||
public void CleanUp() | ||
{ | ||
_dataCache.Commit(); | ||
_snapshotCache.Commit(); | ||
_memoryStore.Reset(); | ||
} | ||
|
||
[TestMethod] | ||
public void SingleSnapshotCacheTest() | ||
{ | ||
var key1 = new KeyBuilder(0, 1); | ||
var value1 = new StorageItem([0x03, 0x04]); | ||
|
||
Assert.IsFalse(_dataCache.Contains(key1)); | ||
_dataCache.Add(key1, value1); | ||
|
||
Assert.IsTrue(_dataCache.Contains(key1)); | ||
Assert.IsFalse(_snapshotCache.Contains(key1)); | ||
Assert.IsFalse(_snapshot.Contains(key1.ToArray())); | ||
Assert.IsFalse(_memoryStore.Contains(key1.ToArray())); | ||
|
||
// After the data cache is committed, it should be dropped | ||
// so its value after the commit is meaningless and should not be used. | ||
_dataCache.Commit(); | ||
|
||
Assert.IsTrue(_dataCache.Contains(key1)); | ||
Assert.IsTrue(_snapshotCache.Contains(key1)); | ||
Assert.IsFalse(_snapshot.Contains(key1.ToArray())); | ||
Assert.IsFalse(_memoryStore.Contains(key1.ToArray())); | ||
|
||
// After the snapshot is committed, it should be dropped | ||
// so its value after the commit is meaningless and should not be used. | ||
_snapshotCache.Commit(); | ||
|
||
Assert.IsTrue(_dataCache.Contains(key1)); | ||
Assert.IsTrue(_snapshotCache.Contains(key1)); | ||
Assert.IsFalse(_snapshot.Contains(key1.ToArray())); | ||
Assert.IsTrue(_memoryStore.Contains(key1.ToArray())); | ||
|
||
// Test delete | ||
|
||
// Reset the snapshot to make it accessible to the new value. | ||
_snapshot = _memoryStore.GetSnapshot() as MemorySnapshot; | ||
_snapshotCache = new SnapshotCache(_snapshot); | ||
_dataCache = _snapshotCache.CreateSnapshot(); | ||
|
||
Assert.IsTrue(_dataCache.Contains(key1)); | ||
_dataCache.Delete(key1); | ||
|
||
Assert.IsFalse(_dataCache.Contains(key1)); | ||
Assert.IsTrue(_snapshotCache.Contains(key1)); | ||
Assert.IsTrue(_snapshot.Contains(key1.ToArray())); | ||
Assert.IsTrue(_memoryStore.Contains(key1.ToArray())); | ||
|
||
// After the data cache is committed, it should be dropped | ||
// so its value after the commit is meaningless and should not be used. | ||
_dataCache.Commit(); | ||
|
||
Assert.IsFalse(_dataCache.Contains(key1)); | ||
Assert.IsFalse(_snapshotCache.Contains(key1)); | ||
Assert.IsTrue(_snapshot.Contains(key1.ToArray())); | ||
Assert.IsTrue(_memoryStore.Contains(key1.ToArray())); | ||
|
||
|
||
// After the snapshot cache is committed, it should be dropped | ||
// so its value after the commit is meaningless and should not be used. | ||
_snapshotCache.Commit(); | ||
|
||
// The reason that datacache, snapshotcache still contains key1 is because | ||
// they can not find the value from its cache, so they fetch it from the snapshot of the store. | ||
Assert.IsTrue(_dataCache.Contains(key1)); | ||
Assert.IsTrue(_snapshotCache.Contains(key1)); | ||
Assert.IsTrue(_snapshot.Contains(key1.ToArray())); | ||
Assert.IsFalse(_memoryStore.Contains(key1.ToArray())); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,121 @@ | ||
// Copyright (C) 2015-2024 The Neo Project. | ||
// | ||
// UT_MemorySnapshot.cs file belongs to the neo project and is free | ||
// software distributed under the MIT software license, see the | ||
// accompanying file LICENSE in the main directory of the | ||
// repository or http://www.opensource.org/licenses/mit-license.php | ||
// for more details. | ||
// | ||
// Redistribution and use in source and binary forms with or without | ||
// modifications are permitted. | ||
|
||
using Microsoft.VisualStudio.TestTools.UnitTesting; | ||
using Neo.Persistence; | ||
using System.Linq; | ||
|
||
namespace Neo.UnitTests.Persistence; | ||
|
||
[TestClass] | ||
public class UT_MemorySnapshot | ||
{ | ||
private MemoryStore _memoryStore; | ||
private MemorySnapshot _snapshot; | ||
|
||
[TestInitialize] | ||
public void Setup() | ||
{ | ||
_memoryStore = new MemoryStore(); | ||
_snapshot = _memoryStore.GetSnapshot() as MemorySnapshot; | ||
} | ||
|
||
[TestCleanup] | ||
public void CleanUp() | ||
{ | ||
_memoryStore.Reset(); | ||
} | ||
|
||
[TestMethod] | ||
public void SingleSnapshotTest() | ||
{ | ||
var key1 = new byte[] { 0x01, 0x02 }; | ||
var value1 = new byte[] { 0x03, 0x04 }; | ||
|
||
_snapshot.Delete(key1); | ||
Assert.IsNull(_snapshot.TryGet(key1)); | ||
|
||
// Both Store and Snapshot can not get the value that are cached in the snapshot | ||
_snapshot.Put(key1, value1); | ||
Assert.IsNull(_snapshot.TryGet(key1)); | ||
Assert.IsNull(_memoryStore.TryGet(key1)); | ||
|
||
_snapshot.Commit(); | ||
|
||
// After commit the snapshot, the value can be get from the store but still can not get from the snapshot | ||
CollectionAssert.AreEqual(value1, _memoryStore.TryGet(key1)); | ||
Assert.IsNull(_snapshot.TryGet(key1)); | ||
|
||
_snapshot.Delete(key1); | ||
|
||
// Deleted value can not be found from the snapshot but can still get from the store | ||
// This is because snapshot has no key1 at all. | ||
Assert.IsFalse(_snapshot.Contains(key1)); | ||
Assert.IsTrue(_memoryStore.Contains(key1)); | ||
|
||
_snapshot.Commit(); | ||
|
||
// After commit the snapshot, the value can not be found from the store | ||
Assert.IsFalse(_memoryStore.Contains(key1)); | ||
|
||
// Test seek in order | ||
_snapshot.Put([0x00, 0x00, 0x04], [0x04]); | ||
_snapshot.Put([0x00, 0x00, 0x00], [0x00]); | ||
_snapshot.Put([0x00, 0x00, 0x01], [0x01]); | ||
_snapshot.Put([0x00, 0x00, 0x02], [0x02]); | ||
_snapshot.Put([0x00, 0x00, 0x03], [0x03]); | ||
|
||
// Can not get anything from the snapshot | ||
var entries = _snapshot.Seek([0x00, 0x00, 0x02]).ToArray(); | ||
Assert.AreEqual(0, entries.Length); | ||
} | ||
|
||
[TestMethod] | ||
public void MultiSnapshotTest() | ||
{ | ||
var key1 = new byte[] { 0x01, 0x02 }; | ||
var value1 = new byte[] { 0x03, 0x04 }; | ||
|
||
_snapshot.Delete(key1); | ||
Assert.IsNull(_snapshot.TryGet(key1)); | ||
|
||
// Both Store and Snapshot can not get the value that are cached in the snapshot | ||
_snapshot.Put(key1, value1); | ||
// After commit the snapshot, the value can be get from the store but still can not get from the snapshot | ||
// But can get the value from a new snapshot | ||
_snapshot.Commit(); | ||
var snapshot2 = _memoryStore.GetSnapshot(); | ||
CollectionAssert.AreEqual(value1, _memoryStore.TryGet(key1)); | ||
Assert.IsNull(_snapshot.TryGet(key1)); | ||
CollectionAssert.AreEqual(value1, snapshot2.TryGet(key1)); | ||
|
||
_snapshot.Delete(key1); | ||
|
||
// Deleted value can not being found from the snapshot but can still get from the store and snapshot2 | ||
Assert.IsFalse(_snapshot.Contains(key1)); | ||
Assert.IsTrue(_memoryStore.Contains(key1)); | ||
Assert.IsTrue(snapshot2.Contains(key1)); | ||
|
||
_snapshot.Commit(); | ||
|
||
// After commit the snapshot, the value can not be found from the store, but can be found in snapshots | ||
// Cause snapshot1 or store can not change the status of snapshot2. | ||
Assert.IsFalse(_memoryStore.Contains(key1)); | ||
Assert.IsTrue(snapshot2.Contains(key1)); | ||
Assert.IsFalse(_snapshot.Contains(key1)); | ||
|
||
// Add value via snapshot2 will not affect snapshot1 at all | ||
snapshot2.Put(key1, value1); | ||
snapshot2.Commit(); | ||
Assert.IsNull(_snapshot.TryGet(key1)); | ||
CollectionAssert.AreEqual(value1, snapshot2.TryGet(key1)); | ||
} | ||
} |
134 changes: 134 additions & 0 deletions
134
tests/Neo.UnitTests/Persistence/UT_MemorySnapshotCache.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,134 @@ | ||
// Copyright (C) 2015-2024 The Neo Project. | ||
// | ||
// UT_MemorySnapshotCache.cs file belongs to the neo project and is free | ||
// software distributed under the MIT software license, see the | ||
// accompanying file LICENSE in the main directory of the | ||
// repository or http://www.opensource.org/licenses/mit-license.php | ||
// for more details. | ||
// | ||
// Redistribution and use in source and binary forms with or without | ||
// modifications are permitted. | ||
|
||
using Microsoft.VisualStudio.TestTools.UnitTesting; | ||
using Neo.Persistence; | ||
using Neo.SmartContract; | ||
using System.Linq; | ||
|
||
namespace Neo.UnitTests.Persistence; | ||
|
||
[TestClass] | ||
public class UT_MemorySnapshotCache | ||
{ | ||
private MemoryStore _memoryStore; | ||
private MemorySnapshot _snapshot; | ||
private SnapshotCache _snapshotCache; | ||
|
||
[TestInitialize] | ||
public void Setup() | ||
{ | ||
_memoryStore = new MemoryStore(); | ||
_snapshot = _memoryStore.GetSnapshot() as MemorySnapshot; | ||
_snapshotCache = new SnapshotCache(_snapshot); | ||
} | ||
|
||
[TestCleanup] | ||
public void CleanUp() | ||
{ | ||
_snapshotCache.Commit(); | ||
_memoryStore.Reset(); | ||
} | ||
|
||
[TestMethod] | ||
public void SingleSnapshotCacheTest() | ||
{ | ||
var key1 = new KeyBuilder(0, 1); | ||
var value1 = new StorageItem([0x03, 0x04]); | ||
|
||
_snapshotCache.Delete(key1); | ||
Assert.IsNull(_snapshotCache.TryGet(key1)); | ||
|
||
// Adding value to the snapshot cache will not affect the snapshot or the store | ||
// But the snapshot cache itself can see the added item right after it is added. | ||
_snapshotCache.Add(key1, value1); | ||
|
||
Assert.AreEqual(value1.Value, _snapshotCache.TryGet(key1).Value); | ||
Assert.IsNull(_snapshot.TryGet(key1.ToArray())); | ||
Assert.IsNull(_memoryStore.TryGet(key1.ToArray())); | ||
|
||
// After commit the snapshot cache, it works the same as commit the snapshot. | ||
// the value can be get from the snapshot cache and store but still can not get from the snapshot | ||
_snapshotCache.Commit(); | ||
|
||
Assert.AreEqual(value1.Value, _snapshotCache.TryGet(key1).Value); | ||
Assert.IsFalse(_snapshot.Contains(key1.ToArray())); | ||
Assert.IsTrue(_memoryStore.Contains(key1.ToArray())); | ||
|
||
// Test delete | ||
|
||
// Reset the snapshot to make it accessible to the new value. | ||
_snapshot = _memoryStore.GetSnapshot() as MemorySnapshot; | ||
_snapshotCache = new SnapshotCache(_snapshot); | ||
|
||
// Delete value to the snapshot cache will not affect the snapshot or the store | ||
// But the snapshot cache itself can not see the added item. | ||
_snapshotCache.Delete(key1); | ||
|
||
// Value is removed from the snapshot cache immediately | ||
Assert.IsNull(_snapshotCache.TryGet(key1)); | ||
// But the underline snapshot will not be changed. | ||
Assert.IsTrue(_snapshot.Contains(key1.ToArray())); | ||
// And the store is also not affected. | ||
Assert.IsNotNull(_memoryStore.TryGet(key1.ToArray())); | ||
|
||
// commit the snapshot cache | ||
_snapshotCache.Commit(); | ||
|
||
// Value is removed from both the store, but the snapshot and snapshot cache remains the same. | ||
Assert.IsTrue(_snapshotCache.Contains(key1)); | ||
Assert.IsTrue(_snapshot.Contains(key1.ToArray())); | ||
Assert.IsFalse(_memoryStore.Contains(key1.ToArray())); | ||
} | ||
|
||
[TestMethod] | ||
public void MultiSnapshotCacheTest() | ||
{ | ||
var key1 = new KeyBuilder(0, 1); | ||
var value1 = new StorageItem([0x03, 0x04]); | ||
|
||
_snapshotCache.Delete(key1); | ||
Assert.IsNull(_snapshotCache.TryGet(key1)); | ||
|
||
// Adding value to the snapshot cache will not affect the snapshot or the store | ||
// But the snapshot cache itself can see the added item. | ||
_snapshotCache.Add(key1, value1); | ||
|
||
// After commit the snapshot cache, it works the same as commit the snapshot. | ||
// the value can be get from the snapshot cache but still can not get from the snapshot | ||
_snapshotCache.Commit(); | ||
|
||
// Get a new snapshot cache to test if the value can be seen from the new snapshot cache | ||
var snapshotCache2 = new SnapshotCache(_snapshot); | ||
Assert.IsNull(snapshotCache2.TryGet(key1)); | ||
Assert.IsFalse(_snapshot.Contains(key1.ToArray())); | ||
|
||
// Test delete | ||
|
||
// Reset the snapshot to make it accessible to the new value. | ||
_snapshot = _memoryStore.GetSnapshot() as MemorySnapshot; | ||
_snapshotCache = new SnapshotCache(_snapshot); | ||
|
||
// Delete value to the snapshot cache will affect the snapshot | ||
// But the snapshot and store itself can still see the item. | ||
_snapshotCache.Delete(key1); | ||
|
||
// Commiting the snapshot cache will change the store, but the existing snapshot remains same. | ||
_snapshotCache.Commit(); | ||
|
||
// reset the snapshotcache2 to snapshot | ||
snapshotCache2 = new SnapshotCache(_snapshot); | ||
// Value is removed from the store, but the snapshot remains the same. | ||
// thus the snapshot cache from the snapshot will remain the same. | ||
Assert.IsNotNull(snapshotCache2.TryGet(key1)); | ||
Assert.IsNull(_memoryStore.TryGet(key1.ToArray())); | ||
} | ||
} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I can't agree with this comment (and with related comments in this PR). One of the purposes of DataCache is to serve sequential block's handling in Blockchain (the main part of all Blockchain: OnPersist + transactions handling + PostPersist). And there DataCache usage goes against this comment: we create a single DataCache instance that should be shared between subsequent transaction runs. If transaction is HALTed, then DataCache is being committed and then it's being reused for the next transaction processing, so next transactions can see the changes made by the previous transaction even after DataCacke commit.
So I think that reusage after commit is a substantial part of the DataCache design for sequential execution and can't be prohibited like it's said in the comment.
neo/src/Neo/Ledger/Blockchain.cs
Lines 442 to 461 in c54c204
Parallel execution is different although, and I'm not really sure that it's designed for parallel execution, I'm not that familiar with all DataCache usages.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@AnnaShaleva As i have said everywhere multiple times, the real problem is a committed snapshotcache and clonedcache no longer have the latest store states, that is the reason. You can not agree with what? like any type of execution should be run based on an out dated snapshot? Why would you want to run execution on old store states and get wrong execution result?