From 5a77b3dc09e33b4edfe8da6ce3ada07d722ed08c Mon Sep 17 00:00:00 2001 From: Wojciech Malota-Wojcik Date: Mon, 7 Oct 2024 11:42:09 +0200 Subject: [PATCH] DB commit unit test --- alloc/allocator.go | 38 +++++- db.go | 25 ++-- db_test.go | 290 +++++++++++++++++++++++++++++++++++++++++++++ list/list.go | 10 +- list/list_test.go | 6 +- types/types.go | 1 - 6 files changed, 348 insertions(+), 22 deletions(-) create mode 100644 db_test.go diff --git a/alloc/allocator.go b/alloc/allocator.go index 556d850..fb97ad7 100644 --- a/alloc/allocator.go +++ b/alloc/allocator.go @@ -113,7 +113,8 @@ func (sa SnapshotAllocator) Copy(data []byte) (types.NodeAddress, []byte, error) // Deallocate marks node for deallocation. func (sa SnapshotAllocator) Deallocate(nodeAddress types.NodeAddress, srcSnapshotID types.SnapshotID) error { if srcSnapshotID == sa.snapshotID { - sa.DeallocateImmediately(nodeAddress) + delete(sa.dirtyNodes, nodeAddress) + sa.allocator.Deallocate(nodeAddress) return nil } @@ -137,8 +138,35 @@ func (sa SnapshotAllocator) Deallocate(nodeAddress types.NodeAddress, srcSnapsho return nil } -// DeallocateImmediately dealocates node immediately. -func (sa SnapshotAllocator) DeallocateImmediately(nodeAddress types.NodeAddress) { - delete(sa.dirtyNodes, nodeAddress) - sa.allocator.Deallocate(nodeAddress) +// NewImmediateSnapshotAllocator creates new immediate snapshot deallocator. +func NewImmediateSnapshotAllocator( + snapshotID types.SnapshotID, + parentSnapshotAllocator types.SnapshotAllocator, +) ImmediateSnapshotAllocator { + return ImmediateSnapshotAllocator{ + snapshotID: snapshotID, + parentSnapshotAllocator: parentSnapshotAllocator, + } +} + +// ImmediateSnapshotAllocator deallocates nodes immediately instead of adding them to deallocation list. +type ImmediateSnapshotAllocator struct { + snapshotID types.SnapshotID + parentSnapshotAllocator types.SnapshotAllocator +} + +// Allocate allocates new node. +func (sa ImmediateSnapshotAllocator) Allocate() (types.NodeAddress, []byte, error) { + return sa.parentSnapshotAllocator.Allocate() +} + +// Copy allocates new node and copies content from existing one. +func (sa ImmediateSnapshotAllocator) Copy(data []byte) (types.NodeAddress, []byte, error) { + return sa.parentSnapshotAllocator.Copy(data) +} + +// Deallocate marks node for deallocation. +func (sa ImmediateSnapshotAllocator) Deallocate(nodeAddress types.NodeAddress, _ types.SnapshotID) error { + // using sa.snapshotID instead of the snapshotID argument causes immediate deallocation. + return sa.parentSnapshotAllocator.Deallocate(nodeAddress, sa.snapshotID) } diff --git a/db.go b/db.go index cd2d946..116413c 100644 --- a/db.go +++ b/db.go @@ -27,13 +27,14 @@ type SpaceToCommit struct { // Snapshot represents snapshot. type Snapshot struct { - SnapshotID types.SnapshotID - SingularityNode *types.SingularityNode - SnapshotInfo *types.SnapshotInfo - Snapshots *space.Space[types.SnapshotID, types.SnapshotInfo] - Spaces *space.Space[types.SpaceID, types.SpaceInfo] - SpacesToCommit map[types.SpaceID]SpaceToCommit - Allocator types.SnapshotAllocator + SnapshotID types.SnapshotID + SingularityNode *types.SingularityNode + SnapshotInfo *types.SnapshotInfo + Snapshots *space.Space[types.SnapshotID, types.SnapshotInfo] + Spaces *space.Space[types.SpaceID, types.SpaceInfo] + DeallocationLists *space.Space[types.SnapshotID, types.NodeAddress] + SpacesToCommit map[types.SpaceID]SpaceToCommit + Allocator types.SnapshotAllocator } // New creates new database. @@ -246,6 +247,7 @@ func (db *DB) prepareNextSnapshot(singularityNode types.SingularityNode) error { deallocationLists, db.listNodeAllocator, ) + immediateAllocator := alloc.NewImmediateSnapshotAllocator(snapshotID, allocator) *deallocationLists = *space.New[types.SnapshotID, types.NodeAddress](space.Config[types.SnapshotID, types.NodeAddress]{ SnapshotID: snapshotID, HashMod: &snapshotInfo.DeallocationRoot.HashMod, @@ -255,7 +257,7 @@ func (db *DB) prepareNextSnapshot(singularityNode types.SingularityNode) error { }, PointerNodeAllocator: db.pointerNodeAllocator, DataNodeAllocator: db.snapshotToNodeNodeAllocator, - Allocator: allocator, + Allocator: immediateAllocator, }) snapshots := space.New[types.SnapshotID, types.SnapshotInfo](space.Config[types.SnapshotID, types.SnapshotInfo]{ @@ -267,7 +269,7 @@ func (db *DB) prepareNextSnapshot(singularityNode types.SingularityNode) error { }, PointerNodeAllocator: db.pointerNodeAllocator, DataNodeAllocator: db.snapshotInfoNodeAllocator, - Allocator: allocator, + Allocator: immediateAllocator, }) if singularityNode.SnapshotRoot.State != types.StateFree { @@ -297,8 +299,9 @@ func (db *DB) prepareNextSnapshot(singularityNode types.SingularityNode) error { DataNodeAllocator: db.spaceInfoNodeAllocator, Allocator: allocator, }), - SpacesToCommit: map[types.SpaceID]SpaceToCommit{}, - Allocator: allocator, + DeallocationLists: deallocationLists, + SpacesToCommit: map[types.SpaceID]SpaceToCommit{}, + Allocator: allocator, } return nil diff --git a/db_test.go b/db_test.go new file mode 100644 index 0000000..480096a --- /dev/null +++ b/db_test.go @@ -0,0 +1,290 @@ +package quantum + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/exp/constraints" + + "github.com/outofforest/quantum/alloc" + "github.com/outofforest/quantum/list" + "github.com/outofforest/quantum/space" + "github.com/outofforest/quantum/types" +) + +const spaceID0 types.SpaceID = 0x00 + +func TestCommitNewSnapshots(t *testing.T) { + requireT := require.New(t) + db, allocator := newDB(requireT) + + // Initial state + + snapshot := db.nextSnapshot + requireT.Equal(types.SingularityNode{ + Version: 0x00, + FirstSnapshotID: 0x00, + LastSnapshotID: 0x00, + SnapshotRoot: types.SpaceInfo{ + State: types.StateFree, + Node: 0x00, + HashMod: 0x00, + }, + }, *snapshot.SingularityNode) + requireT.Equal([]types.SnapshotID{}, collectSpaceKeys(snapshot.Snapshots)) + + // Snapshot 0 + + snapshot = db.nextSnapshot + requireT.NoError(db.Commit()) + + nodesAllocated, nodesDeallocated := allocator.Nodes() + requireT.Equal([]types.NodeAddress{0x01}, nodesAllocated) + requireT.Empty(nodesDeallocated) + requireT.Equal([]types.NodeAddress{0x01}, snapshot.Snapshots.Nodes()) + + requireT.Equal(types.SingularityNode{ + Version: 0x00, + FirstSnapshotID: 0x00, + LastSnapshotID: 0x00, + SnapshotRoot: types.SpaceInfo{ + State: types.StateData, + Node: 0x01, + HashMod: 0x00, + }, + }, *snapshot.SingularityNode) + requireT.Equal([]types.SnapshotID{0x00}, collectSpaceKeys(snapshot.Snapshots)) + snapshotInfo, exists := snapshot.Snapshots.Get(0x00) + requireT.True(exists) + requireT.Equal(types.SnapshotInfo{ + PreviousSnapshotID: 0x00, + NextSnapshotID: 0x01, + DeallocationRoot: types.SpaceInfo{ + State: types.StateFree, + Node: 0x00, + HashMod: 0x00, + }, + SpaceRoot: types.SpaceInfo{ + State: types.StateFree, + Node: 0x00, + HashMod: 0x00, + }, + }, snapshotInfo) + + // Snapshot 1 + + s, err := GetSpace[int, int](spaceID0, db) + requireT.NoError(err) + requireT.NoError(s.Set(0, 0)) + requireT.NoError(s.Set(1, 1)) + + snapshot = db.nextSnapshot + requireT.NoError(db.Commit()) + + nodesAllocated, nodesDeallocated = allocator.Nodes() + requireT.Equal([]types.NodeAddress{0x02, 0x03, 0x04}, nodesAllocated) + requireT.Equal([]types.NodeAddress{0x01}, nodesDeallocated) + requireT.Equal([]types.NodeAddress{0x04}, snapshot.Snapshots.Nodes()) + requireT.Equal([]types.NodeAddress{0x03}, snapshot.Spaces.Nodes()) + requireT.Equal([]types.NodeAddress{0x02}, s.Nodes()) + + requireT.Equal(types.SingularityNode{ + Version: 0x00, + FirstSnapshotID: 0x00, + LastSnapshotID: 0x01, + SnapshotRoot: types.SpaceInfo{ + State: types.StateData, + Node: 0x04, + HashMod: 0x00, + }, + }, *snapshot.SingularityNode) + requireT.Equal([]types.SnapshotID{0x00, 0x01}, collectSpaceKeys(snapshot.Snapshots)) + requireT.Equal([]types.SpaceID{spaceID0}, collectSpaceKeys(snapshot.Spaces)) + + snapshotInfo, exists = snapshot.Snapshots.Get(0x00) + requireT.True(exists) + requireT.Equal(types.SnapshotInfo{ + PreviousSnapshotID: 0x00, + NextSnapshotID: 0x01, + DeallocationRoot: types.SpaceInfo{ + State: types.StateFree, + Node: 0x00, + HashMod: 0x00, + }, + SpaceRoot: types.SpaceInfo{ + State: types.StateFree, + Node: 0x00, + HashMod: 0x00, + }, + }, snapshotInfo) + + snapshotInfo, exists = snapshot.Snapshots.Get(0x01) + requireT.True(exists) + requireT.Equal(types.SnapshotInfo{ + PreviousSnapshotID: 0x00, + NextSnapshotID: 0x02, + DeallocationRoot: types.SpaceInfo{ + State: types.StateFree, + Node: 0x00, + HashMod: 0x00, + }, + SpaceRoot: types.SpaceInfo{ + State: types.StateData, + Node: 0x03, + HashMod: 0x00, + }, + }, snapshotInfo) + + spaceInfo, exists := snapshot.Spaces.Get(spaceID0) + requireT.True(exists) + requireT.Equal(types.SpaceInfo{ + State: types.StateData, + Node: 0x02, + HashMod: 0x00, + }, spaceInfo) + + // Snapshot 2 + + s, err = GetSpace[int, int](spaceID0, db) + requireT.NoError(err) + requireT.NoError(s.Set(0, 10)) + requireT.NoError(s.Set(1, 11)) + + snapshot = db.nextSnapshot + requireT.NoError(db.Commit()) + + nodesAllocated, nodesDeallocated = allocator.Nodes() + requireT.Equal([]types.NodeAddress{0x05, 0x06, 0x07, 0x08, 0x09}, nodesAllocated) + requireT.Equal([]types.NodeAddress{0x04}, nodesDeallocated) + requireT.Equal([]types.NodeAddress{0x09}, snapshot.Snapshots.Nodes()) + requireT.Equal([]types.NodeAddress{0x08}, snapshot.Spaces.Nodes()) + requireT.Equal([]types.NodeAddress{0x05}, s.Nodes()) + + requireT.Equal(types.SingularityNode{ + Version: 0x00, + FirstSnapshotID: 0x00, + LastSnapshotID: 0x02, + SnapshotRoot: types.SpaceInfo{ + State: types.StateData, + Node: 0x09, + HashMod: 0x00, + }, + }, *snapshot.SingularityNode) + requireT.Equal([]types.SnapshotID{0x00, 0x01, 0x02}, collectSpaceKeys(snapshot.Snapshots)) + requireT.Equal([]types.SpaceID{spaceID0}, collectSpaceKeys(snapshot.Spaces)) + + snapshotInfo, exists = snapshot.Snapshots.Get(0x00) + requireT.True(exists) + requireT.Equal(types.SnapshotInfo{ + PreviousSnapshotID: 0x00, + NextSnapshotID: 0x01, + DeallocationRoot: types.SpaceInfo{ + State: types.StateFree, + Node: 0x00, + HashMod: 0x00, + }, + SpaceRoot: types.SpaceInfo{ + State: types.StateFree, + Node: 0x00, + HashMod: 0x00, + }, + }, snapshotInfo) + + snapshotInfo, exists = snapshot.Snapshots.Get(0x01) + requireT.True(exists) + requireT.Equal(types.SnapshotInfo{ + PreviousSnapshotID: 0x00, + NextSnapshotID: 0x02, + DeallocationRoot: types.SpaceInfo{ + State: types.StateFree, + Node: 0x00, + HashMod: 0x00, + }, + SpaceRoot: types.SpaceInfo{ + State: types.StateData, + Node: 0x03, + HashMod: 0x00, + }, + }, snapshotInfo) + + snapshotInfo, exists = snapshot.Snapshots.Get(0x02) + requireT.True(exists) + requireT.Equal(types.SnapshotInfo{ + PreviousSnapshotID: 0x01, + NextSnapshotID: 0x03, + DeallocationRoot: types.SpaceInfo{ + State: types.StateData, + Node: 0x07, + HashMod: 0x00, + }, + SpaceRoot: types.SpaceInfo{ + State: types.StateData, + Node: 0x08, + HashMod: 0x00, + }, + }, snapshotInfo) + + spaceInfo, exists = snapshot.Spaces.Get(spaceID0) + requireT.True(exists) + requireT.Equal(types.SpaceInfo{ + State: types.StateData, + Node: 0x05, + HashMod: 0x00, + }, spaceInfo) + requireT.Equal([]types.NodeAddress{0x07}, snapshot.DeallocationLists.Nodes()) + requireT.Equal([]types.SnapshotID{0x01}, collectSpaceKeys(snapshot.DeallocationLists)) + + dList1Address, exists := snapshot.DeallocationLists.Get(0x01) + requireT.True(exists) + requireT.Equal(types.NodeAddress(0x06), dList1Address) + + dList1 := newList(dList1Address, db) + requireT.Equal([]types.NodeAddress{0x02, 0x03}, collectListItems(dList1)) +} + +func newDB(requireT *require.Assertions) (*DB, *alloc.TestAllocator) { + allocator := alloc.NewTestAllocator(alloc.NewAllocator(alloc.Config{ + TotalSize: 1024 * 1024, + NodeSize: 512, + })) + db, err := New(Config{ + Allocator: allocator, + }) + requireT.NoError(err) + return db, allocator +} + +func newList(nodeAddress types.NodeAddress, db *DB) *list.List { + snapshot := db.nextSnapshot + return list.New(list.Config{ + SnapshotID: snapshot.SnapshotID, + Item: &nodeAddress, + NodeAllocator: db.listNodeAllocator, + Allocator: snapshot.Allocator, + }) +} + +func collectSpaceKeys[K constraints.Ordered, V comparable](s *space.Space[K, V]) []K { + keys := []K{} + for item := range s.Iterator() { + keys = append(keys, item.Key) + } + + sort.Slice(keys, func(i, j int) bool { + return keys[i] < keys[j] + }) + return keys +} + +func collectListItems(l *list.List) []types.NodeAddress { + items := []types.NodeAddress{} + for item := range l.Iterator() { + items = append(items, item) + } + + sort.Slice(items, func(i, j int) bool { + return items[i] < items[j] + }) + return items +} diff --git a/list/list.go b/list/list.go index bcd2e8d..74ff8f5 100644 --- a/list/list.go +++ b/list/list.go @@ -41,6 +41,7 @@ func (l *List) Add(nodeAddress types.NodeAddress) error { return nil } listNodeData, listNode := l.config.NodeAllocator.Get(*l.config.Item) + //nolint:nestif if listNode.Header.NumOfItems+listNode.Header.NumOfSideLists < uint64(len(listNode.Items)) { if listNode.Header.SnapshotID < l.config.SnapshotID { newNodeAddress, newNode, err := l.config.NodeAllocator.Copy(l.config.Allocator, listNodeData) @@ -50,7 +51,9 @@ func (l *List) Add(nodeAddress types.NodeAddress) error { newNode.Header.SnapshotID = l.config.SnapshotID oldNodeAddress := *l.config.Item *l.config.Item = newNodeAddress - l.config.Allocator.DeallocateImmediately(oldNodeAddress) + if err := l.config.Allocator.Deallocate(oldNodeAddress, listNode.Header.SnapshotID); err != nil { + return err + } listNode = newNode } @@ -81,6 +84,7 @@ func (l *List) Attach(nodeAddress types.NodeAddress) error { return nil } listNodeData, listNode := l.config.NodeAllocator.Get(*l.config.Item) + //nolint:nestif if listNode.Header.NumOfItems+listNode.Header.NumOfSideLists < uint64(len(listNode.Items)) { if listNode.Header.SnapshotID < l.config.SnapshotID { newNodeAddress, newNode, err := l.config.NodeAllocator.Copy(l.config.Allocator, listNodeData) @@ -90,7 +94,9 @@ func (l *List) Attach(nodeAddress types.NodeAddress) error { newNode.Header.SnapshotID = l.config.SnapshotID oldNodeAddress := *l.config.Item *l.config.Item = newNodeAddress - l.config.Allocator.DeallocateImmediately(oldNodeAddress) + if err := l.config.Allocator.Deallocate(oldNodeAddress, listNode.Header.SnapshotID); err != nil { + return err + } listNode = newNode } diff --git a/list/list_test.go b/list/list_test.go index e10ec64..73b3764 100644 --- a/list/list_test.go +++ b/list/list_test.go @@ -219,7 +219,7 @@ type env struct { Item *types.NodeAddress snapshotID types.SnapshotID - snapshotAllocator alloc.SnapshotAllocator + snapshotAllocator types.SnapshotAllocator nodeAllocator list.NodeAllocator } @@ -230,12 +230,12 @@ func (e *env) NextSnapshot() *list.List { itemCopy := *e.Item e.Item = &itemCopy - e.snapshotAllocator = alloc.NewSnapshotAllocator( + e.snapshotAllocator = alloc.NewImmediateSnapshotAllocator(snapshotID, alloc.NewSnapshotAllocator( snapshotID, e.Allocator, &space.Space[types.SnapshotID, types.NodeAddress]{}, e.nodeAllocator, - ) + )) return list.New(list.Config{ SnapshotID: snapshotID, diff --git a/types/types.go b/types/types.go index b0dbcaa..09624dc 100644 --- a/types/types.go +++ b/types/types.go @@ -44,7 +44,6 @@ type SnapshotAllocator interface { Allocate() (NodeAddress, []byte, error) Copy(data []byte) (NodeAddress, []byte, error) Deallocate(nodeAddress NodeAddress, srcSnapshotID SnapshotID) error - DeallocateImmediately(nodeAddress NodeAddress) } // SpaceNodeHeader is the header common to all space node types.