From 02103b657f7692c0617c7a5bcfd240b82b583a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Ma=C5=82ota-W=C3=B3jcik?= <59281144+outofforest@users.noreply.github.com> Date: Mon, 23 Dec 2024 16:42:33 +0100 Subject: [PATCH] Test snapshot deletion - phase 1 (#339) --- db.go | 4 +- db_test.go | 423 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 425 insertions(+), 2 deletions(-) diff --git a/db.go b/db.go index 7d3cd73..15234d1 100644 --- a/db.go +++ b/db.go @@ -628,8 +628,6 @@ func deleteSnapshot( } snapshotInfo := snapshotSpace.ReadKey(&snapshotInfoValue) - snapshotSpace.DeleteKey(&snapshotInfoValue, tx) - var nextSnapshotInfoValue space.Entry[types.SnapshotID, types.SnapshotInfo] snapshotSpace.Find(&nextSnapshotInfoValue, snapshotInfo.NextSnapshotID, space.StageData) @@ -677,6 +675,8 @@ func deleteSnapshot( } } + snapshotSpace.DeleteKey(&snapshotInfoValue, tx) + nextSnapshotInfo.DeallocationRoot = snapshotInfo.DeallocationRoot nextSnapshotInfo.PreviousSnapshotID = snapshotInfo.PreviousSnapshotID if err := snapshotSpace.SetKey(&nextSnapshotInfoValue, tx, volatileAllocator, nextSnapshotInfo); err != nil { diff --git a/db_test.go b/db_test.go index 4b9b5bb..159874d 100644 --- a/db_test.go +++ b/db_test.go @@ -504,6 +504,429 @@ func TestCommitWithDeallocationLists(t *testing.T) { requireT.Equal(uint64(numOfDeallocationLists), listRootIndex) } +func TestDeletingLatestSnapshotFails(t *testing.T) { + requireT := require.New(t) + + const ( + snapshotID types.SnapshotID = 10 + deleteSnapshotID types.SnapshotID = 9 + ) + + appState := state.NewForTest(t, 10*types.NodeLength) + txFactory := pipeline.NewTransactionRequestFactory() + volatileAllocator := appState.NewVolatileAllocator() + volatileDeallocator := appState.NewVolatileDeallocator() + persistentDeallocator := appState.NewPersistentDeallocator() + + deallocationNodeAssistant, err := space.NewDataNodeAssistant[deallocationKey, types.ListRoot]() + requireT.NoError(err) + + snapshotInfoNodeAssistant, err := space.NewDataNodeAssistant[types.SnapshotID, types.SnapshotInfo]() + requireT.NoError(err) + + var snapshotRoot types.Pointer + + snapshotSpace := space.New[types.SnapshotID, types.SnapshotInfo](space.Config[types.SnapshotID, types.SnapshotInfo]{ + SpaceRoot: types.NodeRoot{ + Pointer: &snapshotRoot, + }, + State: appState, + DataNodeAssistant: snapshotInfoNodeAssistant, + DeletionCounter: lo.ToPtr[uint64](0), + NoSnapshots: true, + }) + var snapshotEntry space.Entry[types.SnapshotID, types.SnapshotInfo] + snapshotSpace.Find(&snapshotEntry, deleteSnapshotID, space.StageData) + requireT.NoError(snapshotSpace.SetKey(&snapshotEntry, txFactory.New(), volatileAllocator, types.SnapshotInfo{ + PreviousSnapshotID: deleteSnapshotID - 1, + NextSnapshotID: deleteSnapshotID + 1, + })) + + tx := txFactory.New() + requireT.Error(deleteSnapshot(snapshotID, deleteSnapshotID, appState, tx, volatileAllocator, volatileDeallocator, + persistentDeallocator, snapshotSpace, deallocationNodeAssistant)) + requireT.Nil(tx.StoreRequest) + requireT.Nil(tx.ListRequest) +} + +func TestDeletingNonExistingSnapshotFails(t *testing.T) { + requireT := require.New(t) + + const ( + snapshotID types.SnapshotID = 10 + deleteSnapshotID types.SnapshotID = 8 + ) + + appState := state.NewForTest(t, 10*types.NodeLength) + txFactory := pipeline.NewTransactionRequestFactory() + volatileAllocator := appState.NewVolatileAllocator() + volatileDeallocator := appState.NewVolatileDeallocator() + persistentDeallocator := appState.NewPersistentDeallocator() + + deallocationNodeAssistant, err := space.NewDataNodeAssistant[deallocationKey, types.ListRoot]() + requireT.NoError(err) + + snapshotInfoNodeAssistant, err := space.NewDataNodeAssistant[types.SnapshotID, types.SnapshotInfo]() + requireT.NoError(err) + + var snapshotRoot types.Pointer + + snapshotSpace := space.New[types.SnapshotID, types.SnapshotInfo](space.Config[types.SnapshotID, types.SnapshotInfo]{ + SpaceRoot: types.NodeRoot{ + Pointer: &snapshotRoot, + }, + State: appState, + DataNodeAssistant: snapshotInfoNodeAssistant, + DeletionCounter: lo.ToPtr[uint64](0), + NoSnapshots: true, + }) + var snapshotEntry space.Entry[types.SnapshotID, types.SnapshotInfo] + snapshotSpace.Find(&snapshotEntry, deleteSnapshotID+1, space.StageData) + requireT.NoError(snapshotSpace.SetKey(&snapshotEntry, txFactory.New(), volatileAllocator, types.SnapshotInfo{ + PreviousSnapshotID: deleteSnapshotID, + NextSnapshotID: deleteSnapshotID + 2, + })) + + tx := txFactory.New() + requireT.Error(deleteSnapshot(snapshotID, deleteSnapshotID, appState, tx, volatileAllocator, volatileDeallocator, + persistentDeallocator, snapshotSpace, deallocationNodeAssistant)) + requireT.Nil(tx.StoreRequest) + requireT.Nil(tx.ListRequest) +} + +func TestDeletingWithoutDeallocationLists(t *testing.T) { + requireT := require.New(t) + + const ( + snapshotID types.SnapshotID = 10 + deleteSnapshotID types.SnapshotID = 8 + ) + + appState := state.NewForTest(t, 10*types.NodeLength) + txFactory := pipeline.NewTransactionRequestFactory() + volatileAllocator := appState.NewVolatileAllocator() + volatileDeallocator := appState.NewVolatileDeallocator() + persistentAllocator := appState.NewPersistentAllocator() + persistentDeallocator := appState.NewPersistentDeallocator() + + deallocationNodeAssistant, err := space.NewDataNodeAssistant[deallocationKey, types.ListRoot]() + requireT.NoError(err) + + snapshotInfoNodeAssistant, err := space.NewDataNodeAssistant[types.SnapshotID, types.SnapshotInfo]() + requireT.NoError(err) + + var snapshotRoot types.Pointer + + snapshotSpace := space.New[types.SnapshotID, types.SnapshotInfo](space.Config[types.SnapshotID, types.SnapshotInfo]{ + SpaceRoot: types.NodeRoot{ + Pointer: &snapshotRoot, + }, + State: appState, + DataNodeAssistant: snapshotInfoNodeAssistant, + DeletionCounter: lo.ToPtr[uint64](0), + NoSnapshots: true, + }) + + var snapshotEntry1 space.Entry[types.SnapshotID, types.SnapshotInfo] + snapshotSpace.Find(&snapshotEntry1, deleteSnapshotID-1, space.StageData) + requireT.NoError(snapshotSpace.SetKey(&snapshotEntry1, txFactory.New(), volatileAllocator, types.SnapshotInfo{ + PreviousSnapshotID: 0, + NextSnapshotID: deleteSnapshotID, + })) + + var snapshotEntry2 space.Entry[types.SnapshotID, types.SnapshotInfo] + snapshotSpace.Find(&snapshotEntry2, deleteSnapshotID, space.StageData) + requireT.NoError(snapshotSpace.SetKey(&snapshotEntry2, txFactory.New(), volatileAllocator, types.SnapshotInfo{ + PreviousSnapshotID: deleteSnapshotID - 1, + NextSnapshotID: deleteSnapshotID + 1, + })) + + var snapshotEntry3 space.Entry[types.SnapshotID, types.SnapshotInfo] + snapshotSpace.Find(&snapshotEntry3, deleteSnapshotID+1, space.StageData) + requireT.NoError(snapshotSpace.SetKey(&snapshotEntry3, txFactory.New(), volatileAllocator, types.SnapshotInfo{ + PreviousSnapshotID: deleteSnapshotID, + NextSnapshotID: deleteSnapshotID + 2, + })) + + for { + if _, err := volatileAllocator.Allocate(); err != nil { + break + } + } + for { + if _, err := persistentAllocator.Allocate(); err != nil { + break + } + } + + tx := txFactory.New() + requireT.NoError(deleteSnapshot(snapshotID, deleteSnapshotID, appState, tx, volatileAllocator, volatileDeallocator, + persistentDeallocator, snapshotSpace, deallocationNodeAssistant)) + + appState.Commit() + + _, err = volatileAllocator.Allocate() + requireT.Error(err) + + _, err = persistentAllocator.Allocate() + requireT.Error(err) + + _, exists := snapshotSpace.Query(deleteSnapshotID) + requireT.False(exists) + + sInfo, exists := snapshotSpace.Query(deleteSnapshotID - 1) + requireT.True(exists) + requireT.Equal(deleteSnapshotID+1, sInfo.NextSnapshotID) + requireT.Equal(types.SnapshotID(0), sInfo.PreviousSnapshotID) + + sInfo, exists = snapshotSpace.Query(deleteSnapshotID + 1) + requireT.True(exists) + requireT.Equal(deleteSnapshotID+2, sInfo.NextSnapshotID) + requireT.Equal(deleteSnapshotID-1, sInfo.PreviousSnapshotID) + requireT.Zero(sInfo.DeallocationRoot) + + requireT.Nil(tx.ListRequest) + + sr := tx.StoreRequest + for range 3 { + requireT.NotNil(sr) + requireT.True(sr.NoSnapshots) + requireT.Equal(int8(1), sr.PointersToStore) + requireT.Equal(snapshotRoot, *sr.Store[0].Pointer) + requireT.Equal(snapshotRoot.VolatileAddress, sr.Store[0].VolatileAddress) + requireT.Zero(snapshotRoot.PersistentAddress) + requireT.Zero(snapshotRoot.SnapshotID) + requireT.Zero(snapshotRoot.Revision) + sr = sr.Next + } + requireT.Nil(sr) +} + +func TestDeletingWithDeallocationListsWhichShouldNotBeDeallocated(t *testing.T) { + requireT := require.New(t) + + const ( + snapshotID types.SnapshotID = 10 + nextSnapshotID types.SnapshotID = 9 + deleteSnapshotID types.SnapshotID = 6 + previousSnapshotID types.SnapshotID = 3 + ) + + appState := state.NewForTest(t, 20*types.NodeLength) + txFactory := pipeline.NewTransactionRequestFactory() + volatileAllocator := appState.NewVolatileAllocator() + volatileDeallocator := appState.NewVolatileDeallocator() + persistentAllocator := appState.NewPersistentAllocator() + persistentDeallocator := appState.NewPersistentDeallocator() + + deallocationNodeAssistant, err := space.NewDataNodeAssistant[deallocationKey, types.ListRoot]() + requireT.NoError(err) + + snapshotInfoNodeAssistant, err := space.NewDataNodeAssistant[types.SnapshotID, types.SnapshotInfo]() + requireT.NoError(err) + + var snapshotRoot types.Pointer + + snapshotSpace := space.New[types.SnapshotID, types.SnapshotInfo](space.Config[types.SnapshotID, types.SnapshotInfo]{ + SpaceRoot: types.NodeRoot{ + Pointer: &snapshotRoot, + }, + State: appState, + DataNodeAssistant: snapshotInfoNodeAssistant, + DeletionCounter: lo.ToPtr[uint64](0), + NoSnapshots: true, + }) + + tx := txFactory.New() + var snapshotEntry1 space.Entry[types.SnapshotID, types.SnapshotInfo] + snapshotSpace.Find(&snapshotEntry1, previousSnapshotID, space.StageData) + requireT.NoError(snapshotSpace.SetKey(&snapshotEntry1, tx, volatileAllocator, types.SnapshotInfo{ + PreviousSnapshotID: 0, + NextSnapshotID: deleteSnapshotID, + })) + + var snapshotEntry2 space.Entry[types.SnapshotID, types.SnapshotInfo] + snapshotSpace.Find(&snapshotEntry2, deleteSnapshotID, space.StageData) + requireT.NoError(snapshotSpace.SetKey(&snapshotEntry2, tx, volatileAllocator, types.SnapshotInfo{ + PreviousSnapshotID: previousSnapshotID, + NextSnapshotID: nextSnapshotID, + })) + + var deallocationRoot types.Pointer + + deallocationListSpace := space.New[deallocationKey, types.ListRoot]( + space.Config[deallocationKey, types.ListRoot]{ + SpaceRoot: types.NodeRoot{ + Pointer: &deallocationRoot, + }, + State: appState, + DataNodeAssistant: deallocationNodeAssistant, + DeletionCounter: lo.ToPtr[uint64](0), + NoSnapshots: true, + }, + ) + + var list1, list2, list3 types.ListRoot + _, err = list.Add(&list1, 500, appState, volatileAllocator, persistentAllocator) + requireT.NoError(err) + _, err = list.Add(&list1, 501, appState, volatileAllocator, persistentAllocator) + requireT.NoError(err) + _, err = list.Add(&list2, 600, appState, volatileAllocator, persistentAllocator) + requireT.NoError(err) + _, err = list.Add(&list2, 601, appState, volatileAllocator, persistentAllocator) + requireT.NoError(err) + _, err = list.Add(&list3, 700, appState, volatileAllocator, persistentAllocator) + requireT.NoError(err) + _, err = list.Add(&list3, 701, appState, volatileAllocator, persistentAllocator) + requireT.NoError(err) + + list1Key := deallocationKey{ + ListSnapshotID: deleteSnapshotID, + SnapshotID: previousSnapshotID, + } + var deallocEntry1 space.Entry[deallocationKey, types.ListRoot] + deallocationListSpace.Find(&deallocEntry1, list1Key, space.StageData) + requireT.NoError(deallocationListSpace.SetKey(&deallocEntry1, tx, volatileAllocator, list1)) + + list2Key := deallocationKey{ + ListSnapshotID: previousSnapshotID, + SnapshotID: previousSnapshotID - 1, + } + var deallocEntry2 space.Entry[deallocationKey, types.ListRoot] + deallocationListSpace.Find(&deallocEntry2, list2Key, space.StageData) + requireT.NoError(deallocationListSpace.SetKey(&deallocEntry2, tx, volatileAllocator, list2)) + + list3Key := deallocationKey{ + ListSnapshotID: previousSnapshotID, + SnapshotID: previousSnapshotID - 2, + } + var deallocEntry3 space.Entry[deallocationKey, types.ListRoot] + deallocationListSpace.Find(&deallocEntry3, list3Key, space.StageData) + requireT.NoError(deallocationListSpace.SetKey(&deallocEntry3, tx, volatileAllocator, list3)) + + deallocationRootPersistentAdress, err := persistentAllocator.Allocate() + requireT.NoError(err) + deallocationRoot.PersistentAddress = deallocationRootPersistentAdress + deallocationRoot.SnapshotID = nextSnapshotID + + var snapshotEntry3 space.Entry[types.SnapshotID, types.SnapshotInfo] + snapshotSpace.Find(&snapshotEntry3, nextSnapshotID, space.StageData) + requireT.NoError(snapshotSpace.SetKey(&snapshotEntry3, txFactory.New(), volatileAllocator, types.SnapshotInfo{ + PreviousSnapshotID: deleteSnapshotID, + NextSnapshotID: nextSnapshotID + 1, + DeallocationRoot: deallocationRoot, + })) + + for { + if _, err := volatileAllocator.Allocate(); err != nil { + break + } + } + for { + if _, err := persistentAllocator.Allocate(); err != nil { + break + } + } + + volatileDeallocator.Deallocate(0x100) + appState.Commit() + + tx = txFactory.New() + requireT.NoError(deleteSnapshot(snapshotID, deleteSnapshotID, appState, tx, volatileAllocator, volatileDeallocator, + persistentDeallocator, snapshotSpace, deallocationNodeAssistant)) + + volatileDeallocator.Deallocate(0x101) + persistentDeallocator.Deallocate(0x101) + appState.Commit() + + _, err = volatileAllocator.Allocate() + requireT.NoError(err) + + volatileAddress, err := volatileAllocator.Allocate() + requireT.NoError(err) + requireT.Equal(deallocationRoot.VolatileAddress, volatileAddress) + + _, err = volatileAllocator.Allocate() + requireT.Error(err) + + _, err = persistentAllocator.Allocate() + requireT.NoError(err) + + persistentAddress, err := persistentAllocator.Allocate() + requireT.NoError(err) + requireT.Equal(deallocationRoot.PersistentAddress, persistentAddress) + + _, err = persistentAllocator.Allocate() + requireT.Error(err) + + _, exists := snapshotSpace.Query(deleteSnapshotID) + requireT.False(exists) + + sInfo, exists := snapshotSpace.Query(previousSnapshotID) + requireT.True(exists) + requireT.Equal(nextSnapshotID, sInfo.NextSnapshotID) + requireT.Equal(types.SnapshotID(0), sInfo.PreviousSnapshotID) + + sInfo, exists = snapshotSpace.Query(nextSnapshotID) + requireT.True(exists) + requireT.Equal(nextSnapshotID+1, sInfo.NextSnapshotID) + requireT.Equal(previousSnapshotID, sInfo.PreviousSnapshotID) + requireT.NotZero(sInfo.DeallocationRoot.VolatileAddress) + + deallocationListSpace = space.New[deallocationKey, types.ListRoot]( + space.Config[deallocationKey, types.ListRoot]{ + SpaceRoot: types.NodeRoot{ + Pointer: lo.ToPtr(sInfo.DeallocationRoot), + }, + State: appState, + DataNodeAssistant: deallocationNodeAssistant, + DeletionCounter: lo.ToPtr[uint64](0), + NoSnapshots: true, + }, + ) + + listRoot, exists := deallocationListSpace.Query(list1Key) + requireT.True(exists) + requireT.Equal(list1, listRoot) + + listRoot, exists = deallocationListSpace.Query(list2Key) + requireT.True(exists) + requireT.Equal(list2, listRoot) + + listRoot, exists = deallocationListSpace.Query(list3Key) + requireT.True(exists) + requireT.Equal(list3, listRoot) + + requireT.Nil(tx.ListRequest) + + sr := tx.StoreRequest + + for range 3 { + requireT.NotNil(sr) + requireT.True(sr.NoSnapshots) + requireT.Equal(int8(1), sr.PointersToStore) + requireT.Equal(sInfo.DeallocationRoot, *sr.Store[0].Pointer) + requireT.Equal(sInfo.DeallocationRoot.VolatileAddress, sr.Store[0].VolatileAddress) + requireT.Zero(snapshotRoot.PersistentAddress) + requireT.Zero(snapshotRoot.SnapshotID) + requireT.Zero(snapshotRoot.Revision) + sr = sr.Next + } + + for range 3 { + requireT.NotNil(sr) + requireT.True(sr.NoSnapshots) + requireT.Equal(int8(1), sr.PointersToStore) + requireT.Equal(snapshotRoot, *sr.Store[0].Pointer) + requireT.Equal(snapshotRoot.VolatileAddress, sr.Store[0].VolatileAddress) + requireT.Zero(snapshotRoot.PersistentAddress) + requireT.Zero(snapshotRoot.SnapshotID) + requireT.Zero(snapshotRoot.Revision) + sr = sr.Next + } + requireT.Nil(sr) +} + func TestPipe01PrepareTransactionsDoesNothingIfTransactionIsNil(t *testing.T) { requireT := require.New(t)