chore(lib/trie): add Deltas in internal/trie/tracking (#2896)

- Define `Deltas` struct in `internal/trie/tracking` (with unit tests)
- Replace passing of a plain Go `map[string]struct{}` by passing `pendingDeltas DeltaRecorder` in the trie code
- Define trie local deltas interfaces

internal/trie/tracking/deltas.go

@@ -0,0 +1,56 @@
+// Copyright 2022 ChainSafe Systems (ON)
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package tracking
+
+import "github.com/ChainSafe/gossamer/lib/common"
+
+// Deltas tracks the trie deltas, for example deleted node hashes.
+type Deltas struct {
+	deletedNodeHashes map[common.Hash]struct{}
+}
+
+// New returns a new Deltas struct.
+func New() *Deltas {
+	return &Deltas{
+		deletedNodeHashes: make(map[common.Hash]struct{}),
+	}
+}
+
+// RecordDeleted records a node hash as deleted.
+func (d *Deltas) RecordDeleted(nodeHash common.Hash) {
+	d.deletedNodeHashes[nodeHash] = struct{}{}
+}
+
+// Deleted returns a set (map) of all the recorded deleted
+// node hashes. Note the map returned is not deep copied for
+// performance reasons and so it's not safe for mutation.
+func (d *Deltas) Deleted() (nodeHashes map[common.Hash]struct{}) {
+	return d.deletedNodeHashes
+}
+
+// MergeWith merges the deltas given as argument in the receiving
+// deltas struct.
+func (d *Deltas) MergeWith(deltas DeletedGetter) {
+	for nodeHash := range deltas.Deleted() {
+		d.RecordDeleted(nodeHash)
+	}
+}
+
+// DeepCopy returns a deep copy of the deltas.
+func (d *Deltas) DeepCopy() (deepCopy *Deltas) {
+	if d == nil {
+		return nil
+	}
+
+	deepCopy = &Deltas{}
+
+	if d.deletedNodeHashes != nil {
+		deepCopy.deletedNodeHashes = make(map[common.Hash]struct{}, len(d.deletedNodeHashes))
+		for nodeHash := range d.deletedNodeHashes {
+			deepCopy.deletedNodeHashes[nodeHash] = struct{}{}
+		}
+	}
+
+	return deepCopy
+}

internal/trie/tracking/deltas_test.go

@@ -0,0 +1,182 @@
+// Copyright 2022 ChainSafe Systems (ON)
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package tracking
+
+import (
+	"testing"
+
+	"github.com/ChainSafe/gossamer/lib/common"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_New(t *testing.T) {
+	t.Parallel()
+
+	deltas := New()
+
+	expectedDeltas := &Deltas{
+		deletedNodeHashes: make(map[common.Hash]struct{}),
+	}
+	assert.Equal(t, expectedDeltas, deltas)
+}
+
+func Test_Deltas_RecordDeleted(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		deltas         Deltas
+		nodeHash       common.Hash
+		expectedDeltas Deltas
+	}{
+		"set_in_empty_deltas": {
+			deltas: Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{},
+			},
+			nodeHash: common.Hash{1},
+			expectedDeltas: Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{{1}: {}},
+			},
+		},
+		"set_in_non_empty_deltas": {
+			deltas: Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{{1}: {}},
+			},
+			nodeHash: common.Hash{2},
+			expectedDeltas: Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{
+					{1}: {}, {2}: {},
+				},
+			},
+		},
+		"override_in_deltas": {
+			deltas: Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{{1}: {}},
+			},
+			nodeHash: common.Hash{1},
+			expectedDeltas: Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{{1}: {}},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			testCase.deltas.RecordDeleted(testCase.nodeHash)
+			assert.Equal(t, testCase.expectedDeltas, testCase.deltas)
+		})
+	}
+}
+
+func Test_Deltas_Deleted(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		deltas     Deltas
+		nodeHashes map[common.Hash]struct{}
+	}{
+		"empty_deltas": {},
+		"non_empty_deltas": {
+			deltas: Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{{1}: {}},
+			},
+			nodeHashes: map[common.Hash]struct{}{{1}: {}},
+		},
+	}
+
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			nodeHashes := testCase.deltas.Deleted()
+			assert.Equal(t, testCase.nodeHashes, nodeHashes)
+		})
+	}
+}
+
+func Test_Deltas_MergeWith(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		deltas         Deltas
+		deltasArg      DeletedGetter
+		expectedDeltas Deltas
+	}{
+		"merge_empty_deltas": {
+			deltas: Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{{1}: {}},
+			},
+			deltasArg: &Deltas{},
+			expectedDeltas: Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{{1}: {}},
+			},
+		},
+		"merge_deltas": {
+			deltas: Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{{1}: {}},
+			},
+			deltasArg: &Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{
+					{1}: {}, {2}: {},
+				},
+			},
+			expectedDeltas: Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{
+					{1}: {}, {2}: {},
+				},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			testCase.deltas.MergeWith(testCase.deltasArg)
+			assert.Equal(t, testCase.expectedDeltas, testCase.deltas)
+		})
+	}
+}
+
+func Test_Deltas_DeepCopy(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		deltasOriginal *Deltas
+		deltasCopy     *Deltas
+	}{
+		"nil_deltas": {},
+		"empty_deltas": {
+			deltasOriginal: &Deltas{},
+			deltasCopy:     &Deltas{},
+		},
+		"filled_deltas": {
+			deltasOriginal: &Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{{1}: {}},
+			},
+			deltasCopy: &Deltas{
+				deletedNodeHashes: map[common.Hash]struct{}{{1}: {}},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			deepCopy := testCase.deltasOriginal.DeepCopy()
+
+			assert.Equal(t, testCase.deltasCopy, deepCopy)
+			assertPointersNotEqual(t, testCase.deltasOriginal, deepCopy)
+			if testCase.deltasOriginal != nil {
+				assertPointersNotEqual(t, testCase.deltasOriginal.deletedNodeHashes, deepCopy.deletedNodeHashes)
+			}
+		})
+	}
+}

internal/trie/tracking/helpers_test.go

@@ -0,0 +1,37 @@
+// Copyright 2022 ChainSafe Systems (ON)
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package tracking
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func getPointer(x interface{}) (pointer uintptr, ok bool) {
+	func() {
+		defer func() {
+			ok = recover() == nil
+		}()
+		valueOfX := reflect.ValueOf(x)
+		pointer = valueOfX.Pointer()
+	}()
+	return pointer, ok
+}
+
+func assertPointersNotEqual(t *testing.T, a, b interface{}) {
+	t.Helper()
+	pointerA, okA := getPointer(a)
+	pointerB, okB := getPointer(b)
+	require.Equal(t, okA, okB)
+
+	switch {
+	case pointerA == 0 && pointerB == 0: // nil and nil
+	case okA:
+		assert.NotEqual(t, pointerA, pointerB)
+	default: // values like `int`
+	}
+}

internal/trie/tracking/interfaces.go

@@ -0,0 +1,11 @@
+// Copyright 2022 ChainSafe Systems (ON)
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package tracking
+
+import "github.com/ChainSafe/gossamer/lib/common"
+
+// DeletedGetter gets deleted node hashes.
+type DeletedGetter interface {
+	Deleted() (nodeHashes map[common.Hash]struct{})
+}

lib/trie/database.go

@@ -160,6 +160,37 @@ func PopulateNodeHashes(n *Node, nodeHashes map[string]struct{}) {
 	}
 }
 
+// recordAllDeleted records the node hashes of the given node and all its descendants.
+// Note it does not record inlined nodes.
+// It is assumed the node and its descendant nodes have their Merkle value already
+// computed, or the function will panic.
+func recordAllDeleted(n *Node, recorder DeltaRecorder) {
+	if n == nil {
+		return
+	}
+
+	if len(n.MerkleValue) == 0 {
+		panic(fmt.Sprintf("node with key 0x%x has no Merkle value computed", n.PartialKey))
+	}
+
+	isInlined := len(n.MerkleValue) < 32
+	if isInlined {
+		return
+	}
+
+	nodeHash := common.NewHash(n.MerkleValue)
+	recorder.RecordDeleted(nodeHash)
+
+	if n.Kind() == node.Leaf {
+		return
+	}
+
+	branch := n
+	for _, child := range branch.Children {
+		recordAllDeleted(child, recorder)
+	}
+}
+
 // GetFromDB retrieves a value at the given key from the trie using the database.
 // It recursively descends into the trie using the database starting
 // from the root node until it reaches the node with the given key.
@@ -316,12 +347,14 @@ func (t *Trie) GetChangedNodeHashes() (inserted, deleted map[string]struct{}, er
 	inserted = make(map[string]struct{})
 	err = t.getInsertedNodeHashesAtNode(t.root, inserted)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, fmt.Errorf("getting inserted node hashes: %w", err)
 	}
 
-	deleted = make(map[string]struct{}, len(t.deletedMerkleValues))
-	for k := range t.deletedMerkleValues {
-		deleted[k] = struct{}{}
+	deletedNodeHashes := t.deltas.Deleted()
+	// TODO return deletedNodeHashes directly after changing MerkleValue -> NodeHash
+	deleted = make(map[string]struct{}, len(deletedNodeHashes))
+	for nodeHash := range deletedNodeHashes {
+		deleted[string(nodeHash[:])] = struct{}{}
 	}
 
 	return inserted, deleted, nil

lib/trie/helpers_test.go

@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/ChainSafe/gossamer/internal/trie/node"
+	"github.com/ChainSafe/gossamer/internal/trie/tracking"
+	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/stretchr/testify/require"
 )
 
@@ -135,3 +137,12 @@ func checkMerkleValuesAreSet(t *testing.T, n *Node) {
 		checkMerkleValuesAreSet(t, child)
 	}
 }
+
+func newDeltas(deletedNodeHashesHex ...string) (deltas *tracking.Deltas) {
+	deltas = tracking.New()
+	for _, deletedNodeHashHex := range deletedNodeHashesHex {
+		nodeHash := common.MustHexToHash(deletedNodeHashHex)
+		deltas.RecordDeleted(nodeHash)
+	}
+	return deltas
+}