feat: remove nodeKey from node db writes (#602)

Makefile

@@ -55,18 +55,18 @@ lint-fix:
 # bench is the basic tests that shouldn't crash an aws instance
 bench:
 	cd benchmarks && \
-		go test $(LDFLAGS) -tags cleveldb,rocksdb,pebbledb -run=NOTEST -bench=Small . && \
-		go test $(LDFLAGS) -tags cleveldb,rocksdb,pebbledb -run=NOTEST -bench=Medium . && \
+		go test $(LDFLAGS) -tags pebbledb -run=NOTEST -bench=Small . && \
+		go test $(LDFLAGS) -tags pebbledb -run=NOTEST -bench=Medium . && \
 		go test $(LDFLAGS) -run=NOTEST -bench=RandomBytes .
 .PHONY: bench
 
 # fullbench is extra tests needing lots of memory and to run locally
 fullbench:
 	cd benchmarks && \
 		go test $(LDFLAGS) -run=NOTEST -bench=RandomBytes . && \
-		go test $(LDFLAGS) -tags cleveldb,rocksdb,pebbledb -run=NOTEST -bench=Small . && \
-		go test $(LDFLAGS) -tags cleveldb,rocksdb,pebbledb -run=NOTEST -bench=Medium . && \
-		go test $(LDFLAGS) -tags cleveldb,rocksdb,pebbledb -run=NOTEST -timeout=30m -bench=Large . && \
+		go test $(LDFLAGS) -tags pebbledb -run=NOTEST -bench=Small . && \
+		go test $(LDFLAGS) -tags pebbledb -run=NOTEST -bench=Medium . && \
+		go test $(LDFLAGS) -tags pebbledb -run=NOTEST -timeout=30m -bench=Large . && \
 		go test $(LDFLAGS) -run=NOTEST -bench=Mem . && \
 		go test $(LDFLAGS) -run=NOTEST -timeout=60m -bench=LevelDB .
 .PHONY: fullbench

benchmarks/bench_test.go

@@ -308,7 +308,7 @@ func BenchmarkLevelDBLargeData(b *testing.B) {
 		{"goleveldb", 50000, 100, 32, 100},
 		{"goleveldb", 50000, 100, 32, 1000},
 		{"goleveldb", 50000, 100, 32, 10000},
-		{"goleveldb", 50000, 100, 32, 100000},
+		// {"goleveldb", 50000, 100, 32, 100000},
 	}
 	runBenchmarks(b, benchmarks)
 }

go.mod

@@ -5,7 +5,6 @@ go 1.18
 require (
 	github.com/confio/ics23/go v0.7.0
 	github.com/cosmos/cosmos-db v0.0.0-20220822060143-23a8145386c0
-	github.com/cosmos/gogoproto v1.4.2
 	github.com/golang/mock v1.6.0
 	github.com/golangci/golangci-lint v1.50.0
 	github.com/stretchr/testify v1.8.0

go.sum

@@ -136,8 +136,6 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/cosmos/cosmos-db v0.0.0-20220822060143-23a8145386c0 h1:OMu+dCsWVVsHodR4ykMKEj0VtwkNL+xOtyv0vmCmZVQ=
 github.com/cosmos/cosmos-db v0.0.0-20220822060143-23a8145386c0/go.mod h1:n5af5ISKZ7tP0q9hP1TW6MnWh7GrVrNfCLZhg+22gzg=
-github.com/cosmos/gogoproto v1.4.2 h1:UeGRcmFW41l0G0MiefWhkPEVEwvu78SZsHBvI78dAYw=
-github.com/cosmos/gogoproto v1.4.2/go.mod h1:cLxOsn1ljAHSV527CHOtaIP91kK6cCrZETRBrkzItWU=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=

keyformat/key_format.go

@@ -105,6 +105,13 @@ func (kf *KeyFormat) Key(args ...interface{}) []byte {
 	return kf.KeyBytes(segments...)
 }
 
+func (kf *KeyFormat) NodeKey(nodeKey int64) []byte {
+	b := make([]byte, 9)
+	b[0] = kf.prefix
+	binary.BigEndian.PutUint64(b[1:], uint64(nodeKey))
+	return b
+}
+
 // Reads out the bytes associated with each segment of the key format from key.
 func (kf *KeyFormat) ScanBytes(key []byte) [][]byte {
 	segments := make([][]byte, len(kf.layout))

mutable_tree.go

@@ -247,33 +247,36 @@ func (tree *MutableTree) recursiveSet(node *Node, key []byte, value []byte, orph
 		if !tree.skipFastStorageUpgrade {
 			tree.addUnsavedAddition(key, fastnode.NewNode(key, value, version))
 		}
-
+		nonce := node.nodeKey
+		if node.persisted {
+			nonce = tree.IncreaseNonce()
+		} else {
+			node.nodeKey = tree.IncreaseNonce()
+		}
 		switch bytes.Compare(key, node.key) {
 		case -1:
 			return &Node{
 				key:           node.key,
 				subtreeHeight: 1,
 				size:          2,
-				nodeKey:       tree.IncreaseNonce(),
+				nodeKey:       nonce,
+				leftNode:      NewNode(key, value, version, tree.IncreaseNonce()),
 				rightNode:     node,
 				version:       version,
-				leftNode:      NewNode(key, value, version, tree.IncreaseNonce()),
 			}, false, nil
 		case 1:
 			return &Node{
 				key:           key,
 				subtreeHeight: 1,
 				size:          2,
+				nodeKey:       nonce,
 				leftNode:      node,
-				nodeKey:       tree.IncreaseNonce(),
-				version:       version,
 				rightNode:     NewNode(key, value, version, tree.IncreaseNonce()),
+				version:       version,
 			}, false, nil
 		default:
-			nonce := node.nodeKey
 			if node.persisted {
 				*orphans = append(*orphans, node)
-				nonce = tree.IncreaseNonce()
 			}
 			return NewNode(key, value, version, nonce), true, nil
 		}

node.go

@@ -52,7 +52,7 @@ func NewNode(key []byte, value []byte, version, nodeKey int64) *Node {
 //
 // The new node doesn't have its hash saved or set. The caller must set it
 // afterwards.
-func MakeNode(buf []byte) (*Node, error) {
+func MakeNode(nodeKey int64, buf []byte) (*Node, error) {
 	// Read node header (height, size, version, nodeKey, key).
 	height, n, cause := encoding.DecodeVarint(buf)
 	if cause != nil {
@@ -75,12 +75,6 @@ func MakeNode(buf []byte) (*Node, error) {
 	}
 	buf = buf[n:]
 
-	nodeKey, n, cause := encoding.DecodeVarint(buf)
-	if cause != nil {
-		return nil, fmt.Errorf("decoding node.nodeKey, %w", cause)
-	}
-	buf = buf[n:]
-
 	key, n, cause := encoding.DecodeBytes(buf)
 	if cause != nil {
 		return nil, fmt.Errorf("decoding node.key, %w", cause)
@@ -433,7 +427,6 @@ func (node *Node) encodedSize() int {
 	n := 1 +
 		encoding.EncodeVarintSize(node.size) +
 		encoding.EncodeVarintSize(node.version) +
-		encoding.EncodeVarintSize(node.nodeKey) +
 		encoding.EncodeBytesSize(node.key)
 	if node.isLeaf() {
 		n += encoding.EncodeBytesSize(node.value)
@@ -465,10 +458,6 @@ func (node *Node) writeBytes(w io.Writer) error {
 	}
 
 	// Unlike writeHashBytes, nodeKey and key are written for inner nodes.
-	cause = encoding.EncodeVarint(w, node.nodeKey)
-	if cause != nil {
-		return fmt.Errorf("writing nodeKey, %w", cause)
-	}
 	cause = encoding.EncodeBytes(w, node.key)
 	if cause != nil {
 		return fmt.Errorf("writing key, %w", cause)

node_test.go

@@ -29,11 +29,11 @@ func TestNode_encodedSize(t *testing.T) {
 	}
 
 	// leaf node
-	require.Equal(t, 27, node.encodedSize())
+	require.Equal(t, 26, node.encodedSize())
 
 	// non-leaf node
 	node.subtreeHeight = 1
-	require.Equal(t, 20, node.encodedSize())
+	require.Equal(t, 19, node.encodedSize())
 }
 
 func TestNode_encode_decode(t *testing.T) {
@@ -43,7 +43,7 @@ func TestNode_encode_decode(t *testing.T) {
 		expectError bool
 	}{
 		"nil":   {nil, "", true},
-		"empty": {&Node{}, "000000000000", false},
+		"empty": {&Node{}, "0000000000", false},
 		"inner": {&Node{
 			subtreeHeight: 3,
 			version:       2,
@@ -54,15 +54,15 @@ func TestNode_encode_decode(t *testing.T) {
 			nodeKey:       1,
 			leftNodeKey:   2,
 			rightNodeKey:  3,
-		}, "060e0402036b65790404708090a0060410203040", false},
+		}, "060e04036b65790404708090a0060410203040", false},
 		"leaf": {&Node{
 			subtreeHeight: 0,
 			version:       3,
 			size:          1,
 			key:           []byte("key"),
 			value:         []byte("value"),
 			nodeKey:       4,
-		}, "00020608036b65790576616c7565", false},
+		}, "000206036b65790576616c7565", false},
 	}
 	for name, tc := range testcases {
 		tc := tc
@@ -76,7 +76,7 @@ func TestNode_encode_decode(t *testing.T) {
 			require.NoError(t, err)
 			require.Equal(t, tc.expectHex, hex.EncodeToString(buf.Bytes()))
 
-			node, err := MakeNode(buf.Bytes())
+			node, err := MakeNode(tc.node.nodeKey, buf.Bytes())
 			require.NoError(t, err)
 			// since key and value is always decoded to []byte{} we augment the expected struct here
 			if tc.node.key == nil {

nodedb.go

@@ -3,6 +3,7 @@ package iavl
 import (
 	"bytes"
 	"crypto/sha256"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"math"
@@ -133,7 +134,7 @@ func (ndb *nodeDB) GetNode(nodeKey int64) (*Node, error) {
 		return nil, fmt.Errorf("Value missing for key %d corresponding to nodeKey %x", nodeKey, ndb.nodeKey(nodeKey))
 	}
 
-	node, err := MakeNode(buf)
+	node, err := MakeNode(nodeKey, buf)
 	if err != nil {
 		return nil, fmt.Errorf("error reading Node. bytes: %x, error: %v", buf, err)
 	}
@@ -143,7 +144,6 @@ func (ndb *nodeDB) GetNode(nodeKey int64) (*Node, error) {
 		return nil, fmt.Errorf("error getting hash. error: %v", err)
 	}
 
-	node.nodeKey = nodeKey
 	node.persisted = true
 	ndb.nodeCache.Add(node)
 
@@ -657,7 +657,7 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[int64]int64) error {
 		if err != nil {
 			return err
 		}
-		logger.Debug("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, buf.Bytes())
+		logger.Debug("SAVEORPHAN %v-%v %d\n", fromVersion, toVersion, nodeKey)
 		err = ndb.saveOrphan(buf.Bytes(), fromVersion, toVersion)
 		if err != nil {
 			return err
@@ -728,7 +728,7 @@ func (ndb *nodeDB) deleteOrphans(version int64) error {
 }
 
 func (ndb *nodeDB) nodeKey(nodeKey int64) []byte {
-	return nodeKeyFormat.Key(nodeKey)
+	return nodeKeyFormat.NodeKey(nodeKey)
 }
 
 func (ndb *nodeDB) fastNodeKey(key []byte) []byte {
@@ -929,7 +929,8 @@ func (ndb *nodeDB) getRoot(version int64) (int64, int64, error) {
 func (ndb *nodeDB) getRoots() (roots map[int64]struct {
 	rootKey int64
 	nonce   int64
-}, err error) {
+}, err error,
+) {
 	roots = make(map[int64]struct {
 		rootKey int64
 		nonce   int64
@@ -1087,11 +1088,11 @@ func (ndb *nodeDB) traverseNodes(fn func(hash []byte, node *Node) error) error {
 	nodes := []*Node{}
 
 	err := ndb.traversePrefix(nodeKeyFormat.Key(), func(key, value []byte) error {
-		node, err := MakeNode(value)
+		nodeKey := int64(binary.BigEndian.Uint64(key[1:]))
+		node, err := MakeNode(nodeKey, value)
 		if err != nil {
 			return err
 		}
-		nodeKeyFormat.Scan(key, &node.hash)
 		nodes = append(nodes, node)
 		return nil
 	})