Merge pull request #2792 from oasislabs/kostko/feature/mkvs-reorg-dirs

Move storage/mkvs/urkel to just storage/mkvs

.buildkite/code.pipeline.yml

@@ -162,7 +162,7 @@ steps:
     command:
       # Build storage interoperability test helpers first.
       - make build-helpers
-      - export OASIS_STORAGE_PROTOCOL_SERVER_BINARY=$(realpath go/storage/mkvs/urkel/interop/urkel-test-helpers)
+      - export OASIS_STORAGE_PROTOCOL_SERVER_BINARY=$(realpath go/storage/mkvs/interop/mkvs-test-helpers)
       - .buildkite/rust/test_generic.sh .
     agents:
       buildkite_agent_size: large
@@ -194,7 +194,7 @@ steps:
     artifact_paths:
       - coverage-misc.txt
       - coverage-oasis-node.txt
-      - coverage-urkel.txt
+      - coverage-mkvs.txt
     plugins:
       <<: *docker_plugin
 
@@ -247,7 +247,7 @@ steps:
     command:
       # Build storage interoperability test helpers first.
       - make build-helpers
-      - export OASIS_STORAGE_PROTOCOL_SERVER_BINARY=$(realpath go/storage/mkvs/urkel/interop/urkel-test-helpers)
+      - export OASIS_STORAGE_PROTOCOL_SERVER_BINARY=$(realpath go/storage/mkvs/interop/mkvs-test-helpers)
       - .buildkite/rust/coverage.sh
     # Don't cause the build to fail, as tarpaulin is pretty unstable at the moment.
     soft_fail: true

.buildkite/go/test_and_coverage.sh

@@ -31,13 +31,13 @@ pushd go
   env -u GOPATH go test -race -coverprofile=../coverage-misc.txt -covermode=atomic -v \
     $(go list ./... | \
         grep -v github.com/oasislabs/oasis-core/go/oasis-node | \
-        grep -v github.com/oasislabs/oasis-core/go/storage/mkvs/urkel )
+        grep -v github.com/oasislabs/oasis-core/go/storage/mkvs )
   # Oasis node tests.
   pushd oasis-node
     env -u GOPATH go test -race -coverpkg ../... -coverprofile=../../coverage-oasis-node.txt -covermode=atomic -v ./...
   popd
-  # Urkel tree tests.
-  pushd storage/mkvs/urkel
-    env -u GOPATH go test -race -coverpkg ./... -coverprofile=../../../../coverage-urkel.txt -covermode=atomic -v ./...
+  # MKVS tests.
+  pushd storage/mkvs
+    env -u GOPATH go test -race -coverpkg ./... -coverprofile=../../../../coverage-mkvs.txt -covermode=atomic -v ./...
   popd
 popd

.buildkite/rust/coverage.sh

@@ -55,7 +55,7 @@ cargo tarpaulin \
   --exclude test-long-term-client \
   --exclude-files '*generated*' \
   --exclude-files tests \
-  --exclude-files runtime/src/storage/mkvs/urkel/interop \
+  --exclude-files runtime/src/storage/mkvs/interop \
   --coveralls ${coveralls_api_token} \
   -v
 set -x

.changelog/2657.internal.md

@@ -0,0 +1,5 @@
+Move storage/mkvs/urkel to just storage/mkvs
+
+The MKVS implementation has been changed from the initial "Urkel tree"
+structure and it is no longer an actual "Urkel tree" so having "urkel" in its
+name is just confusing.

.codecov.yml

@@ -1,3 +1,3 @@
 ignore:
   - "**/*.pb.go"
-  - "runtime/src/storage/mkvs/urkel/interop" # Urkel interoperability test helpers.
+  - "runtime/src/storage/mkvs/interop" # MKVS interoperability test helpers.

Makefile

@@ -95,7 +95,7 @@ test-targets := test-unit test-e2e
 
 test-unit-rust: build-helpers
 	@$(ECHO) "$(CYAN)*** Running Rust unit tests...$(OFF)"
-	@export OASIS_STORAGE_PROTOCOL_SERVER_BINARY=$(realpath go/$(GO_TEST_HELPER_URKEL_PATH)) && \
+	@export OASIS_STORAGE_PROTOCOL_SERVER_BINARY=$(realpath go/$(GO_TEST_HELPER_MKVS_PATH)) && \
 		CARGO_TARGET_DIR=target/default cargo test
 
 test-unit-go:

client/src/transaction/api/storage.rs

@@ -5,7 +5,7 @@ use serde_derive::{Deserialize, Serialize};
 
 use oasis_core_runtime::{
     common::{crypto::hash::Hash, roothash::Namespace},
-    storage::mkvs::{urkel::sync, WriteLog},
+    storage::mkvs::{sync, WriteLog},
 };
 
 #[derive(Clone, Debug, Serialize, Deserialize)]

client/src/transaction/snapshot.rs

@@ -11,10 +11,7 @@ use oasis_core_runtime::{
         roothash::{Block, Namespace},
     },
     storage::{
-        mkvs::{
-            urkel::{sync::*, Root},
-            Prefix, UrkelTree, WriteLog,
-        },
+        mkvs::{sync::*, Prefix, Root, Tree, WriteLog},
         MKVS,
     },
     transaction::types::{TxnCall, TxnOutput},
@@ -60,15 +57,15 @@ pub struct BlockSnapshot {
     pub block_hash: Hash,
 
     read_syncer: RemoteReadSync,
-    mkvs: UrkelTree,
+    mkvs: Tree,
 }
 
 impl Clone for BlockSnapshot {
     fn clone(&self) -> Self {
         let block = self.block.clone();
         let block_hash = self.block_hash;
         let read_syncer = self.read_syncer.clone();
-        let mkvs = UrkelTree::make()
+        let mkvs = Tree::make()
             .with_root(Root {
                 namespace: self.block.header.namespace,
                 version: self.block.header.round,
@@ -88,7 +85,7 @@ impl Clone for BlockSnapshot {
 impl BlockSnapshot {
     pub(super) fn new(storage_client: api::storage::StorageClient, block: Block) -> Self {
         let read_syncer = RemoteReadSync(storage_client);
-        let mkvs = UrkelTree::make()
+        let mkvs = Tree::make()
             .with_root(Root {
                 namespace: block.header.namespace,
                 version: block.header.round,

common.mk

@@ -96,8 +96,8 @@ endif
 # Go build command to use by default.
 GO_BUILD_CMD := env -u GOPATH $(OASIS_GO) build $(GOFLAGS)
 
-# Path to the Urkel interoperability test helpers binary in go/.
-GO_TEST_HELPER_URKEL_PATH := storage/mkvs/urkel/interop/urkel-test-helpers
+# Path to the MKVS interoperability test helpers binary in go/.
+GO_TEST_HELPER_MKVS_PATH := storage/mkvs/interop/mkvs-test-helpers
 
 # Helper that ensures $(NEXT_VERSION) variable is not empty.
 define ENSURE_NEXT_VERSION =

go/.codecov.yml

@@ -1,10 +1,10 @@
 ignore:
   - "grpc" # Autogenerated protobuf code.
-  - "storage/mkvs/urkel/interop" # Urkel interoperability test helpers.
+  - "storage/mkvs/interop" # MKVS interoperability test helpers.
   - "oasis-node/cmd/debug" # Debug and test utilities.
   - "oasis-test-runner" # E2E test runner.
   - "oasis-net-runner" # Test local network runner.
   - "staking/gen_vectors" # Staking test vector generator.
   - "storage/fuzz" # Fuzz tests.
-  - "storage/mkvs/urkel/fuzz" # Fuzz tests.
+  - "storage/mkvs/fuzz" # Fuzz tests.
   - "consensus/tendermint/fuzz" # Fuzz tests.

go/.gitignore

@@ -6,4 +6,4 @@ oasis-node/integrationrunner/integrationrunner.test
 oasis-test-runner/oasis-test-runner
 oasis-net-runner/oasis-net-runner
 oasis-remote-signer/oasis-remote-signer
-storage/mkvs/urkel/interop/urkel-test-helpers
+storage/mkvs/interop/mkvs-test-helpers

go/Makefile

@@ -25,12 +25,12 @@ build: $(go-binaries)
 
 # Build test helpers.
 # List of test helpers to build.
-test-helpers := urkel
+test-helpers := mkvs
 
-# Urkel interoperability test helpers.
-urkel:
+# MKVS interoperability test helpers.
+mkvs:
 	@$(ECHO) "$(MAGENTA)*** Building test helpers for $@...$(OFF)"
-	@$(GO) build $(GOFLAGS) $(GO_EXTRA_FLAGS) -o ./$(GO_TEST_HELPER_URKEL_PATH) ./$(shell dirname $(GO_TEST_HELPER_URKEL_PATH))
+	@$(GO) build $(GOFLAGS) $(GO_EXTRA_FLAGS) -o ./$(GO_TEST_HELPER_MKVS_PATH) ./$(shell dirname $(GO_TEST_HELPER_MKVS_PATH))
 
 build-helpers: $(test-helpers)
 
@@ -116,11 +116,11 @@ fuzz-storage: storage/fuzz/ oasis-node
 	@oasis-node/oasis-node identity init --datadir /tmp/oasis-node-fuzz-storage/identity
 	$(canned-fuzz-run)
 # Fuzz MKVS data structures.
-fuzz-mkvs/Tree: storage/mkvs/urkel/fuzz
+fuzz-mkvs/Tree: storage/mkvs/fuzz
 	$(canned-fuzz-run)
-fuzz-mkvs/Proof: storage/mkvs/urkel/fuzz
+fuzz-mkvs/Proof: storage/mkvs/fuzz
 	$(canned-fuzz-run)
-fuzz-mkvs/Node: storage/mkvs/urkel/fuzz
+fuzz-mkvs/Node: storage/mkvs/fuzz
 	$(canned-fuzz-run)
 
 # Target that only builds all fuzzing infrastructure.

go/oasis-node/cmd/debug/byzantine/executor.go

@@ -15,9 +15,9 @@ import (
 	"github.com/oasislabs/oasis-core/go/runtime/transaction"
 	scheduler "github.com/oasislabs/oasis-core/go/scheduler/api"
 	storage "github.com/oasislabs/oasis-core/go/storage/api"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/syncer"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/writelog"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs/syncer"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs/writelog"
 	"github.com/oasislabs/oasis-core/go/worker/common/p2p"
 )
 
@@ -27,7 +27,7 @@ type computeBatchContext struct {
 
 	ioTree    *transaction.Tree
 	txs       []*transaction.Transaction
-	stateTree urkel.Tree
+	stateTree mkvs.Tree
 
 	stateWriteLog writelog.WriteLog
 	newStateRoot  hash.Hash
@@ -71,7 +71,7 @@ func (cbc *computeBatchContext) openTrees(ctx context.Context, rs syncer.ReadSyn
 		return errors.Wrap(err, "IO tree GetTransactions")
 	}
 
-	cbc.stateTree = urkel.NewWithRoot(rs, nil, storage.Root{
+	cbc.stateTree = mkvs.NewWithRoot(rs, nil, storage.Root{
 		Namespace: cbc.bd.Header.Namespace,
 		Version:   cbc.bd.Header.Round,
 		Hash:      cbc.bd.Header.StateRoot,

go/oasis-node/cmd/debug/byzantine/storage.go

@@ -20,8 +20,8 @@ import (
 	"github.com/oasislabs/oasis-core/go/common/node"
 	scheduler "github.com/oasislabs/oasis-core/go/scheduler/api"
 	storage "github.com/oasislabs/oasis-core/go/storage/api"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/checkpoint"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/syncer"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs/checkpoint"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs/syncer"
 )
 
 var _ storage.Backend = (*honestNodeStorage)(nil)

go/oasis-node/cmd/debug/storage/export.go

@@ -22,7 +22,7 @@ import (
 	storageAPI "github.com/oasislabs/oasis-core/go/storage/api"
 	storageClient "github.com/oasislabs/oasis-core/go/storage/client"
 	storageDatabase "github.com/oasislabs/oasis-core/go/storage/database"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs"
 )
 
 const cfgExportDir = "storage.export.dir"
@@ -104,8 +104,8 @@ func exportRuntime(dataDir, destDir string, id common.Namespace, rtg *registry.R
 		Version:   rtg.Round,
 		Hash:      rtg.StateRoot,
 	}
-	tree := urkel.NewWithRoot(storageBackend, nil, root)
-	it := tree.NewIterator(context.Background(), urkel.IteratorPrefetch(10_000))
+	tree := mkvs.NewWithRoot(storageBackend, nil, root)
+	it := tree.NewIterator(context.Background(), mkvs.IteratorPrefetch(10_000))
 	defer it.Close()
 
 	fn := fmt.Sprintf("storage-dump-%v-%d.json",
@@ -116,7 +116,7 @@ func exportRuntime(dataDir, destDir string, id common.Namespace, rtg *registry.R
 	return exportIterator(fn, &root, it)
 }
 
-func exportIterator(fn string, root *storageAPI.Root, it urkel.Iterator) error {
+func exportIterator(fn string, root *storageAPI.Root, it mkvs.Iterator) error {
 	// Create the dump file, and initialize a JSON stream encoder.
 	f, err := os.Create(fn)
 	if err != nil {

go/oasis-node/cmd/debug/storage/storage.go

@@ -18,7 +18,7 @@ import (
 	runtimeClient "github.com/oasislabs/oasis-core/go/runtime/client/api"
 	"github.com/oasislabs/oasis-core/go/storage"
 	storageAPI "github.com/oasislabs/oasis-core/go/storage/api"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/node"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs/node"
 	storageWorkerAPI "github.com/oasislabs/oasis-core/go/worker/storage/api"
 	"github.com/oasislabs/oasis-core/go/worker/storage/committee"
 )

go/oasis-node/cmd/registry/runtime/runtime.go

@@ -31,7 +31,7 @@ import (
 	cmdSigner "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/signer"
 	registry "github.com/oasislabs/oasis-core/go/registry/api"
 	storage "github.com/oasislabs/oasis-core/go/storage/api"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs"
 )
 
 const (
@@ -324,8 +324,8 @@ func runtimeFromFlags() (*registry.Runtime, signature.Signer, error) {
 			return nil, nil, err
 		}
 
-		// Use in-memory Urkel tree to calculate the new root.
-		tree := urkel.New(nil, nil)
+		// Use in-memory MKVS tree to calculate the new root.
+		tree := mkvs.New(nil, nil)
 		ctx := context.Background()
 		for _, logEntry := range log {
 			err = tree.Insert(ctx, logEntry.Key, logEntry.Value)

go/oasis-node/cmd/storage/benchmark/benchmark.go

@@ -121,7 +121,7 @@ func doBenchmark(cmd *cobra.Command, args []string) { // nolint: gocyclo
 		buf := make([]byte, sz)
 		key := []byte(strconv.Itoa(sz))
 
-		// This will store the new Urkel tree root for later lookups.
+		// This will store the new MKVS tree root for later lookups.
 		var newRoot storageAPI.Root
 		newRoot.Namespace = ns
 		newRoot.Version = 1

go/oasis-test-runner/scenario/e2e/storage_sync.go

@@ -11,7 +11,7 @@ import (
 	"github.com/oasislabs/oasis-core/go/oasis-test-runner/scenario"
 	runtimeClient "github.com/oasislabs/oasis-core/go/runtime/client/api"
 	"github.com/oasislabs/oasis-core/go/storage/database"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/checkpoint"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs/checkpoint"
 )
 
 var (

go/runtime/registry/storage_router.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/oasislabs/oasis-core/go/common"
 	"github.com/oasislabs/oasis-core/go/storage/api"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/checkpoint"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs/checkpoint"
 )
 
 var _ api.Backend = (*storageRouter)(nil)

go/runtime/transaction/transaction.go

@@ -11,10 +11,10 @@ import (
 	"github.com/oasislabs/oasis-core/go/common/cbor"
 	"github.com/oasislabs/oasis-core/go/common/crypto/hash"
 	"github.com/oasislabs/oasis-core/go/common/keyformat"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/node"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/syncer"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/writelog"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs/node"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs/syncer"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs/writelog"
 )
 
 // NOTE: This should be kept in sync with runtime/src/transaction/tree.rs.
@@ -154,14 +154,14 @@ func (t Transaction) asOutputArtifacts() outputArtifacts {
 // Tree is a Merkle tree containing transaction artifacts.
 type Tree struct {
 	ioRoot node.Root
-	tree   urkel.Tree
+	tree   mkvs.Tree
 }
 
 // NewTree creates a new transaction artifacts tree.
 func NewTree(rs syncer.ReadSyncer, ioRoot node.Root) *Tree {
 	return &Tree{
 		ioRoot: ioRoot,
-		tree:   urkel.NewWithRoot(rs, nil, ioRoot, urkel.Capacity(50000, 16*1024*1024)),
+		tree:   mkvs.NewWithRoot(rs, nil, ioRoot, mkvs.Capacity(50000, 16*1024*1024)),
 	}
 }
 
@@ -216,7 +216,7 @@ func (bo inBatchOrder) Less(i, j int) bool { return bo.order[i] < bo.order[j] }
 
 // GetInputBatch returns a batch of transaction input artifacts in batch order.
 func (t *Tree) GetInputBatch(ctx context.Context) (RawBatch, error) {
-	it := t.tree.NewIterator(ctx, urkel.IteratorPrefetch(prefetchArtifactCount))
+	it := t.tree.NewIterator(ctx, mkvs.IteratorPrefetch(prefetchArtifactCount))
 	defer it.Close()
 
 	var curTx hash.Hash
@@ -256,7 +256,7 @@ func (t *Tree) GetInputBatch(ctx context.Context) (RawBatch, error) {
 // GetTransactions returns a list of all transaction artifacts in the tree
 // in a stable order (transactions are ordered by their hash).
 func (t *Tree) GetTransactions(ctx context.Context) ([]*Transaction, error) {
-	it := t.tree.NewIterator(ctx, urkel.IteratorPrefetch(prefetchArtifactCount))
+	it := t.tree.NewIterator(ctx, mkvs.IteratorPrefetch(prefetchArtifactCount))
 	defer it.Close()
 
 	var curTx hash.Hash
@@ -384,7 +384,7 @@ func (t *Tree) GetTransactionMultiple(ctx context.Context, txHashes []hash.Hash)
 
 // GetTags retrieves all tags emitted in this tree.
 func (t *Tree) GetTags(ctx context.Context) (Tags, error) {
-	it := t.tree.NewIterator(ctx, urkel.IteratorPrefetch(prefetchArtifactCount))
+	it := t.tree.NewIterator(ctx, mkvs.IteratorPrefetch(prefetchArtifactCount))
 	defer it.Close()
 
 	var curTx hash.Hash

go/runtime/transaction/transaction_test.go

@@ -8,14 +8,14 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/oasislabs/oasis-core/go/common/crypto/hash"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/node"
-	"github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/writelog"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs/node"
+	"github.com/oasislabs/oasis-core/go/storage/mkvs/writelog"
 )
 
 func TestTransaction(t *testing.T) {
 	ctx := context.Background()
-	store := urkel.New(nil, nil)
+	store := mkvs.New(nil, nil)
 
 	var emptyRoot node.Root
 	emptyRoot.Empty()