Merge pull request #4279 from oasisprotocol/tjanez/runtime-hex-encoded

go/common: Change Namespace's and Hash's text (un)marshaling to use the hex form

.changelog/4279.breaking.1.md

@@ -0,0 +1 @@
+go/common: Change `Namespace`'s text (un)marshaling to use the hex form

.changelog/4279.breaking.2.md

@@ -0,0 +1 @@
+go/common/crypto/hash: Change `Hash`'s text (un)marshaling to use the hex form

docs/setup/deploying-a-runtime.md

@@ -90,9 +90,8 @@ set up environment variables to simplify instructions.
   `/tmp/runtime-example/net-runner/network/genesis.json`).
 - `RUNTIME_ID` - See [runtime identifiers] on how to choose a runtime
   identifier. In this example we use
-  `gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEjRWc=`, which is the `base64` encoded:
-  `8000000000000000000000000000000000000000000000000000000001234567` - a test
-  identifier that will not work outside local tests.
+  `8000000000000000000000000000000000000000000000000000000001234567` which is a
+  test identifier that will not work outside local tests.
 - `RUNTIME_GENESIS_JSON` - Path to the runtime genesis state file. The runtime
   used in this example does not use a genesis file.
 - `NONCE` - Entity account nonce. If you followed the guide, nonce `0` would be
@@ -104,7 +103,7 @@ set up environment variables to simplify instructions.
 export ENTITY_DIR=/tmp/runtime-example/net-runner/network/entity-2/
 export ENTITY_ID=+MJpnSTzc11dNI5emMa+asCJH5cxBiBCcpbYE4XBdso=
 export GENESIS_JSON=/tmp/runtime-example/net-runner/network/genesis.json
-export RUNTIME_ID=gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEjRWc=
+export RUNTIME_ID=8000000000000000000000000000000000000000000000000000000001234567
 export RUNTIME_DESCRIPTOR=/tmp/runtime-example/runtime_descriptor.json
 export NONCE=0
 ```
@@ -118,7 +117,7 @@ cat << EOF > "${RUNTIME_DESCRIPTOR}"
   "id": "${RUNTIME_ID}",
   "entity_id": "${ENTITY_ID}",
   "genesis": {
-    "state_root": "xnK40e9W7Sirh8NiLFEUBpvdOte4+XN0mNDAHs7wlno=",
+    "state_root": "c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a",
     "state": null,
     "storage_receipts": null,
     "round": 0
@@ -222,14 +221,57 @@ oasis-node registry runtime list \
 
 Should give output similar to
 
-<!-- markdownlint-disable line-length -->
-
 ```
-{"v":2,"id":"gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEjRWc=","entity_id":"+MJpnSTzc11dNI5emMa+asCJH5cxBiBCcpbYE4XBdso=","genesis":{"state_root":"xnK40e9W7Sirh8NiLFEUBpvdOte4+XN0mNDAHs7wlno=","state":null,"storage_receipts":null,"round":0},"kind":1,"tee_hardware":0,"versions":{"version":{}},"executor":{"group_size":1,"group_backup_size":0,"allowed_stragglers":0,"round_timeout":5,"max_messages":32},"txn_scheduler":{"algorithm":"simple","batch_flush_timeout":1000000000,"max_batch_size":1000,"max_batch_size_bytes":16777216,"propose_batch_timeout":5},"storage":{"group_size":1,"min_write_replication":1,"max_apply_write_log_entries":100000,"max_apply_ops":2,"checkpoint_interval":10000,"checkpoint_num_kept":2,"checkpoint_chunk_size":8388608},"admission_policy":{"entity_whitelist":{"entities":{"+MJpnSTzc11dNI5emMa+asCJH5cxBiBCcpbYE4XBdso=":{}}}},"staking":{},"governance_model":"entity"}
+{
+  "v": 2,
+  "id": "8000000000000000000000000000000000000000000000000000000001234567",
+  "entity_id": "+MJpnSTzc11dNI5emMa+asCJH5cxBiBCcpbYE4XBdso=",
+  "genesis": {
+    "state_root": "c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a",
+    "state": null,
+    "storage_receipts": null,
+    "round": 0
+  },
+  "kind": 1,
+  "tee_hardware": 0,
+  "versions": {
+    "version": {}
+  },
+  "executor": {
+    "group_size": 1,
+    "group_backup_size": 0,
+    "allowed_stragglers": 0,
+    "round_timeout": 5,
+    "max_messages": 32
+  },
+  "txn_scheduler": {
+    "algorithm": "simple",
+    "batch_flush_timeout": 1000000000,
+    "max_batch_size": 1000,
+    "max_batch_size_bytes": 16777216,
+    "propose_batch_timeout": 5
+  },
+  "storage": {
+    "group_size": 1,
+    "min_write_replication": 1,
+    "max_apply_write_log_entries": 100000,
+    "max_apply_ops": 2,
+    "checkpoint_interval": 10000,
+    "checkpoint_num_kept": 2,
+    "checkpoint_chunk_size": 8388608
+  },
+  "admission_policy": {
+    "entity_whitelist": {
+      "entities": {
+        "+MJpnSTzc11dNI5emMa+asCJH5cxBiBCcpbYE4XBdso=": {}
+      }
+    }
+  },
+  "staking": {},
+  "governance_model": "entity"
+}
 ```
 
-<!-- markdownlint-enable line-length -->
-
 {% hint style="info" %} Since we did not setup any runtime nodes, the runtime
 will get [suspended] until nodes for the runtime register. {% endhint %}
 

go/common/crypto/hash/hash.go

@@ -50,17 +50,28 @@ func (h *Hash) UnmarshalBinary(data []byte) error {
 
 // MarshalText encodes a Hash into text form.
 func (h Hash) MarshalText() (data []byte, err error) {
-	return []byte(base64.StdEncoding.EncodeToString(h[:])), nil
+	return h.MarshalHex()
 }
 
 // UnmarshalText decodes a text marshaled Hash.
 func (h *Hash) UnmarshalText(text []byte) error {
-	b, err := base64.StdEncoding.DecodeString(string(text))
+	err := h.UnmarshalHex(string(text))
 	if err != nil {
-		return err
+		// For backwards compatibility (e.g. to be able to load the
+		// Cobalt Upgrade genesis file), fallback to accepting
+		// Base64-encoded Hash values.
+		b, err := base64.StdEncoding.DecodeString(string(text))
+		if err != nil {
+			return err
+		}
+		return h.UnmarshalBinary(b)
 	}
+	return nil
+}
 
-	return h.UnmarshalBinary(b)
+// MarshalHex encodes a Hash into a hexadecimal form.
+func (h *Hash) MarshalHex() ([]byte, error) {
+	return []byte(hex.EncodeToString(h[:])), nil
 }
 
 // UnmarshalHex deserializes a hexadecimal text string into the given type.

go/common/namespace.go

@@ -63,18 +63,29 @@ func (n *Namespace) UnmarshalBinary(data []byte) error {
 }
 
 // MarshalText encodes a namespace identifier into text form.
-func (n Namespace) MarshalText() (data []byte, err error) {
-	return []byte(base64.StdEncoding.EncodeToString(n[:])), nil
+func (n Namespace) MarshalText() ([]byte, error) {
+	return n.MarshalHex()
 }
 
 // UnmarshalText decodes a text marshaled namespace identifier.
 func (n *Namespace) UnmarshalText(text []byte) error {
-	b, err := base64.StdEncoding.DecodeString(string(text))
+	err := n.UnmarshalHex(string(text))
 	if err != nil {
-		return err
+		// For backwards compatibility (e.g. to be able to load the
+		// Cobalt Upgrade genesis file), fallback to accepting
+		// Base64-encoded namespace identifiers.
+		b, err := base64.StdEncoding.DecodeString(string(text))
+		if err != nil {
+			return err
+		}
+		return n.UnmarshalBinary(b)
 	}
+	return nil
+}
 
-	return n.UnmarshalBinary(b)
+// MarshalHex encodes a namespace identifier into a hexadecimal form.
+func (n *Namespace) MarshalHex() ([]byte, error) {
+	return []byte(hex.EncodeToString(n[:])), nil
 }
 
 // UnmarshalHex deserializes a hexadecimal text string into the given type.

go/oasis-test-runner/scenario/e2e/genesis_file.go

@@ -4,7 +4,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net/http"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -15,6 +18,12 @@ import (
 	"github.com/oasisprotocol/oasis-core/go/oasis-test-runner/scenario"
 )
 
+const (
+	// Parameters from https://docs.oasis.dev/general/oasis-network/network-parameters.
+	latestMainnetGenesisURL          = "https://github.com/oasisprotocol/mainnet-artifacts/releases/download/2021-04-28/genesis.json"
+	latestMainnetGenesisDocumentHash = "53852332637bacb61b91b6411ab4095168ba02a50be4c3f82448438826f23898"
+)
+
 // GenesisFile is the scenario for testing the correctness of marshalled genesis
 // documents.
 var GenesisFile scenario.Scenario = &genesisFileImpl{
@@ -60,7 +69,7 @@ func (s *genesisFileImpl) Run(childEnv *env.Env) error {
 	cfg := s.Net.Config()
 	cfg.GenesisFile = s.Net.GenesisPath()
 
-	if err := s.runGenesisCheckCmd(childEnv, s.Net.GenesisPath()); err != nil {
+	if _, err := s.runGenesisCheckCmd(childEnv, s.Net.GenesisPath()); err != nil {
 		return fmt.Errorf("e2e/genesis-file: running genesis check failed: %w", err)
 	}
 	s.Logger.Info("manually provisioned genesis file passed genesis check command")
@@ -88,20 +97,74 @@ func (s *genesisFileImpl) Run(childEnv *env.Env) error {
 		return fmt.Errorf("e2e/genesis-file: failed to dump state: error: %w output: %s", err, out.String())
 	}
 
-	if err = s.runGenesisCheckCmd(childEnv, dumpPath); err != nil {
+	if _, err = s.runGenesisCheckCmd(childEnv, dumpPath); err != nil {
 		return fmt.Errorf("e2e/genesis-file: running genesis check failed: %w", err)
 	}
 	s.Logger.Info("genesis file from dumped network state passed genesis check command")
 
-	uncanonicalPath := filepath.Join(childEnv.Dir(), "genesis_uncanonical.json")
-	if err = s.createUncanonicalGenesisFile(childEnv, uncanonicalPath); err != nil {
-		return fmt.Errorf("e2e/genesis-file: creating uncanonical genesis file failed: %w", err)
+	// Check if the latest Mainnet genesis file passes genesis check.
+	latestMainnetGenesis := filepath.Join(childEnv.Dir(), "genesis_mainnet.json")
+	if err = s.downloadLatestMainnetGenesisFile(childEnv, latestMainnetGenesis); err != nil {
+		return fmt.Errorf("e2e/genesis-file: failed to download latest Mainnet genesis "+
+			"file at '%s': %w", latestMainnetGenesisURL, err)
 	}
-	err = s.runGenesisCheckCmd(childEnv, uncanonicalPath)
+	_, err = s.runGenesisCheckCmd(childEnv, latestMainnetGenesis)
 	expectedError := "genesis file is not in canonical form, see the diff on stderr"
 	switch {
 	case err == nil:
-		return fmt.Errorf("e2e/genesis-file: running genesis check for an uncanonical genesis file should fail")
+		return fmt.Errorf("e2e/genesis-file: running genesis check for the latest Mainnet"+
+			" genesis file at '%s' should fail with '%s'",
+			latestMainnetGenesisURL, expectedError,
+		)
+	case !strings.Contains(err.Error(), expectedError):
+		return fmt.Errorf(
+			"e2e/genesis-file: running genesis check for the latest Mainnet genesis "+
+				"file should fail with an error containing: '%s' (actual error: %s)",
+			expectedError, err,
+		)
+	default:
+		s.Logger.Info("latest Mainnet genesis file is OK, but is not in canonical form")
+	}
+
+	// Convert latest Mainnet genesis file to canonical form and ensure its Genesis document's
+	// hash matches the authoritative one.
+	latestMainnetGenesisCanonical := filepath.Join(childEnv.Dir(), "genesis_mainnet_canonical.json")
+	if err = s.runFixGenesisCmd(childEnv, latestMainnetGenesis, latestMainnetGenesisCanonical); err != nil {
+		return fmt.Errorf("e2e/genesis-file: failed run fix-genesis on latest Mainnet genesis "+
+			"file at '%s': %w", latestMainnetGenesisURL, err)
+	}
+	checkOut, err := s.runGenesisCheckCmd(childEnv, latestMainnetGenesisCanonical)
+	switch {
+	case err != nil:
+		return fmt.Errorf("e2e/genesis-file: running genesis check for the latest Mainnet"+
+			" genesis file at '%s' converted to canonical form failed: %w",
+			latestMainnetGenesisURL, err,
+		)
+	case !strings.Contains(checkOut, latestMainnetGenesisDocumentHash):
+		return fmt.Errorf(
+			"e2e/genesis-file: running genesis check for the latest Mainnet genesis "+
+				"file converted to canonical form should return the correct "+
+				"genesis document's hash: '%s' (actual output: %s)",
+			latestMainnetGenesisDocumentHash, checkOut,
+		)
+	default:
+		s.Logger.Info("latest Mainnet genesis file converted to canonical form is OK")
+	}
+
+	// Make sure a genesis file in an uncanonical form doesn't pass genesis check command and
+	// returns an appropriate error.
+	uncanonicalGenesis := filepath.Join(childEnv.Dir(), "genesis_uncanonical.json")
+	if err = s.createUncanonicalGenesisFile(childEnv, uncanonicalGenesis); err != nil {
+		return fmt.Errorf("e2e/genesis-file: creating uncanonical genesis file failed: %w", err)
+	}
+	_, err = s.runGenesisCheckCmd(childEnv, uncanonicalGenesis)
+	expectedError = "genesis file is not in canonical form, see the diff on stderr"
+	switch {
+	case err == nil:
+		return fmt.Errorf("e2e/genesis-file: running genesis check for an uncanonical "+
+			"genesis file should fail with '%s'",
+			expectedError,
+		)
 	case !strings.Contains(err.Error(), expectedError):
 		return fmt.Errorf(
 			"e2e/genesis-file: running genesis check for an uncanonical genesis file "+
@@ -115,7 +178,7 @@ func (s *genesisFileImpl) Run(childEnv *env.Env) error {
 	return nil
 }
 
-func (s *genesisFileImpl) runGenesisCheckCmd(childEnv *env.Env, genesisFilePath string) error {
+func (s *genesisFileImpl) runGenesisCheckCmd(childEnv *env.Env, genesisFilePath string) (string, error) {
 	args := []string{
 		"genesis", "check",
 		"--genesis.file", genesisFilePath,
@@ -124,7 +187,45 @@ func (s *genesisFileImpl) runGenesisCheckCmd(childEnv *env.Env, genesisFilePath
 	}
 	out, err := cli.RunSubCommandWithOutput(childEnv, s.Logger, "genesis-file", s.Net.Config().NodeBinary, args)
 	if err != nil {
-		return fmt.Errorf("genesis check failed: error: %w output: %s", err, out.String())
+		return "", fmt.Errorf("genesis check failed: error: %w output: %s", err, out.String())
+	}
+	return out.String(), nil
+}
+
+func (s *genesisFileImpl) runFixGenesisCmd(childEnv *env.Env, genesisFilePath, fixedGenesisFilePath string) error {
+	args := []string{
+		"debug", "fix-genesis",
+		"--genesis.file", genesisFilePath,
+		"--genesis.new_file", fixedGenesisFilePath,
+		"--debug.dont_blame_oasis",
+		"--debug.allow_test_keys",
+	}
+	out, err := cli.RunSubCommandWithOutput(childEnv, s.Logger, "genesis-file", s.Net.Config().NodeBinary, args)
+	if err != nil {
+		return fmt.Errorf("debug fix-genesis failed: error: %w output: %s", err, out.String())
+	}
+	return nil
+}
+
+func (s *genesisFileImpl) downloadLatestMainnetGenesisFile(childEnv *env.Env, latestMainnetGenesisFilePath string) error {
+	// Get the data.
+	resp, err := http.Get(latestMainnetGenesisURL)
+	if err != nil {
+		return fmt.Errorf("failed to download genesis file: %w", err)
+	}
+	defer resp.Body.Close()
+
+	// Create the file.
+	out, err := os.Create(latestMainnetGenesisFilePath)
+	if err != nil {
+		return fmt.Errorf("failed to create genesis file: %w", err)
+	}
+	defer out.Close()
+
+	// Write the body to the file.
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to copy genesis file: %w", err)
 	}
 	return nil
 }

go/storage/mkvs/db/badger/testdata/case-chunkrestore.json

@@ -1,5 +1,5 @@
 {
-	"pending_root": "E/ElH8YWL7cSjeiWeSuiIIvGNic//du4F3JxOUihdc0=",
+	"pending_root": "13f1251fc6162fb7128de896792ba2208bc636273ffddbb81772713948a175cd",
 	"long_roots": null,
 	"pending_version": 2,
 	"entries": [
@@ -106,4 +106,4 @@
 			"delete": false
 		}
 	]
-}
+}

go/storage/mkvs/db/badger/testdata/case-long-toempty.json

@@ -1,10 +1,10 @@
 {
-	"pending_root": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+	"pending_root": "0000000000000000000000000000000000000000000000000000000000000000",
 	"long_roots": [
-		"u2FcysON2z8WXzHpyrFsvhDLOEDWtt8ch+9DZOVEJuo=",
-		"Qyp9ISN2PZFq8mZZn6r9wocwzTQMh1w8jCPzMXW2jyI=",
-		"31kArL/8JmlyoNWonGf15MtNorbXxjkncEtNKkjFqKU=",
-		"RwG1WgZ9GgPS4fUdED8uF2zWpdqS5zQFhtGkZAVAIWg="
+		"bb615ccac38ddb3f165f31e9cab16cbe10cb3840d6b6df1c87ef4364e54426ea",
+		"432a7d2123763d916af266599faafdc28730cd340c875c3c8c23f33175b68f22",
+		"df5900acbffc266972a0d5a89c67f5e4cb4da2b6d7c63927704b4d2a48c5a8a5",
+		"4701b55a067d1a03d2e1f51d103f2e176cd6a5da92e7340586d1a46405402168"
 	],
 	"pending_version": 0,
 	"entries": [
@@ -315,4 +315,4 @@
 			"delete": false
 		}
 	]
-}
+}