Merge pull request #4045 from oasisprotocol/ptrus/stable/21.2.x/backp…

…ort-multi

[BACKPORT/21.2.x] Multiple dependency updates and fixes

.buildkite/code.pipeline.yml

@@ -70,6 +70,18 @@ retry: &retry_agent_failure
       limit: 2
 
 steps:
+  # Require explicit confirmation for some PRs to avoid CI spam.
+  - block: Confirm CI run
+    prompt: Run CI for this pull request?
+    blocked_state: running
+    if: |
+      build.pull_request.id != null &&
+      !(build.pull_request.labels includes "s:ready-ci") &&
+      (
+        build.pull_request.draft ||
+        !(build.creator.teams includes "everyone")
+      )
+
   ###########
   # Lint jobs
   ###########
@@ -230,7 +242,30 @@ steps:
   ###########################
   # E2E test jobs - intel-sgx
   ###########################
-  - label: E2E tests - intel-sgx
+  - label: E2E tests - intel-sgx (basic)
+    branches: "!master !stable/*"
+    timeout_in_minutes: 40
+    command:
+      - .buildkite/scripts/download_e2e_test_artifacts.sh
+      # Only run runtime scenarios as others do not use SGX.
+      - .buildkite/scripts/test_e2e.sh --scenario e2e/runtime/runtime
+    artifact_paths:
+      - coverage-merged-e2e-*.txt
+      - /tmp/e2e/**/*.log
+    env:
+      OASIS_E2E_COVERAGE: enable
+      TEST_BASE_DIR: /tmp
+      # libp2p logging.
+      IPFS_LOGGING: debug
+    agents:
+      queue: intel-sgx
+    retry:
+      <<: *retry_agent_failure
+    plugins:
+      <<: *docker_plugin_sgx
+
+  - label: E2E tests - intel-sgx (full)
+    branches: master stable/*
     parallelism: 5
     timeout_in_minutes: 40
     command:
@@ -253,10 +288,11 @@ steps:
     plugins:
       <<: *docker_plugin_sgx
 
-  ###############################
-  # E2E test - intel-sgx with IAS
-  ###############################
+  ################################################
+  # E2E test - intel-sgx with IAS (only on master)
+  ################################################
   - label: E2E tests - intel-sgx - IAS
+    branches: master stable/*
     timeout_in_minutes: 15
     command:
       - .buildkite/scripts/sgx_ias_tests.sh
@@ -302,7 +338,7 @@ steps:
   ####################################
 
   - label: Coverage Rust crates
-    branches: master
+    branches: master stable/*
     command:
       # Build storage interoperability test helpers first.
       - make build-helpers

.changelog/4024.feature.2.md

@@ -0,0 +1,4 @@
+go/oasis-net-runner: allow setting runtime provisioner
+
+On systems that do not support bwrap, the runtime provisioner can be
+set to `unconfined`.

.changelog/4024.feature.3.md

@@ -0,0 +1,4 @@
+go/oasis-net-runner: allow running without a keymanager
+
+This feature makes it easier to run debug runtimes that are not (yet)
+associated with a keymanager.

.changelog/4024.feature.md

@@ -0,0 +1 @@
+runtime-loader: allow bulding for non-SGX use on platforms that aren't Linux

.changelog/4037.bugfix.md

@@ -0,0 +1,8 @@
+badger/migration: keep only last version when migrating non-managed DB
+
+Even though oasis-core always configures non-managed badger databases to keep
+a single version of keys, some databases in the wild contain multiple versions
+of the same key.
+
+Skip migrating more than the latest version when migrating non-managed
+badger databases.

.changelog/4037.cfg.md

@@ -0,0 +1,5 @@
+badger/migration: `badger.migrate.num_go_routines` flag
+
+The flag enables controlling the number of go routines badger uses when
+doing the v2 -> v3 migration. Use the flag to lower memory pressure during
+the migration, by lowering the amount of workers to e.g. 1.

.changelog/4040.feature.md

@@ -0,0 +1,12 @@
+Build `oasis-node` with `jemalloc` tag (used by BadgerDB)
+
+In BadgerDB V3 using `jemalloc` seems to be recommended and better supported
+option ([1], [2]). Based on testing using `jemalloc` reduces BadgerDB memory
+usage.
+
+To build `oasis-node` without `jemalloc` requirement, set the
+`OASIS_BADGER_NO_JEMALLOC="1"` environment variable before invoking the
+makefile.
+
+[1]: https://dgraph.io/blog/post/manual-memory-management-golang-jemalloc/
+[2]: https://discuss.dgraph.io/t/memory-issue-during-stream-operation/13033

.changelog/4041.cfg.md

@@ -0,0 +1,5 @@
+badger/migration: `badger.migrate.num_go_routines` flag
+
+The flag enables controlling the number of go routines badger uses when
+doing the v2 -> v3 migration. Use the flag to lower memory pressure during
+the migration, by lowering the amount of workers to e.g. 1.

.changelog/4042.trivial.md

@@ -0,0 +1 @@
+go/oasis-net-runner: Fix keymanager use determination

.github/workflows/ci-reproducibility.yml

@@ -17,6 +17,8 @@ env:
   GORELEASER_URL_PREFIX: https://github.com/goreleaser/goreleaser/releases/download/
   GORELEASER_VERSION: 0.152.0
   CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail
+  JEMALLOC_VERSION: 5.2.1
+  JEMALLOC_CHECKSUM: 34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
 
 jobs:
 
@@ -49,6 +51,14 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install make libseccomp-dev protobuf-compiler
+      - name: Install jemalloc
+        run: |
+          wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
+          echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c
+          tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION}
+          ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto'
+          make
+          sudo make install
       - name: Install GoReleaser
         run: |
           cd $(mktemp --directory /tmp/goreleaser.XXXXX)

.github/workflows/release-dev.yml

@@ -14,6 +14,8 @@ env:
   GORELEASER_URL_PREFIX: https://github.com/goreleaser/goreleaser/releases/download/
   GORELEASER_VERSION: 0.152.0
   CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail
+  JEMALLOC_VERSION: 5.2.1
+  JEMALLOC_CHECKSUM: 34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
 
 jobs:
 
@@ -40,6 +42,14 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install make libseccomp-dev protobuf-compiler
+      - name: Install jemalloc
+        run: |
+          wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
+          echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c
+          tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION}
+          ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto'
+          make
+          sudo make install
       - name: Install GoReleaser
         run: |
           cd $(mktemp --directory /tmp/goreleaser.XXXXX)

.github/workflows/release.yml

@@ -16,6 +16,8 @@ env:
   GORELEASER_URL_PREFIX: https://github.com/goreleaser/goreleaser/releases/download/
   GORELEASER_VERSION: 0.152.0
   CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail
+  JEMALLOC_VERSION: 5.2.1
+  JEMALLOC_CHECKSUM: 34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
 
 jobs:
 
@@ -42,6 +44,14 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install make libseccomp-dev protobuf-compiler
+      - name: Install jemalloc
+        run: |
+          wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
+          echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c
+          tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION}
+          ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto'
+          make
+          sudo make install
       - name: Install GoReleaser
         run: |
           cd $(mktemp --directory /tmp/goreleaser.XXXXX)

.goreleaser.yml

@@ -18,6 +18,10 @@ builds:
     dir: go/
     flags:
       - -trimpath
+      # Build oasis-node with jemalloc tag (used by badgerdb).
+      # TODO: Use 'tags' attribute when GoReleaser is udpated to newer version:
+      # https://github.com/goreleaser/goreleaser/pull/2268
+      - -tags=jemalloc
     ldflags:
       # NOTE: At the moment, GoReleaser produces different binaries when
       # releases are built from different git paths, unless -buildid= is added

docker/development/Dockerfile

@@ -10,6 +10,8 @@ ARG GOLANGCILINT_VERSION=1.28.2
 ARG GOCOVMERGE_VERSION=b5bfa59ec0adc420475f97f89b58045c721d761c
 ARG GOFUMPT_VERSION=abc0db2c416aca0f60ea33c23c76665f6e7ba0b6
 ARG RUST_NIGHTLY_VERSION=2021-05-20
+ARG JEMALLOC_VERSION=5.2.1
+ARG JEMALLOC_CHECKSUM=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -72,6 +74,14 @@ RUN wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz && \
     GO111MODULE=on go get mvdan.cc/gofumpt@${GOFUMPT_VERSION} && \
     GO111MODULE=on go get mvdan.cc/gofumpt/gofumports@${GOFUMPT_VERSION}
 
+# Install jemalloc (used by badgerdb).
+RUN wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 && \
+    echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c && \
+    tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION} && \
+    ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto' && \
+    make && make install && \
+    cd .. && rm jemalloc.tar.bz2 && rm -rf jemalloc-${JEMALLOC_VERSION}
+
 # Install bubblewrap (we need at least version 0.3.3 which is not available for 18.04).
 RUN wget http://archive.ubuntu.com/ubuntu/pool/main/b/bubblewrap/bubblewrap_0.4.1-1_amd64.deb && \
     echo '25de452f209e4fdb4b009851c33ca9a0269ebf0b92f4bd9b86186480592cc3e2 bubblewrap_0.4.1-1_amd64.deb' | sha256sum -c && \

docs/setup/building.md

@@ -17,6 +17,15 @@ export OASIS_UNSAFE_SKIP_KM_POLICY="1"
 make
 ```
 
+To build BadgerDB without `jemalloc` support (and avoid installing `jemalloc`
+on your system), set
+
+```
+export OASIS_BADGER_NO_JEMALLOC="1"
+```
+
+Not using `jemalloc` is fine for development purposes.
+
 This will build all the required parts (build tools, Oasis node, runtime
 libraries, runtime loader, key manager and test runtimes). The AVR and KM flags
 are supported on production SGX systems only and these features must be disabled

docs/setup/prerequisites.md

@@ -169,6 +169,28 @@ Core:
   committed for convenience.  Installing protoc-gen-go is only required if you
   are a developer making changes to protobuf definitions used by Go._
 
+* (**OPTIONAL**) [jemalloc] (version 5.2.1, built with `'je_'` jemalloc-prefix)
+
+  Alternatively set `OASIS_BADGER_NO_JEMALLOC="1"` environment variable when
+  building `oasis-node` code, to build BadgerDB without `jemalloc` support.
+
+  Download and install `jemalloc` with:
+
+  <!-- markdownlint-disable line-length -->
+  ```
+  export JEMALLOC_VERSION=5.2.1
+  export JEMALLOC_CHECKSUM=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
+  wget -O jemalloc.tar.bz2 "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2"
+  # Ensure checksum matches.
+  echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c
+  tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner
+  cd jemalloc-${JEMALLOC_VERSION}
+  ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto'
+  make
+  make install
+  ```
+  <!-- markdownlint-enable line-length -->
+
 In the following instructions, the top-level directory is the directory
 where the code has been checked out.
 
@@ -192,6 +214,7 @@ where the code has been checked out.
 [Fortanix Rust EDP]: https://edp.fortanix.com
 [gofumpt and gofumports]: https://github.com/mvdan/gofumpt
 [protoc-gen-go]: https://github.com/golang/protobuf
+[jemalloc]: https://github.com/jemalloc/jemalloc
 
 ## Using the Development Docker Image
 

go/Makefile

@@ -5,6 +5,11 @@ ifneq ($(GOLDFLAGS),)
 	GO_EXTRA_FLAGS += -ldflags $(GOLDFLAGS)
 endif
 
+# Build code with jemalloc tag unless explicitly disabled (used by badgerdb).
+ifneq ($(OASIS_BADGER_NO_JEMALLOC), 1)
+	GO_EXTRA_FLAGS += -tags jemalloc
+endif
+
 # Set all target as the default target.
 all: build
 

go/common/badger/helpers.go

@@ -5,14 +5,14 @@ import (
 	"bufio"
 	"fmt"
 	"io"
-	"math"
 	"os"
 	"strings"
 	"sync"
 	"time"
 
 	badgerV2 "github.com/dgraph-io/badger/v2"
 	"github.com/dgraph-io/badger/v3"
+	"github.com/spf13/viper"
 
 	"github.com/oasisprotocol/oasis-core/go/common/logging"
 )
@@ -187,9 +187,15 @@ func migrateDatabase(opts badger.Options, managed bool) error {
 		openFnV3 = badger.OpenManaged
 	}
 
+	// All non-managed databases used by oasis-core are configured to keep only one version.
+	// Part of the migrator assumes this, therefore fail in case this is not the case.
+	if !managed && opts.NumVersionsToKeep != 1 {
+		return fmt.Errorf("migration assumes 1 version to keep for non-managed databases")
+	}
+
 	// Open the database as Badger v2.
 	optsV2 := badgerV2.DefaultOptions(opts.Dir)
-	optsV2 = optsV2.WithNumVersionsToKeep(math.MaxInt32)
+	optsV2 = optsV2.WithNumVersionsToKeep(opts.NumVersionsToKeep)
 	optsV2 = optsV2.WithLogger(nil)
 
 	dbV2, err := openFnV2(optsV2)
@@ -199,9 +205,10 @@ func migrateDatabase(opts badger.Options, managed bool) error {
 	defer dbV2.Close()
 
 	// Open the destination database as Badger v3.
-	optsV3 := badger.DefaultOptions(temporaryDbName)
-	optsV3 = optsV3.WithNumVersionsToKeep(math.MaxInt32)
-	optsV3 = optsV3.WithLogger(NewLogAdapter(logger))
+	optsV3 := opts
+	optsV3 = optsV3.WithDir(temporaryDbName)
+	optsV3 = optsV3.WithValueDir(temporaryDbName)
+	optsV3 = optsV3.WithNumGoroutines(viper.GetInt(cfgMigrateNumGoRoutines))
 
 	dbV3, err := openFnV3(optsV3)
 	if err != nil {

go/common/badger/migrate.go

@@ -10,8 +10,17 @@ import (
 	badgerV2 "github.com/dgraph-io/badger/v2"
 	"github.com/dgraph-io/badger/v2/pb"
 	"github.com/golang/protobuf/proto" //nolint: staticcheck
+	flag "github.com/spf13/pflag"
+	"github.com/spf13/viper"
 )
 
+const (
+	cfgMigrateNumGoRoutines = "badger.migrate.num_go_routines"
+)
+
+// MigrationFlags has the migration flags.
+var MigrationFlags = flag.NewFlagSet("", flag.ContinueOnError)
+
 // Adapted from Badger v2 which is Copyright 2017 Dgraph Labs, Inc. and Contributors, released
 // under the Apache-2 license.
 
@@ -23,6 +32,7 @@ func backup(db *badgerV2.DB, w io.Writer, managed bool) (uint64, error) {
 	case false:
 		stream = db.NewStream()
 	}
+	stream.NumGo = viper.GetInt(cfgMigrateNumGoRoutines)
 
 	stream.LogPrefix = "migration"
 	stream.KeyToList = func(key []byte, itr *badgerV2.Iterator) (*pb.KVList, error) {
@@ -57,6 +67,14 @@ func backup(db *badgerV2.DB, w io.Writer, managed bool) (uint64, error) {
 				Meta:      []byte{meta},
 			}
 			list.Kv = append(list.Kv, kv)
+
+			if !managed {
+				// Migrate only last version of the key in case this is a non-managed database.
+				// All non-managed oasis-core databases are configured to only keep one version,
+				// but due to what looks like a badger bug, it can happen that a key can have
+				// multiple historical versions in the database.
+				return list, nil
+			}
 		}
 		return list, nil
 	}
@@ -84,3 +102,8 @@ func backup(db *badgerV2.DB, w io.Writer, managed bool) (uint64, error) {
 	}
 	return maxVersion, nil
 }
+
+func init() {
+	MigrationFlags.Int(cfgMigrateNumGoRoutines, 8, "number of go routines to use when migrating (useful to lower memory pressure during migration)")
+	_ = viper.BindPFlags(MigrationFlags)
+}