From 893c4c6b5446d1398930ed0ec98d1c0e78563d6a Mon Sep 17 00:00:00 2001 From: Jernej Kos Date: Wed, 16 Jun 2021 14:23:43 +0200 Subject: [PATCH 01/12] ci: Only run the full SGX suite on master/stable --- .buildkite/code.pipeline.yml | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/.buildkite/code.pipeline.yml b/.buildkite/code.pipeline.yml index c232d85c69f..3969d9c49c7 100644 --- a/.buildkite/code.pipeline.yml +++ b/.buildkite/code.pipeline.yml @@ -230,7 +230,30 @@ steps: ########################### # E2E test jobs - intel-sgx ########################### - - label: E2E tests - intel-sgx + - label: E2E tests - intel-sgx (basic) + branches: "!master !stable/*" + timeout_in_minutes: 40 + command: + - .buildkite/scripts/download_e2e_test_artifacts.sh + # Only run runtime scenarios as others do not use SGX. + - .buildkite/scripts/test_e2e.sh --scenario e2e/runtime/runtime + artifact_paths: + - coverage-merged-e2e-*.txt + - /tmp/e2e/**/*.log + env: + OASIS_E2E_COVERAGE: enable + TEST_BASE_DIR: /tmp + # libp2p logging. + IPFS_LOGGING: debug + agents: + queue: intel-sgx + retry: + <<: *retry_agent_failure + plugins: + <<: *docker_plugin_sgx + + - label: E2E tests - intel-sgx (full) + branches: master stable/* parallelism: 5 timeout_in_minutes: 40 command: @@ -253,10 +276,11 @@ steps: plugins: <<: *docker_plugin_sgx - ############################### - # E2E test - intel-sgx with IAS - ############################### + ################################################ + # E2E test - intel-sgx with IAS (only on master) + ################################################ - label: E2E tests - intel-sgx - IAS + branches: master stable/* timeout_in_minutes: 15 command: - .buildkite/scripts/sgx_ias_tests.sh @@ -302,7 +326,7 @@ steps: #################################### - label: Coverage Rust crates - branches: master + branches: master stable/* command: # Build storage interoperability test helpers first. - make build-helpers From 285d3aee8df3153886a01610fc48d4c94b9ae3b8 Mon Sep 17 00:00:00 2001 From: Jernej Kos Date: Wed, 16 Jun 2021 14:49:44 +0200 Subject: [PATCH 02/12] ci: Require explicit confirmation for some PRs --- .buildkite/code.pipeline.yml | 12 ++++++++++++ .changelog/4039.trivial.md | 0 2 files changed, 12 insertions(+) create mode 100644 .changelog/4039.trivial.md diff --git a/.buildkite/code.pipeline.yml b/.buildkite/code.pipeline.yml index 3969d9c49c7..a661a5cbd59 100644 --- a/.buildkite/code.pipeline.yml +++ b/.buildkite/code.pipeline.yml @@ -70,6 +70,18 @@ retry: &retry_agent_failure limit: 2 steps: + # Require explicit confirmation for some PRs to avoid CI spam. + - block: Confirm CI run + prompt: Run CI for this pull request? + blocked_state: running + if: | + build.pull_request.id != null && + !(build.pull_request.labels includes "s:ready-ci") && + ( + build.pull_request.draft || + !(build.creator.teams includes "everyone") + ) + ########### # Lint jobs ########### diff --git a/.changelog/4039.trivial.md b/.changelog/4039.trivial.md new file mode 100644 index 00000000000..e69de29bb2d From 62ba2c752b604d5624ea9d7b13cb909f85cf2b70 Mon Sep 17 00:00:00 2001 From: ptrus Date: Wed, 16 Jun 2021 13:19:52 +0200 Subject: [PATCH 03/12] badger/migration: keep only last version when migrating non-managed DB Even though oasis-core always configures non-managed badger databases to keep a single version of keys, some databases in the wild contain multiple versions of the same key. Skip migrating more than the latest version when migrating non-managed badger databases. --- .changelog/4037.bugfix.md | 8 ++++++++ go/common/badger/helpers.go | 15 ++++++++++----- go/common/badger/migrate.go | 8 ++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 .changelog/4037.bugfix.md diff --git a/.changelog/4037.bugfix.md b/.changelog/4037.bugfix.md new file mode 100644 index 00000000000..01e774f259e --- /dev/null +++ b/.changelog/4037.bugfix.md @@ -0,0 +1,8 @@ +badger/migration: keep only last version when migrating non-managed DB + +Even though oasis-core always configures non-managed badger databases to keep +a single version of keys, some databases in the wild contain multiple versions +of the same key. + +Skip migrating more than the latest version when migrating non-managed +badger databases. diff --git a/go/common/badger/helpers.go b/go/common/badger/helpers.go index 2d5c0381991..37317a76c21 100644 --- a/go/common/badger/helpers.go +++ b/go/common/badger/helpers.go @@ -5,7 +5,6 @@ import ( "bufio" "fmt" "io" - "math" "os" "strings" "sync" @@ -187,9 +186,15 @@ func migrateDatabase(opts badger.Options, managed bool) error { openFnV3 = badger.OpenManaged } + // All non-managed databases used by oasis-core are configured to keep only one version. + // Part of the migrator assumes this, therefore fail in case this is not the case. + if !managed && opts.NumVersionsToKeep != 1 { + return fmt.Errorf("migration assumes 1 version to keep for non-managed databases") + } + // Open the database as Badger v2. optsV2 := badgerV2.DefaultOptions(opts.Dir) - optsV2 = optsV2.WithNumVersionsToKeep(math.MaxInt32) + optsV2 = optsV2.WithNumVersionsToKeep(opts.NumVersionsToKeep) optsV2 = optsV2.WithLogger(nil) dbV2, err := openFnV2(optsV2) @@ -199,9 +204,9 @@ func migrateDatabase(opts badger.Options, managed bool) error { defer dbV2.Close() // Open the destination database as Badger v3. - optsV3 := badger.DefaultOptions(temporaryDbName) - optsV3 = optsV3.WithNumVersionsToKeep(math.MaxInt32) - optsV3 = optsV3.WithLogger(NewLogAdapter(logger)) + optsV3 := opts + optsV3 = optsV3.WithDir(temporaryDbName) + optsV3 = optsV3.WithValueDir(temporaryDbName) dbV3, err := openFnV3(optsV3) if err != nil { diff --git a/go/common/badger/migrate.go b/go/common/badger/migrate.go index 3c0d4a51571..d343213b4b2 100644 --- a/go/common/badger/migrate.go +++ b/go/common/badger/migrate.go @@ -57,6 +57,14 @@ func backup(db *badgerV2.DB, w io.Writer, managed bool) (uint64, error) { Meta: []byte{meta}, } list.Kv = append(list.Kv, kv) + + if !managed { + // Migrate only last version of the key in case this is a non-managed database. + // All non-managed oasis-core databases are configured to only keep one version, + // but due to what looks like a badger bug, it can happen that a key can have + // multiple historical versions in the database. + return list, nil + } } return list, nil } From aca9231c274d76ef7b76801452798d520f3d9b85 Mon Sep 17 00:00:00 2001 From: ptrus Date: Wed, 16 Jun 2021 17:51:15 +0200 Subject: [PATCH 04/12] badger/migrate: add migrate num_go_routines flag --- go/common/badger/helpers.go | 2 ++ go/common/badger/migrate.go | 15 +++++++++++++++ go/oasis-node/cmd/node/node.go | 2 ++ 3 files changed, 19 insertions(+) diff --git a/go/common/badger/helpers.go b/go/common/badger/helpers.go index 37317a76c21..8fe991ecd07 100644 --- a/go/common/badger/helpers.go +++ b/go/common/badger/helpers.go @@ -12,6 +12,7 @@ import ( badgerV2 "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/badger/v3" + "github.com/spf13/viper" "github.com/oasisprotocol/oasis-core/go/common/logging" ) @@ -207,6 +208,7 @@ func migrateDatabase(opts badger.Options, managed bool) error { optsV3 := opts optsV3 = optsV3.WithDir(temporaryDbName) optsV3 = optsV3.WithValueDir(temporaryDbName) + optsV3 = optsV3.WithNumGoroutines(viper.GetInt(cfgMigrateNumGoRoutines)) dbV3, err := openFnV3(optsV3) if err != nil { diff --git a/go/common/badger/migrate.go b/go/common/badger/migrate.go index d343213b4b2..0dd965e106b 100644 --- a/go/common/badger/migrate.go +++ b/go/common/badger/migrate.go @@ -10,8 +10,17 @@ import ( badgerV2 "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/badger/v2/pb" "github.com/golang/protobuf/proto" //nolint: staticcheck + flag "github.com/spf13/pflag" + "github.com/spf13/viper" ) +const ( + cfgMigrateNumGoRoutines = "badger.migrate.num_go_routines" +) + +// MigrationFlags has the migration flags. +var MigrationFlags = flag.NewFlagSet("", flag.ContinueOnError) + // Adapted from Badger v2 which is Copyright 2017 Dgraph Labs, Inc. and Contributors, released // under the Apache-2 license. @@ -23,6 +32,7 @@ func backup(db *badgerV2.DB, w io.Writer, managed bool) (uint64, error) { case false: stream = db.NewStream() } + stream.NumGo = viper.GetInt(cfgMigrateNumGoRoutines) stream.LogPrefix = "migration" stream.KeyToList = func(key []byte, itr *badgerV2.Iterator) (*pb.KVList, error) { @@ -92,3 +102,8 @@ func backup(db *badgerV2.DB, w io.Writer, managed bool) (uint64, error) { } return maxVersion, nil } + +func init() { + MigrationFlags.Int(cfgMigrateNumGoRoutines, 8, "number of go routines to use when migrating (useful to lower memory pressure during migration)") + _ = viper.BindPFlags(MigrationFlags) +} diff --git a/go/oasis-node/cmd/node/node.go b/go/oasis-node/cmd/node/node.go index 45d7b86e895..f2ee50d595c 100644 --- a/go/oasis-node/cmd/node/node.go +++ b/go/oasis-node/cmd/node/node.go @@ -14,6 +14,7 @@ import ( beacon "github.com/oasisprotocol/oasis-core/go/beacon/api" "github.com/oasisprotocol/oasis-core/go/common" + "github.com/oasisprotocol/oasis-core/go/common/badger" "github.com/oasisprotocol/oasis-core/go/common/crash" "github.com/oasisprotocol/oasis-core/go/common/crypto/signature" "github.com/oasisprotocol/oasis-core/go/common/grpc" @@ -810,6 +811,7 @@ func init() { workerSentry.Flags, workerConsensusRPC.Flags, crash.InitFlags(), + badger.MigrationFlags, } { Flags.AddFlagSet(v) } From e2dbfdc2652c10c69b5594fe0579d842ba7b4640 Mon Sep 17 00:00:00 2001 From: ptrus Date: Wed, 16 Jun 2021 19:31:55 +0200 Subject: [PATCH 05/12] badger/migrate: num_go_routines flag fragment --- .changelog/4037.cfg.md | 5 +++++ .changelog/4041.cfg.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changelog/4037.cfg.md create mode 100644 .changelog/4041.cfg.md diff --git a/.changelog/4037.cfg.md b/.changelog/4037.cfg.md new file mode 100644 index 00000000000..0e37ac1a963 --- /dev/null +++ b/.changelog/4037.cfg.md @@ -0,0 +1,5 @@ +badger/migration: `badger.migrate.num_go_routines` flag + +The flag enables controlling the number of go routines badger uses when +doing the v2 -> v3 migration. Use the flag to lower memory pressure during +the migration, by lowering the amount of workers to e.g. 1. diff --git a/.changelog/4041.cfg.md b/.changelog/4041.cfg.md new file mode 100644 index 00000000000..0e37ac1a963 --- /dev/null +++ b/.changelog/4041.cfg.md @@ -0,0 +1,5 @@ +badger/migration: `badger.migrate.num_go_routines` flag + +The flag enables controlling the number of go routines badger uses when +doing the v2 -> v3 migration. Use the flag to lower memory pressure during +the migration, by lowering the amount of workers to e.g. 1. From 3675f0fa9889ef75b4f84e81be188683e6906d2e Mon Sep 17 00:00:00 2001 From: ptrus Date: Wed, 16 Jun 2021 15:21:40 +0200 Subject: [PATCH 06/12] Build BadgerDB with `jemalloc` tag --- .changelog/4040.feature.md | 12 ++++++++++++ .github/workflows/ci-reproducibility.yml | 10 ++++++++++ .github/workflows/release-dev.yml | 10 ++++++++++ .github/workflows/release.yml | 10 ++++++++++ .goreleaser.yml | 4 ++++ docker/development/Dockerfile | 10 ++++++++++ docs/setup/building.md | 9 +++++++++ docs/setup/prerequisites.md | 23 +++++++++++++++++++++++ go/Makefile | 5 +++++ 9 files changed, 93 insertions(+) create mode 100644 .changelog/4040.feature.md diff --git a/.changelog/4040.feature.md b/.changelog/4040.feature.md new file mode 100644 index 00000000000..8f7879136ea --- /dev/null +++ b/.changelog/4040.feature.md @@ -0,0 +1,12 @@ +Build `oasis-node` with `jemalloc` tag (used by BadgerDB) + +In BadgerDB V3 using `jemalloc` seems to be recommended and better supported +option ([1], [2]). Based on testing using `jemalloc` reduces BadgerDB memory +usage. + +To build `oasis-node` without `jemalloc` requirement, set the +`OASIS_BADGER_NO_JEMALLOC="1"` environment variable before invoking the +makefile. + +[1]: https://dgraph.io/blog/post/manual-memory-management-golang-jemalloc/ +[2]: https://discuss.dgraph.io/t/memory-issue-during-stream-operation/13033 diff --git a/.github/workflows/ci-reproducibility.yml b/.github/workflows/ci-reproducibility.yml index 5f0307d37fe..fb9866262e9 100644 --- a/.github/workflows/ci-reproducibility.yml +++ b/.github/workflows/ci-reproducibility.yml @@ -17,6 +17,8 @@ env: GORELEASER_URL_PREFIX: https://github.com/goreleaser/goreleaser/releases/download/ GORELEASER_VERSION: 0.152.0 CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail + JEMALLOC_VERSION: 5.2.1 + JEMALLOC_CHECKSUM: 34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jobs: @@ -49,6 +51,14 @@ jobs: run: | sudo apt-get update sudo apt-get install make libseccomp-dev protobuf-compiler + - name: Install jemalloc + run: | + wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 + echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c + tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION} + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto' + make + sudo make install - name: Install GoReleaser run: | cd $(mktemp --directory /tmp/goreleaser.XXXXX) diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index b5d7b9064c4..2cd6b69e707 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -14,6 +14,8 @@ env: GORELEASER_URL_PREFIX: https://github.com/goreleaser/goreleaser/releases/download/ GORELEASER_VERSION: 0.152.0 CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail + JEMALLOC_VERSION: 5.2.1 + JEMALLOC_CHECKSUM: 34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jobs: @@ -40,6 +42,14 @@ jobs: run: | sudo apt-get update sudo apt-get install make libseccomp-dev protobuf-compiler + - name: Install jemalloc + run: | + wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 + echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c + tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION} + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto' + make + sudo make install - name: Install GoReleaser run: | cd $(mktemp --directory /tmp/goreleaser.XXXXX) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4fbdfa8de04..de0b1dd671e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,6 +16,8 @@ env: GORELEASER_URL_PREFIX: https://github.com/goreleaser/goreleaser/releases/download/ GORELEASER_VERSION: 0.152.0 CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail + JEMALLOC_VERSION: 5.2.1 + JEMALLOC_CHECKSUM: 34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jobs: @@ -42,6 +44,14 @@ jobs: run: | sudo apt-get update sudo apt-get install make libseccomp-dev protobuf-compiler + - name: Install jemalloc + run: | + wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 + echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c + tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION} + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto' + make + sudo make install - name: Install GoReleaser run: | cd $(mktemp --directory /tmp/goreleaser.XXXXX) diff --git a/.goreleaser.yml b/.goreleaser.yml index d9b8833728d..07e7d15a47d 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -18,6 +18,10 @@ builds: dir: go/ flags: - -trimpath + # Build oasis-node with jemalloc tag (used by badgerdb). + # TODO: Use 'tags' attribute when GoReleaser is udpated to newer version: + # https://github.com/goreleaser/goreleaser/pull/2268 + - -tags=jemalloc ldflags: # NOTE: At the moment, GoReleaser produces different binaries when # releases are built from different git paths, unless -buildid= is added diff --git a/docker/development/Dockerfile b/docker/development/Dockerfile index 1fc6ce76c78..d4ead8bfcf7 100644 --- a/docker/development/Dockerfile +++ b/docker/development/Dockerfile @@ -10,6 +10,8 @@ ARG GOLANGCILINT_VERSION=1.28.2 ARG GOCOVMERGE_VERSION=b5bfa59ec0adc420475f97f89b58045c721d761c ARG GOFUMPT_VERSION=abc0db2c416aca0f60ea33c23c76665f6e7ba0b6 ARG RUST_NIGHTLY_VERSION=2021-05-20 +ARG JEMALLOC_VERSION=5.2.1 +ARG JEMALLOC_CHECKSUM=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 ARG DEBIAN_FRONTEND=noninteractive @@ -72,6 +74,14 @@ RUN wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz && \ GO111MODULE=on go get mvdan.cc/gofumpt@${GOFUMPT_VERSION} && \ GO111MODULE=on go get mvdan.cc/gofumpt/gofumports@${GOFUMPT_VERSION} +# Install jemalloc (used by badgerdb). +RUN wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 && \ + echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c && \ + tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION} && \ + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto' && \ + make && make install && \ + cd .. && rm jemalloc.tar.bz2 && rm -rf jemalloc-${JEMALLOC_VERSION} + # Install bubblewrap (we need at least version 0.3.3 which is not available for 18.04). RUN wget http://archive.ubuntu.com/ubuntu/pool/main/b/bubblewrap/bubblewrap_0.4.1-1_amd64.deb && \ echo '25de452f209e4fdb4b009851c33ca9a0269ebf0b92f4bd9b86186480592cc3e2 bubblewrap_0.4.1-1_amd64.deb' | sha256sum -c && \ diff --git a/docs/setup/building.md b/docs/setup/building.md index a8948635510..0489995117c 100644 --- a/docs/setup/building.md +++ b/docs/setup/building.md @@ -17,6 +17,15 @@ export OASIS_UNSAFE_SKIP_KM_POLICY="1" make ``` +To build BadgerDB without `jemalloc` support (and avoid installing `jemalloc` +on your system), set + +``` +export OASIS_BADGER_NO_JEMALLOC="1" +``` + +Not using `jemalloc` is fine for development purposes. + This will build all the required parts (build tools, Oasis node, runtime libraries, runtime loader, key manager and test runtimes). The AVR and KM flags are supported on production SGX systems only and these features must be disabled diff --git a/docs/setup/prerequisites.md b/docs/setup/prerequisites.md index c77b5a5d33d..4b55d0cc584 100644 --- a/docs/setup/prerequisites.md +++ b/docs/setup/prerequisites.md @@ -169,6 +169,28 @@ Core: committed for convenience. Installing protoc-gen-go is only required if you are a developer making changes to protobuf definitions used by Go._ +* (**OPTIONAL**) [jemalloc] (version 5.2.1, built with `'je_'` jemalloc-prefix) + + Alternatively set `OASIS_BADGER_NO_JEMALLOC="1"` environment variable when + building `oasis-node` code, to build BadgerDB without `jemalloc` support. + + Download and install `jemalloc` with: + + + ``` + export JEMALLOC_VERSION=5.2.1 + export JEMALLOC_CHECKSUM=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 + wget -O jemalloc.tar.bz2 "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" + # Ensure checksum matches. + echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c + tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner + cd jemalloc-${JEMALLOC_VERSION} + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto' + make + make install + ``` + + In the following instructions, the top-level directory is the directory where the code has been checked out. @@ -192,6 +214,7 @@ where the code has been checked out. [Fortanix Rust EDP]: https://edp.fortanix.com [gofumpt and gofumports]: https://github.com/mvdan/gofumpt [protoc-gen-go]: https://github.com/golang/protobuf +[jemalloc]: https://github.com/jemalloc/jemalloc ## Using the Development Docker Image diff --git a/go/Makefile b/go/Makefile index e2207c2e1a4..bb54d7444a2 100644 --- a/go/Makefile +++ b/go/Makefile @@ -5,6 +5,11 @@ ifneq ($(GOLDFLAGS),) GO_EXTRA_FLAGS += -ldflags $(GOLDFLAGS) endif +# Build code with jemalloc tag unless explicitly disabled (used by badgerdb). +ifneq ($(OASIS_BADGER_NO_JEMALLOC), 1) + GO_EXTRA_FLAGS += -tags jemalloc +endif + # Set all target as the default target. all: build From db8aa23ea502131241d26b863f848b970a6335c3 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Sat, 12 Jun 2021 13:03:07 -0500 Subject: [PATCH 07/12] runtime-loader: compile on macos --- runtime-loader/Cargo.toml | 8 +++++--- runtime-loader/bin/main.rs | 7 ++++++- runtime-loader/src/lib.rs | 9 ++++----- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/runtime-loader/Cargo.toml b/runtime-loader/Cargo.toml index 4cccae95ef4..9763ea757b4 100644 --- a/runtime-loader/Cargo.toml +++ b/runtime-loader/Cargo.toml @@ -5,14 +5,16 @@ authors = ["Oasis Protocol Foundation "] edition = "2018" [dependencies] -aesm-client = { version = "0.5.3", features = ["sgxs"] } -enclave-runner = "0.4.0" -sgxs-loaders = "0.3.1" clap = "2.29.1" failure = "0.1.5" futures = { version = "0.3.7", features = ["compat", "io-compat"] } tokio = { version = "0.2", features = ["full"] } +[target.'cfg(target_os = "linux")'.dependencies] +aesm-client = { version = "0.5.3", features = ["sgxs"] } +enclave-runner = "0.4.0" +sgxs-loaders = "0.3.1" + [[bin]] name = "oasis-core-runtime-loader" path ="bin/main.rs" diff --git a/runtime-loader/bin/main.rs b/runtime-loader/bin/main.rs index 0887c6344b7..c11c2b1b8ff 100644 --- a/runtime-loader/bin/main.rs +++ b/runtime-loader/bin/main.rs @@ -5,7 +5,9 @@ use std::path::Path; use clap::{App, Arg}; -use oasis_core_runtime_loader::{ElfLoader, Loader, SgxsLoader}; +#[cfg(target_os = "linux")] +use oasis_core_runtime_loader::SgxsLoader; +use oasis_core_runtime_loader::{ElfLoader, Loader}; fn main() { let matches = App::new("Oasis runtime loader") @@ -52,7 +54,10 @@ fn main() { // Create appropriate loader and run the runtime. let loader: Box = match mode { + #[cfg(target_os = "linux")] "sgxs" => Box::new(SgxsLoader), + #[cfg(not(target_os = "linux"))] + "sgxs" => panic!("SGXS loader is only supported on Linux"), "elf" => Box::new(ElfLoader), _ => panic!("Invalid runtime type specified"), }; diff --git a/runtime-loader/src/lib.rs b/runtime-loader/src/lib.rs index 098b0b4179f..456ecfbf200 100644 --- a/runtime-loader/src/lib.rs +++ b/runtime-loader/src/lib.rs @@ -1,10 +1,7 @@ //! Oasis runtime loader. -extern crate aesm_client; -extern crate enclave_runner; -extern crate failure; -extern crate sgxs_loaders; pub mod elf; +#[cfg(target_os = "linux")] pub mod sgxs; use failure::Fallible; @@ -21,4 +18,6 @@ pub trait Loader { } // Re-exports. -pub use self::{elf::ElfLoader, sgxs::SgxsLoader}; +pub use elf::ElfLoader; +#[cfg(target_os = "linux")] +pub use sgxs::SgxsLoader; From 4ed6f5b1112e9eb6a307dab3e88b93bcb2499963 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Sat, 12 Jun 2021 17:44:35 -0500 Subject: [PATCH 08/12] go/oasis-test-runner: add runtime provisioner cfgs --- go/oasis-net-runner/fixtures/default.go | 16 +++++++---- go/oasis-test-runner/oasis/client.go | 33 ++++++++++++++-------- go/oasis-test-runner/oasis/fixture.go | 18 ++++++++---- go/oasis-test-runner/oasis/keymanager.go | 35 ++++++++++++++---------- 4 files changed, 65 insertions(+), 37 deletions(-) diff --git a/go/oasis-net-runner/fixtures/default.go b/go/oasis-net-runner/fixtures/default.go index f4c33f67d23..e869f0ff248 100644 --- a/go/oasis-net-runner/fixtures/default.go +++ b/go/oasis-net-runner/fixtures/default.go @@ -30,6 +30,7 @@ const ( cfgNumEntities = "fixture.default.num_entities" cfgRuntimeID = "fixture.default.runtime.id" cfgRuntimeBinary = "fixture.default.runtime.binary" + cfgRuntimeProvisioner = "fixture.default.runtime.provisioner" cfgRuntimeGenesisState = "fixture.default.runtime.genesis_state" cfgRuntimeLoader = "fixture.default.runtime.loader" cfgSetupRuntimes = "fixture.default.setup_runtimes" @@ -105,8 +106,12 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) { fixture.Entities = append(fixture.Entities, oasis.EntityCfg{}) } + runtimeProvisioner := viper.GetString(cfgRuntimeProvisioner) + // Always run a client node. - fixture.Clients = []oasis.ClientFixture{{}} + fixture.Clients = []oasis.ClientFixture{{ + RuntimeProvisioner: runtimeProvisioner, + }} if viper.GetBool(cfgSetupRuntimes) { fixture.Runtimes = []oasis.RuntimeFixture{ @@ -129,15 +134,15 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) { {Runtime: 0, Serial: 1}, } fixture.Keymanagers = []oasis.KeymanagerFixture{ - {Runtime: 0, Entity: 1}, + {Runtime: 0, Entity: 1, RuntimeProvisioner: runtimeProvisioner}, } fixture.StorageWorkers = []oasis.StorageWorkerFixture{ {Backend: "badger", Entity: 1}, } fixture.ComputeWorkers = []oasis.ComputeWorkerFixture{ - {Entity: 1, Runtimes: []int{}}, - {Entity: 1, Runtimes: []int{}}, - {Entity: 1, Runtimes: []int{}}, + {Entity: 1, Runtimes: []int{}, RuntimeProvisioner: runtimeProvisioner}, + {Entity: 1, Runtimes: []int{}, RuntimeProvisioner: runtimeProvisioner}, + {Entity: 1, Runtimes: []int{}, RuntimeProvisioner: runtimeProvisioner}, } var runtimeIDs []common.Namespace @@ -215,6 +220,7 @@ func init() { DefaultFixtureFlags.String(cfgNodeBinary, "oasis-node", "path to the oasis-node binary") DefaultFixtureFlags.StringSlice(cfgRuntimeID, []string{"8000000000000000000000000000000000000000000000000000000000000000"}, "runtime ID") DefaultFixtureFlags.StringSlice(cfgRuntimeBinary, []string{"simple-keyvalue"}, "path to the runtime binary") + DefaultFixtureFlags.String(cfgRuntimeProvisioner, "sandboxed", "the runtime provisioner: mock, unconfined, or sandboxed") // []string{""} as default doesn't work and ends up as an empty slice. DefaultFixtureFlags.StringSlice(cfgRuntimeGenesisState, []string{"", ""}, "path to the runtime genesis state") DefaultFixtureFlags.String(cfgRuntimeLoader, "oasis-core-runtime-loader", "path to the runtime loader") diff --git a/go/oasis-test-runner/oasis/client.go b/go/oasis-test-runner/oasis/client.go index 60f8a25e8e7..33a9a84a37d 100644 --- a/go/oasis-test-runner/oasis/client.go +++ b/go/oasis-test-runner/oasis/client.go @@ -4,15 +4,17 @@ import ( "fmt" "github.com/oasisprotocol/oasis-core/go/common/node" + runtimeRegistry "github.com/oasisprotocol/oasis-core/go/runtime/registry" ) // Client is an Oasis client node. type Client struct { *Node - runtimes []int - runtimeConfig map[int]map[string]interface{} - maxTransactionAge int64 + runtimes []int + runtimeProvisioner string + runtimeConfig map[int]map[string]interface{} + maxTransactionAge int64 consensusPort uint16 p2pPort uint16 @@ -22,15 +24,17 @@ type Client struct { type ClientCfg struct { NodeCfg - Runtimes []int - RuntimeConfig map[int]map[string]interface{} - MaxTransactionAge int64 + Runtimes []int + RuntimeProvisioner string + RuntimeConfig map[int]map[string]interface{} + MaxTransactionAge int64 } func (client *Client) AddArgs(args *argBuilder) error { args.debugDontBlameOasis(). debugAllowTestKeys(). debugEnableProfiling(client.Node.pprofPort). + runtimeProvisioner(client.runtimeProvisioner). tendermintPrune(client.consensus.PruneNumKept). tendermintRecoverCorruptedWAL(client.consensus.TendermintRecoverCorruptedWAL). tendermintCoreAddress(client.consensusPort). @@ -64,13 +68,18 @@ func (net *Network) NewClient(cfg *ClientCfg) (*Client, error) { return nil, err } + if cfg.RuntimeProvisioner == "" { + cfg.RuntimeProvisioner = runtimeRegistry.RuntimeProvisionerSandboxed + } + client := &Client{ - Node: host, - runtimes: cfg.Runtimes, - runtimeConfig: cfg.RuntimeConfig, - maxTransactionAge: cfg.MaxTransactionAge, - consensusPort: host.getProvisionedPort(nodePortConsensus), - p2pPort: host.getProvisionedPort(nodePortP2P), + Node: host, + runtimes: cfg.Runtimes, + runtimeProvisioner: cfg.RuntimeProvisioner, + runtimeConfig: cfg.RuntimeConfig, + maxTransactionAge: cfg.MaxTransactionAge, + consensusPort: host.getProvisionedPort(nodePortConsensus), + p2pPort: host.getProvisionedPort(nodePortP2P), } net.clients = append(net.clients, client) diff --git a/go/oasis-test-runner/oasis/fixture.go b/go/oasis-test-runner/oasis/fixture.go index 5299a887078..ac5a21b9f3d 100644 --- a/go/oasis-test-runner/oasis/fixture.go +++ b/go/oasis-test-runner/oasis/fixture.go @@ -308,6 +308,8 @@ type KeymanagerFixture struct { Entity int `json:"entity"` Policy int `json:"policy"` + RuntimeProvisioner string `json:"runtime_provisioner"` + AllowEarlyTermination bool `json:"allow_early_termination"` AllowErrorTermination bool `json:"allow_error_termination"` @@ -353,9 +355,10 @@ func (f *KeymanagerFixture) Create(net *Network) (*Keymanager, error) { NoAutoStart: f.NoAutoStart, Entity: entity, }, - Runtime: runtime, - Policy: policy, - SentryIndices: f.Sentries, + RuntimeProvisioner: f.RuntimeProvisioner, + Runtime: runtime, + Policy: policy, + SentryIndices: f.Sentries, }) } @@ -546,6 +549,8 @@ type ClientFixture struct { // Runtimes contains the indexes of the runtimes to enable. Runtimes []int `json:"runtimes,omitempty"` + RuntimeProvisioner string `json:"runtime_provisioner"` + // RuntimeConfig contains the per-runtime node-local configuration. RuntimeConfig map[int]map[string]interface{} `json:"runtime_config,omitempty"` @@ -564,9 +569,10 @@ func (f *ClientFixture) Create(net *Network) (*Client, error) { SupplementarySanityInterval: f.Consensus.SupplementarySanityInterval, EnableProfiling: f.EnableProfiling, }, - MaxTransactionAge: f.MaxTransactionAge, - Runtimes: f.Runtimes, - RuntimeConfig: f.RuntimeConfig, + MaxTransactionAge: f.MaxTransactionAge, + Runtimes: f.Runtimes, + RuntimeProvisioner: f.RuntimeProvisioner, + RuntimeConfig: f.RuntimeConfig, }) } diff --git a/go/oasis-test-runner/oasis/keymanager.go b/go/oasis-test-runner/oasis/keymanager.go index 7f1eeb846cf..52529a556d3 100644 --- a/go/oasis-test-runner/oasis/keymanager.go +++ b/go/oasis-test-runner/oasis/keymanager.go @@ -155,8 +155,9 @@ type Keymanager struct { // nolint: maligned sentryIndices []int - runtime *Runtime - policy *KeymanagerPolicy + runtime *Runtime + policy *KeymanagerPolicy + runtimeProvisioner string sentryPubKey signature.PublicKey tmAddress string @@ -172,8 +173,9 @@ type KeymanagerCfg struct { SentryIndices []int - Runtime *Runtime - Policy *KeymanagerPolicy + Runtime *Runtime + Policy *KeymanagerPolicy + RuntimeProvisioner string } // IdentityKeyPath returns the paths to the node's identity key. @@ -263,7 +265,7 @@ func (km *Keymanager) AddArgs(args *argBuilder) error { tendermintPrune(km.consensus.PruneNumKept). tendermintRecoverCorruptedWAL(km.consensus.TendermintRecoverCorruptedWAL). workerClientPort(km.workerClientPort). - runtimeProvisioner(runtimeRegistry.RuntimeProvisionerSandboxed). + runtimeProvisioner(km.runtimeProvisioner). runtimeSGXLoader(km.net.cfg.RuntimeSGXLoaderBinary). // XXX: could support configurable binary idx if ever needed. runtimePath(km.runtime.id, km.runtime.binaries[km.runtime.teeHardware][0]). @@ -316,16 +318,21 @@ func (net *Network) NewKeymanager(cfg *KeymanagerCfg) (*Keymanager, error) { return nil, fmt.Errorf("oasis/keymanager: sentry client public key unmarshal failure: %w", err) } + if cfg.RuntimeProvisioner == "" { + cfg.RuntimeProvisioner = runtimeRegistry.RuntimeProvisionerSandboxed + } + km := &Keymanager{ - Node: host, - runtime: cfg.Runtime, - policy: cfg.Policy, - sentryIndices: cfg.SentryIndices, - tmAddress: crypto.PublicKeyToTendermint(&host.p2pSigner).Address().String(), - sentryPubKey: sentryPubKey, - consensusPort: host.getProvisionedPort(nodePortConsensus), - workerClientPort: host.getProvisionedPort(nodePortClient), - mayGenerate: len(net.keymanagers) == 0, + Node: host, + runtime: cfg.Runtime, + policy: cfg.Policy, + runtimeProvisioner: cfg.RuntimeProvisioner, + sentryIndices: cfg.SentryIndices, + tmAddress: crypto.PublicKeyToTendermint(&host.p2pSigner).Address().String(), + sentryPubKey: sentryPubKey, + consensusPort: host.getProvisionedPort(nodePortConsensus), + workerClientPort: host.getProvisionedPort(nodePortClient), + mayGenerate: len(net.keymanagers) == 0, } net.keymanagers = append(net.keymanagers, km) From 788f56e0aa2e7fb532476af3397f298c5ad71bff Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Tue, 15 Jun 2021 08:32:33 -0500 Subject: [PATCH 09/12] go/oasis-net-runner: allow running without a keymanager --- go/oasis-net-runner/fixtures/default.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/go/oasis-net-runner/fixtures/default.go b/go/oasis-net-runner/fixtures/default.go index e869f0ff248..bc375921d98 100644 --- a/go/oasis-net-runner/fixtures/default.go +++ b/go/oasis-net-runner/fixtures/default.go @@ -113,6 +113,8 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) { RuntimeProvisioner: runtimeProvisioner, }} + usingKeymanager := viper.IsSet(cfgKeymanagerBinary) + if viper.GetBool(cfgSetupRuntimes) { fixture.Runtimes = []oasis.RuntimeFixture{ // Key manager runtime. @@ -130,11 +132,13 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) { GovernanceModel: registry.GovernanceEntity, }, } - fixture.KeymanagerPolicies = []oasis.KeymanagerPolicyFixture{ - {Runtime: 0, Serial: 1}, - } - fixture.Keymanagers = []oasis.KeymanagerFixture{ - {Runtime: 0, Entity: 1, RuntimeProvisioner: runtimeProvisioner}, + if usingKeymanager { + fixture.KeymanagerPolicies = []oasis.KeymanagerPolicyFixture{ + {Runtime: 0, Serial: 1}, + } + fixture.Keymanagers = []oasis.KeymanagerFixture{ + {Runtime: 0, Entity: 1, RuntimeProvisioner: runtimeProvisioner}, + } } fixture.StorageWorkers = []oasis.StorageWorkerFixture{ {Backend: "badger", Entity: 1}, @@ -163,13 +167,18 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) { cmdCommon.EarlyLogAndExit(fmt.Errorf("missing runtime genesis states, provided: %d, required: %d", l1, l2)) } + keymanagerIdx := -1 + if usingKeymanager { + keymanagerIdx = 0 + } + for i, rt := range runtimes { // Compute runtime. fixture.Runtimes = append(fixture.Runtimes, oasis.RuntimeFixture{ ID: runtimeIDs[i], Kind: registry.KindCompute, Entity: 0, - Keymanager: 0, + Keymanager: keymanagerIdx, Binaries: map[node.TEEHardware][]string{ tee: {rt}, }, From b18e345ccbc099ed16d378cec8bbb8efd987dd05 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Mon, 14 Jun 2021 13:49:10 -0500 Subject: [PATCH 10/12] Add changelog fragment --- .changelog/4024.feature.2.md | 4 ++++ .changelog/4024.feature.3.md | 4 ++++ .changelog/4024.feature.md | 1 + 3 files changed, 9 insertions(+) create mode 100644 .changelog/4024.feature.2.md create mode 100644 .changelog/4024.feature.3.md create mode 100644 .changelog/4024.feature.md diff --git a/.changelog/4024.feature.2.md b/.changelog/4024.feature.2.md new file mode 100644 index 00000000000..2c7162150c8 --- /dev/null +++ b/.changelog/4024.feature.2.md @@ -0,0 +1,4 @@ +go/oasis-net-runner: allow setting runtime provisioner + +On systems that do not support bwrap, the runtime provisioner can be +set to `unconfined`. diff --git a/.changelog/4024.feature.3.md b/.changelog/4024.feature.3.md new file mode 100644 index 00000000000..3b1ec3a3f58 --- /dev/null +++ b/.changelog/4024.feature.3.md @@ -0,0 +1,4 @@ +go/oasis-net-runner: allow running without a keymanager + +This feature makes it easier to run debug runtimes that are not (yet) +associated with a keymanager. diff --git a/.changelog/4024.feature.md b/.changelog/4024.feature.md new file mode 100644 index 00000000000..cb6d734ef74 --- /dev/null +++ b/.changelog/4024.feature.md @@ -0,0 +1 @@ +runtime-loader: allow bulding for non-SGX use on platforms that aren't Linux From 148fcd68ccd4d4e7c9ff87b9fa56e840ed4f0e4f Mon Sep 17 00:00:00 2001 From: Jernej Kos Date: Thu, 17 Jun 2021 08:51:32 +0200 Subject: [PATCH 11/12] go/oasis-test-runner: Don't panic without keymanager runtimes --- go/oasis-test-runner/oasis/keymanager.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/go/oasis-test-runner/oasis/keymanager.go b/go/oasis-test-runner/oasis/keymanager.go index 52529a556d3..92d2cb70019 100644 --- a/go/oasis-test-runner/oasis/keymanager.go +++ b/go/oasis-test-runner/oasis/keymanager.go @@ -256,6 +256,11 @@ func (km *Keymanager) AddArgs(args *argBuilder) error { return err } + runtimeBinaries := km.runtime.binaries[km.runtime.teeHardware] + if len(runtimeBinaries) < 1 { + return fmt.Errorf("oasis/keymanager: no runtime binaries configured") + } + args.debugDontBlameOasis(). debugAllowTestKeys(). debugEnableProfiling(km.Node.pprofPort). @@ -268,7 +273,7 @@ func (km *Keymanager) AddArgs(args *argBuilder) error { runtimeProvisioner(km.runtimeProvisioner). runtimeSGXLoader(km.net.cfg.RuntimeSGXLoaderBinary). // XXX: could support configurable binary idx if ever needed. - runtimePath(km.runtime.id, km.runtime.binaries[km.runtime.teeHardware][0]). + runtimePath(km.runtime.id, runtimeBinaries[0]). workerKeymanagerEnabled(). workerKeymanagerRuntimeID(km.runtime.id). configureDebugCrashPoints(km.crashPointsProbability). From 4d99acba29cba60d454675b72f4e6e5d9cb47901 Mon Sep 17 00:00:00 2001 From: Jernej Kos Date: Thu, 17 Jun 2021 08:52:12 +0200 Subject: [PATCH 12/12] go/oasis-net-runner: Fix keymanager use determination --- .changelog/4042.trivial.md | 1 + go/oasis-net-runner/fixtures/default.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 .changelog/4042.trivial.md diff --git a/.changelog/4042.trivial.md b/.changelog/4042.trivial.md new file mode 100644 index 00000000000..0691152614c --- /dev/null +++ b/.changelog/4042.trivial.md @@ -0,0 +1 @@ +go/oasis-net-runner: Fix keymanager use determination diff --git a/go/oasis-net-runner/fixtures/default.go b/go/oasis-net-runner/fixtures/default.go index bc375921d98..06aa4188d2b 100644 --- a/go/oasis-net-runner/fixtures/default.go +++ b/go/oasis-net-runner/fixtures/default.go @@ -113,7 +113,7 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) { RuntimeProvisioner: runtimeProvisioner, }} - usingKeymanager := viper.IsSet(cfgKeymanagerBinary) + usingKeymanager := len(viper.GetString(cfgKeymanagerBinary)) > 0 if viper.GetBool(cfgSetupRuntimes) { fixture.Runtimes = []oasis.RuntimeFixture{ @@ -124,7 +124,7 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) { Entity: 0, Keymanager: -1, Binaries: map[node.TEEHardware][]string{ - tee: viper.GetStringSlice(cfgKeymanagerBinary), + tee: {viper.GetString(cfgKeymanagerBinary)}, }, AdmissionPolicy: registry.RuntimeAdmissionPolicy{ AnyNode: ®istry.AnyNodeRuntimeAdmissionPolicy{},