From b591791f8bf46a57302e6d658f76d052c9cb4994 Mon Sep 17 00:00:00 2001 From: ptrus Date: Wed, 16 Jun 2021 15:21:40 +0200 Subject: [PATCH] Build BadgerDB with `jemalloc` tag --- .changelog/4040.feature.md | 12 ++++++++++++ .github/workflows/ci-reproducibility.yml | 10 ++++++++++ .github/workflows/release-dev.yml | 10 ++++++++++ .github/workflows/release.yml | 10 ++++++++++ .goreleaser.yml | 4 ++++ docker/development/Dockerfile | 10 ++++++++++ docs/setup/building.md | 9 +++++++++ docs/setup/prerequisites.md | 23 +++++++++++++++++++++++ go/Makefile | 5 +++++ 9 files changed, 93 insertions(+) create mode 100644 .changelog/4040.feature.md diff --git a/.changelog/4040.feature.md b/.changelog/4040.feature.md new file mode 100644 index 00000000000..8f7879136ea --- /dev/null +++ b/.changelog/4040.feature.md @@ -0,0 +1,12 @@ +Build `oasis-node` with `jemalloc` tag (used by BadgerDB) + +In BadgerDB V3 using `jemalloc` seems to be recommended and better supported +option ([1], [2]). Based on testing using `jemalloc` reduces BadgerDB memory +usage. + +To build `oasis-node` without `jemalloc` requirement, set the +`OASIS_BADGER_NO_JEMALLOC="1"` environment variable before invoking the +makefile. + +[1]: https://dgraph.io/blog/post/manual-memory-management-golang-jemalloc/ +[2]: https://discuss.dgraph.io/t/memory-issue-during-stream-operation/13033 diff --git a/.github/workflows/ci-reproducibility.yml b/.github/workflows/ci-reproducibility.yml index 5f0307d37fe..fb9866262e9 100644 --- a/.github/workflows/ci-reproducibility.yml +++ b/.github/workflows/ci-reproducibility.yml @@ -17,6 +17,8 @@ env: GORELEASER_URL_PREFIX: https://github.com/goreleaser/goreleaser/releases/download/ GORELEASER_VERSION: 0.152.0 CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail + JEMALLOC_VERSION: 5.2.1 + JEMALLOC_CHECKSUM: 34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jobs: @@ -49,6 +51,14 @@ jobs: run: | sudo apt-get update sudo apt-get install make libseccomp-dev protobuf-compiler + - name: Install jemalloc + run: | + wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 + echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c + tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION} + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto' + make + sudo make install - name: Install GoReleaser run: | cd $(mktemp --directory /tmp/goreleaser.XXXXX) diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index b5d7b9064c4..2cd6b69e707 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -14,6 +14,8 @@ env: GORELEASER_URL_PREFIX: https://github.com/goreleaser/goreleaser/releases/download/ GORELEASER_VERSION: 0.152.0 CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail + JEMALLOC_VERSION: 5.2.1 + JEMALLOC_CHECKSUM: 34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jobs: @@ -40,6 +42,14 @@ jobs: run: | sudo apt-get update sudo apt-get install make libseccomp-dev protobuf-compiler + - name: Install jemalloc + run: | + wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 + echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c + tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION} + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto' + make + sudo make install - name: Install GoReleaser run: | cd $(mktemp --directory /tmp/goreleaser.XXXXX) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4fbdfa8de04..de0b1dd671e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,6 +16,8 @@ env: GORELEASER_URL_PREFIX: https://github.com/goreleaser/goreleaser/releases/download/ GORELEASER_VERSION: 0.152.0 CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail + JEMALLOC_VERSION: 5.2.1 + JEMALLOC_CHECKSUM: 34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jobs: @@ -42,6 +44,14 @@ jobs: run: | sudo apt-get update sudo apt-get install make libseccomp-dev protobuf-compiler + - name: Install jemalloc + run: | + wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 + echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c + tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION} + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto' + make + sudo make install - name: Install GoReleaser run: | cd $(mktemp --directory /tmp/goreleaser.XXXXX) diff --git a/.goreleaser.yml b/.goreleaser.yml index d9b8833728d..07e7d15a47d 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -18,6 +18,10 @@ builds: dir: go/ flags: - -trimpath + # Build oasis-node with jemalloc tag (used by badgerdb). + # TODO: Use 'tags' attribute when GoReleaser is udpated to newer version: + # https://github.com/goreleaser/goreleaser/pull/2268 + - -tags=jemalloc ldflags: # NOTE: At the moment, GoReleaser produces different binaries when # releases are built from different git paths, unless -buildid= is added diff --git a/docker/development/Dockerfile b/docker/development/Dockerfile index 1fc6ce76c78..d4ead8bfcf7 100644 --- a/docker/development/Dockerfile +++ b/docker/development/Dockerfile @@ -10,6 +10,8 @@ ARG GOLANGCILINT_VERSION=1.28.2 ARG GOCOVMERGE_VERSION=b5bfa59ec0adc420475f97f89b58045c721d761c ARG GOFUMPT_VERSION=abc0db2c416aca0f60ea33c23c76665f6e7ba0b6 ARG RUST_NIGHTLY_VERSION=2021-05-20 +ARG JEMALLOC_VERSION=5.2.1 +ARG JEMALLOC_CHECKSUM=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 ARG DEBIAN_FRONTEND=noninteractive @@ -72,6 +74,14 @@ RUN wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz && \ GO111MODULE=on go get mvdan.cc/gofumpt@${GOFUMPT_VERSION} && \ GO111MODULE=on go get mvdan.cc/gofumpt/gofumports@${GOFUMPT_VERSION} +# Install jemalloc (used by badgerdb). +RUN wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 && \ + echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c && \ + tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION} && \ + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto' && \ + make && make install && \ + cd .. && rm jemalloc.tar.bz2 && rm -rf jemalloc-${JEMALLOC_VERSION} + # Install bubblewrap (we need at least version 0.3.3 which is not available for 18.04). RUN wget http://archive.ubuntu.com/ubuntu/pool/main/b/bubblewrap/bubblewrap_0.4.1-1_amd64.deb && \ echo '25de452f209e4fdb4b009851c33ca9a0269ebf0b92f4bd9b86186480592cc3e2 bubblewrap_0.4.1-1_amd64.deb' | sha256sum -c && \ diff --git a/docs/setup/building.md b/docs/setup/building.md index a8948635510..0489995117c 100644 --- a/docs/setup/building.md +++ b/docs/setup/building.md @@ -17,6 +17,15 @@ export OASIS_UNSAFE_SKIP_KM_POLICY="1" make ``` +To build BadgerDB without `jemalloc` support (and avoid installing `jemalloc` +on your system), set + +``` +export OASIS_BADGER_NO_JEMALLOC="1" +``` + +Not using `jemalloc` is fine for development purposes. + This will build all the required parts (build tools, Oasis node, runtime libraries, runtime loader, key manager and test runtimes). The AVR and KM flags are supported on production SGX systems only and these features must be disabled diff --git a/docs/setup/prerequisites.md b/docs/setup/prerequisites.md index c77b5a5d33d..4b55d0cc584 100644 --- a/docs/setup/prerequisites.md +++ b/docs/setup/prerequisites.md @@ -169,6 +169,28 @@ Core: committed for convenience. Installing protoc-gen-go is only required if you are a developer making changes to protobuf definitions used by Go._ +* (**OPTIONAL**) [jemalloc] (version 5.2.1, built with `'je_'` jemalloc-prefix) + + Alternatively set `OASIS_BADGER_NO_JEMALLOC="1"` environment variable when + building `oasis-node` code, to build BadgerDB without `jemalloc` support. + + Download and install `jemalloc` with: + + + ``` + export JEMALLOC_VERSION=5.2.1 + export JEMALLOC_CHECKSUM=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 + wget -O jemalloc.tar.bz2 "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" + # Ensure checksum matches. + echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c + tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner + cd jemalloc-${JEMALLOC_VERSION} + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto' + make + make install + ``` + + In the following instructions, the top-level directory is the directory where the code has been checked out. @@ -192,6 +214,7 @@ where the code has been checked out. [Fortanix Rust EDP]: https://edp.fortanix.com [gofumpt and gofumports]: https://github.com/mvdan/gofumpt [protoc-gen-go]: https://github.com/golang/protobuf +[jemalloc]: https://github.com/jemalloc/jemalloc ## Using the Development Docker Image diff --git a/go/Makefile b/go/Makefile index e2207c2e1a4..bb54d7444a2 100644 --- a/go/Makefile +++ b/go/Makefile @@ -5,6 +5,11 @@ ifneq ($(GOLDFLAGS),) GO_EXTRA_FLAGS += -ldflags $(GOLDFLAGS) endif +# Build code with jemalloc tag unless explicitly disabled (used by badgerdb). +ifneq ($(OASIS_BADGER_NO_JEMALLOC), 1) + GO_EXTRA_FLAGS += -tags jemalloc +endif + # Set all target as the default target. all: build