oasisprotocol · ptrus · Jun 17, 2021 · Jun 16, 2021
@@ -0,0 +1,12 @@
+Build `oasis-node` with `jemalloc` tag (used by BadgerDB)
+
+In BadgerDB V3 using `jemalloc` seems to be recommended and better supported
+option ([1], [2]). Based on testing using `jemalloc` reduces BadgerDB memory
+usage.
+
+To build `oasis-node` without `jemalloc` requirement, set the
+`OASIS_BADGER_NO_JEMALLOC="1"` environment variable before invoking the
+makefile.
+
+[1]: https://dgraph.io/blog/post/manual-memory-management-golang-jemalloc/
+[2]: https://discuss.dgraph.io/t/memory-issue-during-stream-operation/13033
@@ -17,6 +17,8 @@ env:
   GORELEASER_URL_PREFIX: https://github.com/goreleaser/goreleaser/releases/download/
   GORELEASER_VERSION: 0.152.0
   CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail
+  JEMALLOC_VERSION: 5.2.1
+  JEMALLOC_CHECKSUM: 34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
 
 jobs:
 
@@ -49,6 +51,14 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install make libseccomp-dev protobuf-compiler
+      - name: Install jemalloc
+        run: |
+          wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
+          echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c
+          tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION}
+          ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto'
+          make
+          sudo make install
       - name: Install GoReleaser
         run: |
           cd $(mktemp --directory /tmp/goreleaser.XXXXX)

@@ -14,6 +14,8 @@ env:
   GORELEASER_URL_PREFIX: https://github.com/goreleaser/goreleaser/releases/download/
   GORELEASER_VERSION: 0.152.0
   CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail
+  JEMALLOC_VERSION: 5.2.1
+  JEMALLOC_CHECKSUM: 34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
 
 jobs:
 
@@ -40,6 +42,14 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install make libseccomp-dev protobuf-compiler
+      - name: Install jemalloc
+        run: |
+          wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
+          echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c
+          tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION}
+          ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto'
+          make
+          sudo make install
       - name: Install GoReleaser
         run: |
           cd $(mktemp --directory /tmp/goreleaser.XXXXX)

@@ -16,6 +16,8 @@ env:
   GORELEASER_URL_PREFIX: https://github.com/goreleaser/goreleaser/releases/download/
   GORELEASER_VERSION: 0.152.0
   CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail
+  JEMALLOC_VERSION: 5.2.1
+  JEMALLOC_CHECKSUM: 34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
 
 jobs:
 
@@ -42,6 +44,14 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install make libseccomp-dev protobuf-compiler
+      - name: Install jemalloc
+        run: |
+          wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
+          echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c
+          tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION}
+          ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto'
+          make
+          sudo make install
       - name: Install GoReleaser
         run: |
           cd $(mktemp --directory /tmp/goreleaser.XXXXX)

@@ -18,6 +18,10 @@ builds:
     dir: go/
     flags:
       - -trimpath
+      # Build oasis-node with jemalloc tag (used by badgerdb).
+      # TODO: Use 'tags' attribute when GoReleaser is udpated to newer version:
+      # https://github.com/goreleaser/goreleaser/pull/2268
+      - -tags=jemalloc
     ldflags:
       # NOTE: At the moment, GoReleaser produces different binaries when
       # releases are built from different git paths, unless -buildid= is added

@@ -10,6 +10,8 @@ ARG GOLANGCILINT_VERSION=1.28.2
 ARG GOCOVMERGE_VERSION=b5bfa59ec0adc420475f97f89b58045c721d761c
 ARG GOFUMPT_VERSION=abc0db2c416aca0f60ea33c23c76665f6e7ba0b6
 ARG RUST_NIGHTLY_VERSION=2021-05-20
+ARG JEMALLOC_VERSION=5.2.1
+ARG JEMALLOC_CHECKSUM=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -72,6 +74,14 @@ RUN wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz && \
     GO111MODULE=on go get mvdan.cc/gofumpt@${GOFUMPT_VERSION} && \
     GO111MODULE=on go get mvdan.cc/gofumpt/gofumports@${GOFUMPT_VERSION}
 
+# Install jemalloc (used by badgerdb).
+RUN wget -O jemalloc.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 && \
+    echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c && \
+    tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner && cd jemalloc-${JEMALLOC_VERSION} && \
+    ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto' && \
+    make && make install && \
+    cd .. && rm jemalloc.tar.bz2 && rm -rf jemalloc-${JEMALLOC_VERSION}
+
 # Install bubblewrap (we need at least version 0.3.3 which is not available for 18.04).
 RUN wget http://archive.ubuntu.com/ubuntu/pool/main/b/bubblewrap/bubblewrap_0.4.1-1_amd64.deb && \
     echo '25de452f209e4fdb4b009851c33ca9a0269ebf0b92f4bd9b86186480592cc3e2 bubblewrap_0.4.1-1_amd64.deb' | sha256sum -c && \

@@ -17,6 +17,15 @@ export OASIS_UNSAFE_SKIP_KM_POLICY="1"
 make
 ```
 
+To build BadgerDB without `jemalloc` support (and avoid installing `jemalloc`
+on your system), set
+
+```
+export OASIS_BADGER_NO_JEMALLOC="1"
+```
+
+Not using `jemalloc` is fine for development purposes.
+
 This will build all the required parts (build tools, Oasis node, runtime
 libraries, runtime loader, key manager and test runtimes). The AVR and KM flags
 are supported on production SGX systems only and these features must be disabled

@@ -169,6 +169,28 @@ Core:
   committed for convenience.  Installing protoc-gen-go is only required if you
   are a developer making changes to protobuf definitions used by Go._
 
+* (**OPTIONAL**) [jemalloc] (version 5.2.1, built with `'je_'` jemalloc-prefix)
+
+  Alternatively set `OASIS_BADGER_NO_JEMALLOC="1"` environment variable when
+  building `oasis-node` code, to build BadgerDB without `jemalloc` support.
+
+  Download and install `jemalloc` with:
+
+  <!-- markdownlint-disable line-length -->
+  ```
+  export JEMALLOC_VERSION=5.2.1
+  export JEMALLOC_CHECKSUM=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
+  wget -O jemalloc.tar.bz2 "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2"
+  # Ensure checksum matches.
+  echo "${JEMALLOC_CHECKSUM} jemalloc.tar.bz2" | sha256sum -c
+  tar -xjf ./jemalloc.tar.bz2 -v --no-same-owner
+  cd jemalloc-${JEMALLOC_VERSION}
+  ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto'
+  make
+  make install
+  ```
+  <!-- markdownlint-enable line-length -->
+
 In the following instructions, the top-level directory is the directory
 where the code has been checked out.
 
@@ -192,6 +214,7 @@ where the code has been checked out.
 [Fortanix Rust EDP]: https://edp.fortanix.com
 [gofumpt and gofumports]: https://github.com/mvdan/gofumpt
 [protoc-gen-go]: https://github.com/golang/protobuf
+[jemalloc]: https://github.com/jemalloc/jemalloc
 
 ## Using the Development Docker Image
 

@@ -5,6 +5,11 @@ ifneq ($(GOLDFLAGS),)
 	GO_EXTRA_FLAGS += -ldflags $(GOLDFLAGS)
 endif
 
+# Build code with jemalloc tag unless explicitly disabled (used by badgerdb).
+ifneq ($(OASIS_BADGER_NO_JEMALLOC), 1)
+	GO_EXTRA_FLAGS += -tags jemalloc
+endif
+
 # Set all target as the default target.
 all: build