Fix GoReleaser oasis-node build reproducibility issue

.changelog/2571.bugfix.md

@@ -0,0 +1,10 @@
+Make oasis-node binaries made with GoReleaser via GitHub Actions reproducible
+again.
+
+Add `-buildid=` back to `ldflags` to make builds reproducible again.
+
+As noted in [60641ce](
+https://github.com/oasislabs/oasis-core/commit/60641ce41a9c2402f1b539375e1dd4e0eb45272d),
+this should be no longer necessary with Go 1.13.4+, but there appears to be a
+[specific issue with GoReleaser's build handling](
+https://github.com/oasislabs/goreleaser/issues/1).

.changelog/2590.internal.md

@@ -0,0 +1,9 @@
+github: Add ci-reproducibility workflow
+
+The workflow spawns two build jobs that use the same build environment, except
+for the path of the git checkout.
+The `oasis-node` binary is built two times, once directly via Make's `go build`
+invocation and the second time using the [GoReleaser](https://goreleaser.com/)
+tool that is used to make the official Oasis Core releases.
+The last workflow job compares both checksums of both builds and errors if they
+are not the same.

.github/workflows/ci.yml → .github/workflows/ci-lint.yml

@@ -1,4 +1,5 @@
-name: Continuous integration
+# NOTE: This name appears in GitHub's Checks API and in workflow's status badge.
+name: ci-lint
 
 # Trigger the workflow when:
 on:
@@ -17,6 +18,8 @@ on:
 jobs:
 
   lint:
+    # NOTE: This name appears in GitHub's Checks API.
+    name: lint
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code

.github/workflows/ci-reproducibility.yml

@@ -0,0 +1,106 @@
+# NOTE: This name appears in GitHub's Checks API and in workflow's status badge.
+name: ci-reproducibility
+
+# Trigger the workflow when:
+on:
+  # A push occurs to one of the matched branches.
+  push:
+    branches:
+      - master
+      - stable/*
+  # Or when a pull request event occurs for a pull request against one of the
+  # matched branches.
+  pull_request:
+    branches:
+      - master
+      - stable/*
+
+jobs:
+
+  build-code:
+    # NOTE: This name appears in GitHub's Checks API.
+    name: build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        build_number: [1, 2]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          # NOTE: We make a git checkout to a unique directory for each build to
+          # catch reproducibility issues with code in different local paths.
+          path: build${{ matrix.build_number }}
+      - name: Set up Go 1.13
+        uses: actions/[email protected]
+        with:
+          go-version: "1.13.x"
+      - name: Install oasis-node build prerequisites
+        run: |
+          sudo apt-get update
+          sudo apt-get install make libseccomp-dev
+      - name: Install GoReleaser
+        run: |
+          cd $(mktemp --directory /tmp/goreleaser.XXXXX)
+          ${CURL_CMD} ${GORELEASER_URL_PREFIX}/v${GORELEASER_VERSION}/${GORELEASER_TARBALL} \
+            --output ${GORELEASER_TARBALL}
+          ${CURL_CMD} ${GORELEASER_URL_PREFIX}/v${GORELEASER_VERSION}/goreleaser_checksums.txt \
+            --output CHECKSUMS
+          sha256sum --check --ignore-missing CHECKSUMS
+          tar -xf ${GORELEASER_TARBALL}
+          sudo mv goreleaser /usr/local/bin
+        env:
+          GORELEASER_URL_PREFIX: https://github.com/oasislabs/goreleaser/releases/download/
+          GORELEASER_VERSION: 0.123.3-oasis1
+          GORELEASER_TARBALL: goreleaser_Linux_x86_64.tar.gz
+          CURL_CMD: curl --proto =https --tlsv1.2 -sSL
+      - name: Build oasis-node with Make
+        run: |
+          cd build${{ matrix.build_number }}/go
+          make oasis-node
+      - name: Build oasis-node with GoReleaser
+        run: |
+          cd build${{ matrix.build_number }}
+          make release
+      - name: Get checksums of oasis-node builds
+        run: |
+          cd build${{ matrix.build_number }}
+          sha256sum ${OASIS_NODE_MAKE_PATH} ${OASIS_NODE_GORELEASER_PATH} > ../oasis-node-SHA256SUMs
+        env:
+          OASIS_NODE_MAKE_PATH: go/oasis-node/oasis-node
+          OASIS_NODE_GORELEASER_PATH: dist/oasis-node_linux_amd64/oasis-node
+      - name: Upload checksums
+        uses: actions/upload-artifact@v1
+        with:
+          name: oasis-node-SHA256SUMs-build${{ matrix.build_number }}
+          path: oasis-node-SHA256SUMs
+
+  compare-checksums:
+    # NOTE: This name appears in GitHub's Checks API.
+    name: checksums
+    # NOTE: Requiring a matrix job waits for all the matrix jobs to be complete.
+    needs: build-code
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download checksums for build 1
+        uses: actions/download-artifact@v1
+        with:
+          name: oasis-node-SHA256SUMs-build1
+      - name: Download checksum for build 2
+        uses: actions/download-artifact@v1
+        with:
+          name: oasis-node-SHA256SUMs-build2
+      - name: Check if checksums are the same
+        shell: bash
+        run: |
+          for i in 1 2; do
+            echo "Checksums for build $i are: "
+            cat oasis-node-SHA256SUMs-build$i/oasis-node-SHA256SUMs
+            echo
+          done
+          if ! DIFF=$(diff --unified=0 oasis-node-SHA256SUMs-build{1,2}/oasis-node-SHA256SUMs); then
+            echo "Error: Checksums for builds 1 and 2 differ: "
+            echo -e "$DIFF"
+            exit 1
+          fi
+

.github/workflows/release.yml

@@ -42,6 +42,8 @@ jobs:
         run: |
           make release
         env:
+          # Instruct Make to create a real release.
+          OASIS_CORE_REAL_RELEASE: "true"
           # Pass automatically created GitHub App installation token to the action.
           # For more info, see:
           # https://help.github.com/en/actions/automating-your-workflow-with-github-actions/authenticating-with-the-github_token.

.goreleaser.yml

@@ -12,15 +12,18 @@ env:
   - GO111MODULE=on
 
 builds:
-  - env:
-      - CGO_ENABLED=1
-    id: oasis-node
+  - id: oasis-node
     main: ./oasis-node/main.go
     binary: oasis-node
     dir: go/
     flags:
       - -trimpath
     ldflags:
+      # NOTE: At the moment, GoReleaser produces different binaries when
+      # releases are built from different git paths, unless -buildid= is added
+      # to ldflags.
+      # For more details, see: https://github.com/oasislabs/goreleaser/issues/1.
+      - -buildid=
       - -X github.com/oasislabs/oasis-core/go/common/version.SoftwareVersion={{.Env.VERSION}}
     goos:
       - linux

README.md

@@ -1,6 +1,8 @@
 # Oasis Core
 
 [![Build status](https://badge.buildkite.com/15a8faccee0d5b5ab1af7e75eb6f9daf2d493c543fbc67dce5.svg?branch=master)](https://buildkite.com/oasislabs/oasis-core-ci)
+[![CI lint status](https://github.com/oasislabs/oasis-core/workflows/ci-lint/badge.svg)](https://github.com/oasislabs/oasis-core/actions?query=workflow:ci-lint)
+[![CI reproducibility status](https://github.com/oasislabs/oasis-core/workflows/ci-reproducibility/badge.svg)](https://github.com/oasislabs/oasis-core/actions?query=workflow:ci-reproducibility)
 [![Coverage Status](https://coveralls.io/repos/github/oasislabs/oasis-core/badge.svg?t=HsLWgi)](https://coveralls.io/github/oasislabs/oasis-core) Rust
 [![codecov](https://codecov.io/gh/oasislabs/oasis-core/branch/master/graph/badge.svg?token=DqjRsufMqf)](https://codecov.io/gh/oasislabs/oasis-core) Go
 [![GoDoc](https://godoc.org/github.com/oasislabs/oasis-core?status.svg)](https://godoc.org/github.com/oasislabs/oasis-core)

common.mk

@@ -101,35 +101,12 @@ GO_TEST_HELPER_URKEL_PATH := storage/mkvs/urkel/interop/urkel-test-helpers
 
 # Instruct GoReleaser to create a "snapshot" release by default.
 GORELEASER_ARGS ?= release --snapshot --rm-dist
-
-# If running inside GitHub Actions, create a real release.
-ifeq ($(GITHUB_ACTIONS), true)
-
-# Auxiliary variable that defines a new line for later substitution.
-define newline
-
-
-endef
-
-# GitHub release' text in Markdown format.
-define RELEASE_TEXT =
-For a list of changes in this release, see the [Change Log].
-
-*NOTE: If you are upgrading from an earlier release, please **carefully review**
-the [Change Log] for **Removals and Breaking changes**.*
-
-If you would like to become a node operator for the Oasis Network, see the
-[Operator Docs](https://docs.oasis.dev/operators/overview.html).
-
-[Change Log]: https://github.com/oasislabs/oasis-core/blob/v$(VERSION)/CHANGELOG.md
-
-endef
-
-# Temporary file with GitHub release's text.
+# If the appropriate environment variable is set, create a real release.
+ifeq ($(OASIS_CORE_REAL_RELEASE), true)
+# Create temporary file with GitHub release's text.
 _RELEASE_NOTES_FILE := $(shell mktemp /tmp/oasis-core.XXXXX)
 _ := $(shell printf "$(subst ",\",$(subst $(newline),\n,$(RELEASE_TEXT)))" > $(_RELEASE_NOTES_FILE))
 GORELEASER_ARGS = release --release-notes $(_RELEASE_NOTES_FILE)
-
 endif
 
 # Helper that ensures $(NEXT_VERSION) variable is not empty.
@@ -169,3 +146,23 @@ define ENSURE_NEXT_VERSION_IN_CHANGELOG =
 		exit 1; \
 	fi
 endef
+
+# Auxiliary variable that defines a new line for later substitution.
+define newline
+
+
+endef
+
+# GitHub release' text in Markdown format.
+define RELEASE_TEXT =
+For a list of changes in this release, see the [Change Log].
+
+*NOTE: If you are upgrading from an earlier release, please **carefully review**
+the [Change Log] for **Removals and Breaking changes**.*
+
+If you would like to become a node operator for the Oasis Network, see the
+[Operator Docs](https://docs.oasis.dev/operators/overview.html).
+
+[Change Log]: https://github.com/oasislabs/oasis-core/blob/v$(VERSION)/CHANGELOG.md
+
+endef