diff --git a/.changelog/2571.bugfix.md b/.changelog/2571.bugfix.md new file mode 100644 index 00000000000..ed6cf4e92fc --- /dev/null +++ b/.changelog/2571.bugfix.md @@ -0,0 +1,10 @@ +Make oasis-node binaries made with GoReleaser via GitHub Actions reproducible +again. + +Add `-buildid=` back to `ldflags` to make builds reproducible again. + +As noted in [60641ce]( +https://github.com/oasislabs/oasis-core/commit/60641ce41a9c2402f1b539375e1dd4e0eb45272d), +this should be no longer necessary with Go 1.13.4+, but there appears to be a +[specific issue with GoReleaser's build handling]( +https://github.com/oasislabs/goreleaser/issues/1). diff --git a/.changelog/2590.internal.md b/.changelog/2590.internal.md new file mode 100644 index 00000000000..cfd1f36f407 --- /dev/null +++ b/.changelog/2590.internal.md @@ -0,0 +1,9 @@ +github: Add ci-reproducibility workflow + +The workflow spawns two build jobs that use the same build environment, except +for the path of the git checkout. +The `oasis-node` binary is built two times, once directly via Make's `go build` +invocation and the second time using the [GoReleaser](https://goreleaser.com/) +tool that is used to make the official Oasis Core releases. +The last workflow job compares both checksums of both builds and errors if they +are not the same. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci-lint.yml similarity index 88% rename from .github/workflows/ci.yml rename to .github/workflows/ci-lint.yml index 853a3b8064d..04092887e4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci-lint.yml @@ -1,4 +1,5 @@ -name: Continuous integration +# NOTE: This name appears in GitHub's Checks API and in workflow's status badge. +name: ci-lint # Trigger the workflow when: on: @@ -17,6 +18,8 @@ on: jobs: lint: + # NOTE: This name appears in GitHub's Checks API. + name: lint runs-on: ubuntu-latest steps: - name: Checkout code diff --git a/.github/workflows/ci-reproducibility.yml b/.github/workflows/ci-reproducibility.yml new file mode 100644 index 00000000000..fda25745b59 --- /dev/null +++ b/.github/workflows/ci-reproducibility.yml @@ -0,0 +1,106 @@ +# NOTE: This name appears in GitHub's Checks API and in workflow's status badge. +name: ci-reproducibility + +# Trigger the workflow when: +on: + # A push occurs to one of the matched branches. + push: + branches: + - master + - stable/* + # Or when a pull request event occurs for a pull request against one of the + # matched branches. + pull_request: + branches: + - master + - stable/* + +jobs: + + build-code: + # NOTE: This name appears in GitHub's Checks API. + name: build + runs-on: ubuntu-latest + strategy: + matrix: + build_number: [1, 2] + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + # NOTE: We make a git checkout to a unique directory for each build to + # catch reproducibility issues with code in different local paths. + path: build${{ matrix.build_number }} + - name: Set up Go 1.13 + uses: actions/setup-go@v1.1.2 + with: + go-version: "1.13.x" + - name: Install oasis-node build prerequisites + run: | + sudo apt-get update + sudo apt-get install make libseccomp-dev + - name: Install GoReleaser + run: | + cd $(mktemp --directory /tmp/goreleaser.XXXXX) + ${CURL_CMD} ${GORELEASER_URL_PREFIX}/v${GORELEASER_VERSION}/${GORELEASER_TARBALL} \ + --output ${GORELEASER_TARBALL} + ${CURL_CMD} ${GORELEASER_URL_PREFIX}/v${GORELEASER_VERSION}/goreleaser_checksums.txt \ + --output CHECKSUMS + sha256sum --check --ignore-missing CHECKSUMS + tar -xf ${GORELEASER_TARBALL} + sudo mv goreleaser /usr/local/bin + env: + GORELEASER_URL_PREFIX: https://github.com/oasislabs/goreleaser/releases/download/ + GORELEASER_VERSION: 0.123.3-oasis1 + GORELEASER_TARBALL: goreleaser_Linux_x86_64.tar.gz + CURL_CMD: curl --proto =https --tlsv1.2 -sSL + - name: Build oasis-node with Make + run: | + cd build${{ matrix.build_number }}/go + make oasis-node + - name: Build oasis-node with GoReleaser + run: | + cd build${{ matrix.build_number }} + make release + - name: Get checksums of oasis-node builds + run: | + cd build${{ matrix.build_number }} + sha256sum ${OASIS_NODE_MAKE_PATH} ${OASIS_NODE_GORELEASER_PATH} > ../oasis-node-SHA256SUMs + env: + OASIS_NODE_MAKE_PATH: go/oasis-node/oasis-node + OASIS_NODE_GORELEASER_PATH: dist/oasis-node_linux_amd64/oasis-node + - name: Upload checksums + uses: actions/upload-artifact@v1 + with: + name: oasis-node-SHA256SUMs-build${{ matrix.build_number }} + path: oasis-node-SHA256SUMs + + compare-checksums: + # NOTE: This name appears in GitHub's Checks API. + name: checksums + # NOTE: Requiring a matrix job waits for all the matrix jobs to be complete. + needs: build-code + runs-on: ubuntu-latest + steps: + - name: Download checksums for build 1 + uses: actions/download-artifact@v1 + with: + name: oasis-node-SHA256SUMs-build1 + - name: Download checksum for build 2 + uses: actions/download-artifact@v1 + with: + name: oasis-node-SHA256SUMs-build2 + - name: Check if checksums are the same + shell: bash + run: | + for i in 1 2; do + echo "Checksums for build $i are: " + cat oasis-node-SHA256SUMs-build$i/oasis-node-SHA256SUMs + echo + done + if ! DIFF=$(diff --unified=0 oasis-node-SHA256SUMs-build{1,2}/oasis-node-SHA256SUMs); then + echo "Error: Checksums for builds 1 and 2 differ: " + echo -e "$DIFF" + exit 1 + fi + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d9141603c4..ba7baeefbdd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,6 +42,8 @@ jobs: run: | make release env: + # Instruct Make to create a real release. + OASIS_CORE_REAL_RELEASE: "true" # Pass automatically created GitHub App installation token to the action. # For more info, see: # https://help.github.com/en/actions/automating-your-workflow-with-github-actions/authenticating-with-the-github_token. diff --git a/.goreleaser.yml b/.goreleaser.yml index 1886b03fd37..5696d50a2b4 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -12,15 +12,18 @@ env: - GO111MODULE=on builds: - - env: - - CGO_ENABLED=1 - id: oasis-node + - id: oasis-node main: ./oasis-node/main.go binary: oasis-node dir: go/ flags: - -trimpath ldflags: + # NOTE: At the moment, GoReleaser produces different binaries when + # releases are built from different git paths, unless -buildid= is added + # to ldflags. + # For more details, see: https://github.com/oasislabs/goreleaser/issues/1. + - -buildid= - -X github.com/oasislabs/oasis-core/go/common/version.SoftwareVersion={{.Env.VERSION}} goos: - linux diff --git a/README.md b/README.md index 2120ece8997..d427409657a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Oasis Core [![Build status](https://badge.buildkite.com/15a8faccee0d5b5ab1af7e75eb6f9daf2d493c543fbc67dce5.svg?branch=master)](https://buildkite.com/oasislabs/oasis-core-ci) +[![CI lint status](https://github.com/oasislabs/oasis-core/workflows/ci-lint/badge.svg)](https://github.com/oasislabs/oasis-core/actions?query=workflow:ci-lint) +[![CI reproducibility status](https://github.com/oasislabs/oasis-core/workflows/ci-reproducibility/badge.svg)](https://github.com/oasislabs/oasis-core/actions?query=workflow:ci-reproducibility) [![Coverage Status](https://coveralls.io/repos/github/oasislabs/oasis-core/badge.svg?t=HsLWgi)](https://coveralls.io/github/oasislabs/oasis-core) Rust [![codecov](https://codecov.io/gh/oasislabs/oasis-core/branch/master/graph/badge.svg?token=DqjRsufMqf)](https://codecov.io/gh/oasislabs/oasis-core) Go [![GoDoc](https://godoc.org/github.com/oasislabs/oasis-core?status.svg)](https://godoc.org/github.com/oasislabs/oasis-core) diff --git a/common.mk b/common.mk index 34de3be81fb..0450dbf19da 100644 --- a/common.mk +++ b/common.mk @@ -101,35 +101,12 @@ GO_TEST_HELPER_URKEL_PATH := storage/mkvs/urkel/interop/urkel-test-helpers # Instruct GoReleaser to create a "snapshot" release by default. GORELEASER_ARGS ?= release --snapshot --rm-dist - -# If running inside GitHub Actions, create a real release. -ifeq ($(GITHUB_ACTIONS), true) - -# Auxiliary variable that defines a new line for later substitution. -define newline - - -endef - -# GitHub release' text in Markdown format. -define RELEASE_TEXT = -For a list of changes in this release, see the [Change Log]. - -*NOTE: If you are upgrading from an earlier release, please **carefully review** -the [Change Log] for **Removals and Breaking changes**.* - -If you would like to become a node operator for the Oasis Network, see the -[Operator Docs](https://docs.oasis.dev/operators/overview.html). - -[Change Log]: https://github.com/oasislabs/oasis-core/blob/v$(VERSION)/CHANGELOG.md - -endef - -# Temporary file with GitHub release's text. +# If the appropriate environment variable is set, create a real release. +ifeq ($(OASIS_CORE_REAL_RELEASE), true) +# Create temporary file with GitHub release's text. _RELEASE_NOTES_FILE := $(shell mktemp /tmp/oasis-core.XXXXX) _ := $(shell printf "$(subst ",\",$(subst $(newline),\n,$(RELEASE_TEXT)))" > $(_RELEASE_NOTES_FILE)) GORELEASER_ARGS = release --release-notes $(_RELEASE_NOTES_FILE) - endif # Helper that ensures $(NEXT_VERSION) variable is not empty. @@ -169,3 +146,23 @@ define ENSURE_NEXT_VERSION_IN_CHANGELOG = exit 1; \ fi endef + +# Auxiliary variable that defines a new line for later substitution. +define newline + + +endef + +# GitHub release' text in Markdown format. +define RELEASE_TEXT = +For a list of changes in this release, see the [Change Log]. + +*NOTE: If you are upgrading from an earlier release, please **carefully review** +the [Change Log] for **Removals and Breaking changes**.* + +If you would like to become a node operator for the Oasis Network, see the +[Operator Docs](https://docs.oasis.dev/operators/overview.html). + +[Change Log]: https://github.com/oasislabs/oasis-core/blob/v$(VERSION)/CHANGELOG.md + +endef