diff --git a/.buildkite/code-skip.pipeline.yml b/.buildkite/code-skip.pipeline.yml new file mode 100644 index 00000000000..ff08e3119db --- /dev/null +++ b/.buildkite/code-skip.pipeline.yml @@ -0,0 +1,10 @@ +## +# Code-skip pipeline +## +# +# Buildkite pipeline for skipping running code-related linters and tests. +# + +steps: + - label: No code-related changes detected, skipping Code pipeline + command: exit 0 diff --git a/.buildkite/pipeline.yml b/.buildkite/code.pipeline.yml similarity index 94% rename from .buildkite/pipeline.yml rename to .buildkite/code.pipeline.yml index f9b8760bca7..9d15e022bde 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/code.pipeline.yml @@ -1,6 +1,10 @@ -################ -# Build pipeline -################ +## +# Code pipeline +## +# +# Main Buildkite pipeline for running code-related linters and tests. +# + docker_plugin_default_config: &docker_plugin_default_config image: "oasislabs/testing:0.3.0" always_pull: true @@ -90,7 +94,6 @@ steps: # .buildkite/scripts/download_utils.sh. - label: Build Go node command: - - . .buildkite/scripts/skip_if_only_docs_changes.sh - .buildkite/go/build.sh # Upload the built artifacts. @@ -109,7 +112,6 @@ steps: # .buildkite/scripts/download_utils.sh. - label: Build Rust runtime loader command: - - . .buildkite/scripts/skip_if_only_docs_changes.sh - .buildkite/rust/build_generic.sh /workdir -p oasis-core-runtime-loader - .buildkite/rust/build_generic.sh /workdir -p test-long-term-client - .buildkite/rust/build_generic.sh /workdir -p simple-keyvalue-client @@ -136,7 +138,6 @@ steps: # .buildkite/rust/test_runtime_and_gateway.sh and .buildkite/scripts/download_utils.sh. - label: Build key manager runtime command: - - . .buildkite/scripts/skip_if_only_docs_changes.sh - .buildkite/rust/build_runtime.sh keymanager-runtime - .buildkite/rust/build_runtime.sh tests/runtimes/simple-keyvalue @@ -157,7 +158,6 @@ steps: ########### - label: Test Rust crates command: - - . .buildkite/scripts/skip_if_only_docs_changes.sh # Build storage interoperability test helpers first. - make build-helpers - export OASIS_STORAGE_PROTOCOL_SERVER_BINARY=$(realpath go/storage/mkvs/urkel/interop/urkel-test-helpers) @@ -177,7 +177,6 @@ steps: - label: Test fuzz builds branches: master command: - - . .buildkite/scripts/skip_if_only_docs_changes.sh # TODO: Consider making this a part of the development Docker image. - go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build - make -C go build-fuzz @@ -189,7 +188,6 @@ steps: ##################################### - label: Test Go node command: - - . .buildkite/scripts/skip_if_only_docs_changes.sh - .buildkite/go/test_and_coverage.sh artifact_paths: - coverage-misc.txt @@ -205,7 +203,6 @@ steps: parallelism: 6 timeout_in_minutes: 9 command: - - . .buildkite/scripts/skip_if_only_docs_changes.sh - .buildkite/scripts/download_e2e_test_artifacts.sh - .buildkite/scripts/test_e2e.sh artifact_paths: @@ -226,7 +223,6 @@ steps: parallelism: 5 timeout_in_minutes: 19 command: - - . .buildkite/scripts/skip_if_only_docs_changes.sh - .buildkite/scripts/download_e2e_test_artifacts.sh - .buildkite/scripts/test_e2e.sh artifact_paths: @@ -281,7 +277,6 @@ steps: - label: "Merge and upload coverage" command: - - . .buildkite/scripts/skip_if_only_docs_changes.sh - .buildkite/scripts/merge_coverage.sh - .buildkite/scripts/upload_coverage.sh artifact_paths: diff --git a/.buildkite/pipeline.sh b/.buildkite/pipeline.sh new file mode 100755 index 00000000000..81844744e6c --- /dev/null +++ b/.buildkite/pipeline.sh @@ -0,0 +1,45 @@ +#!/bin/bash +## +# Dynamic Buildkite pipeline generator. +## +# +# It outputs valid Buildkite pipeline in YAML format. +# +# To use it, define the following Steps under your Buildkite's Pipeline Settings: +# +# steps: +# - command: .buildkite/pipeline.sh | buildkite-agent pipeline upload +# label: ":pipeline: Upload" +# +# For more details, see: +# https://buildkite.com/docs/pipelines/defining-steps#dynamic-pipelines. +# + +set -eux + +# Helper that ensures the build is triggered for a pull request and that there +# are no code-related changes compared to the pull request's base branch. +pr_and_no_code_related_changes() { + # Check if the build was triggered for a pull request. + if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then + return 1 + fi + # Get the list of changes files, excluding changes unrelated to code. + # NOTE: The exclude patterns below use git's pathspec syntax: + # https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefpathspecapathspec. + git diff --name-only --exit-code "refs/remotes/origin/$BUILDKITE_PULL_REQUEST_BASE_BRANCH.." 1>&2 -- \ + ':(exclude)*.md' \ + ':(exclude).changelog/' \ + ':(exclude).github/' \ + ':(exclude).gitlint' \ + ':(exclude).markdownlint.yml' \ + ':(exclude).punch_config.py' \ + ':(exclude)docs/' \ + ':(exclude)towncrier.toml' +} + +if pr_and_no_code_related_changes; then + cat .buildkite/code-skip.pipeline.yml +else + cat .buildkite/code.pipeline.yml +fi diff --git a/.buildkite/scripts/skip_if_only_docs_changes.sh b/.buildkite/scripts/skip_if_only_docs_changes.sh deleted file mode 100644 index 65c8c2459f7..00000000000 --- a/.buildkite/scripts/skip_if_only_docs_changes.sh +++ /dev/null @@ -1,23 +0,0 @@ -# Source this file rather than fork-exec-ing it. - -only_docs_changes() { - # https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefpathspecapathspec - # Some non-docs things related to workflows are also in .github/, but this script currently - # is specific to buildkite, where we wouldn't care about changes there anyway. If we later - # use this in other CI places, we can narrow it down. - test -n "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" && - git diff --quiet "refs/remotes/origin/$BUILDKITE_PULL_REQUEST_BASE_BRANCH.." -- \ - ':(exclude)*.md' \ - ':(exclude).changelog/' \ - ':(exclude).github/' \ - ':(exclude).gitlint' \ - ':(exclude).markdownlint.yml' \ - ':(exclude).punch_config.py' \ - ':(exclude)docs/' \ - ':(exclude)towncrier.toml' -} - -if only_docs_changes; then - echo "Only docs changes. Skipping" - exit 0 -fi diff --git a/.changelog/2689.internal.md b/.changelog/2689.internal.md new file mode 100644 index 00000000000..96c08a4e1e4 --- /dev/null +++ b/.changelog/2689.internal.md @@ -0,0 +1 @@ +Bump minimum golang version to 1.13.8 diff --git a/.changelog/2702.internal.md b/.changelog/2702.internal.md new file mode 100644 index 00000000000..65de41d29a4 --- /dev/null +++ b/.changelog/2702.internal.md @@ -0,0 +1,6 @@ +ci: Skip some steps for non-code changes + +When one makes a pull request that e.g. only adds documentation or +assembles the Change Log from fragments, all the *heavy* Buildkite +pipeline steps (e.g. Go/Rust building, Go tests, E2E tests) should be +skipped. diff --git a/.changelog/2704.internal.md b/.changelog/2704.internal.md new file mode 100644 index 00000000000..0404495776d --- /dev/null +++ b/.changelog/2704.internal.md @@ -0,0 +1 @@ +github: Don't trigger ci-reproducibility workflow for pull requests diff --git a/.github/workflows/ci-reproducibility.yml b/.github/workflows/ci-reproducibility.yml index fda25745b59..8d0c8dc666e 100644 --- a/.github/workflows/ci-reproducibility.yml +++ b/.github/workflows/ci-reproducibility.yml @@ -8,12 +8,9 @@ on: branches: - master - stable/* - # Or when a pull request event occurs for a pull request against one of the - # matched branches. - pull_request: - branches: - - master - - stable/* + # Or every day at 00:00 UTC. + schedule: + - cron: "0 0 * * *" jobs: diff --git a/README.md b/README.md index 38453bf4a74..b0a60b9151a 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ Prerequisites: sudo apt install bubblewrap gcc g++ protobuf-compiler make cmake libssl-dev libseccomp-dev ``` -* [Go](https://golang.org) (at least version 1.13). +* [Go](https://golang.org) (at least version 1.13.8). If your distribution provides a new-enough version of Go, just use that. @@ -87,18 +87,18 @@ Prerequisites: * [ensure `$GOPATH/bin` is in your `PATH`]( https://tip.golang.org/doc/code.html#GOPATH), * [install the desired version of Go]( - https://golang.org/doc/install#extra_versions), e.g. 1.13, with: + https://golang.org/doc/install#extra_versions), e.g. 1.13.8, with: ``` - go get golang.org/dl/go1.13 - go1.13 download + go get golang.org/dl/go1.13.8 + go1.13.8 download ``` * instruct the build system to use this particular version of Go by setting the `OASIS_GO` environment variable in your `~/.bashrc`: ``` - export OASIS_GO=go1.13 + export OASIS_GO=go1.13.8 ``` * [Rust](https://www.rust-lang.org) and the nightly toolchain. diff --git a/docker/development/Dockerfile b/docker/development/Dockerfile index 2ed996c0f17..b7458b2eceb 100644 --- a/docker/development/Dockerfile +++ b/docker/development/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 # Package versions. -ARG GO_VERSION=1.13.4 +ARG GO_VERSION=1.13.8 ARG GO_PROTOC_VERSION=3.6.1 ARG GO_PROTOC_GEN_GO_VERSION=1.2.0 ARG GOLANGCILINT_VERSION=1.18.0