From f6b6e5254d0142c219b45f7cd20c296e0aee57c1 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 24 Feb 2020 10:41:03 +0100 Subject: [PATCH 1/3] Define Bazel version in one shared location Instead of each CI pipeline separately defining the Bazel version, we define it in one shared location `.ci/bazelversion`. Ideally, we would use `.bazelversion` in the top-level which would be compatible with [bazelisk](https://github.com/bazelbuild/bazelisk). Unfortunately, this is incompatible with the way Bazel is packaged in nixpkgs right now. See, `.ci/check-bazel-version`. The pipelines that use nixpkgs to provision Bazel check that the Bazel version matches using `.ci/check-bazel-version`. The pipelines that fetch a Bazel binary distribution use `.ci/fetch-bazel-bindist` which in turn consults the `.ci/bazelversion` file. To avoid issues with corrupted downloads we check the downloaded bindist against expected sha256 hashes defined in `.ci/bazel-sha256`. The files `.ci/bazelversion` and `.ci/bazel-sha256` need to be kept in sync. The script `.ci/update-bazel-version` helps with this. Given a Bazel version number it will update both files. The nixpkgs revision needs to be updated separately. The `MAINTAINERS.md` file has been updated accordingly. --- .buildkite/bazel-sha256 | 1 - .buildkite/bindists-pipeline | 2 +- .buildkite/check-bazel-version | 11 ------- .buildkite/fetch-bazel-bindist | 10 ------- .buildkite/pipeline.yml | 4 +-- .ci/bazel-sha256 | 3 ++ .buildkite/bazel-version => .ci/bazelversion | 0 .ci/check-bazel-version | 14 +++++++++ .ci/fetch-bazel-bindist | 30 ++++++++++++++++++++ .ci/update-bazel-version | 25 ++++++++++++++++ .circleci/config.yml | 6 ++++ .netlify/install.sh | 9 ++---- MAINTAINERS.md | 4 +-- azure-pipelines.yml | 7 ++--- 14 files changed, 88 insertions(+), 38 deletions(-) delete mode 100644 .buildkite/bazel-sha256 delete mode 100755 .buildkite/check-bazel-version delete mode 100755 .buildkite/fetch-bazel-bindist create mode 100644 .ci/bazel-sha256 rename .buildkite/bazel-version => .ci/bazelversion (100%) create mode 100755 .ci/check-bazel-version create mode 100755 .ci/fetch-bazel-bindist create mode 100755 .ci/update-bazel-version diff --git a/.buildkite/bazel-sha256 b/.buildkite/bazel-sha256 deleted file mode 100644 index 678efe0a6..000000000 --- a/.buildkite/bazel-sha256 +++ /dev/null @@ -1 +0,0 @@ -4df79462c6c3ecdeeee7af99fc269b52ab1aa4828ef3bc359c1837d3fafeeee7 bazel diff --git a/.buildkite/bindists-pipeline b/.buildkite/bindists-pipeline index fa7ecaccb..f8be8a9d4 100755 --- a/.buildkite/bindists-pipeline +++ b/.buildkite/bindists-pipeline @@ -5,7 +5,7 @@ set -euo pipefail -BAZEL_DIR="$(.buildkite/fetch-bazel-bindist)" +BAZEL_DIR="$(.ci/fetch-bazel-bindist)" REPO_CACHE="$HOME/repo_cache" trap "rm -rf '$BAZEL_DIR'" EXIT diff --git a/.buildkite/check-bazel-version b/.buildkite/check-bazel-version deleted file mode 100755 index 3796f760c..000000000 --- a/.buildkite/check-bazel-version +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -# nixpkgs Bazel version ends on '- (@non-git)'. -VERSION_EXPECTED="bazel $(cat "$DIR/bazel-version")- (@non-git)" -VERSION_ACTUAL=$(bazel version --gnu_format) -[[ "$VERSION_ACTUAL" = "$VERSION_EXPECTED" ]] || { - echo "Expected Bazel version $VERSION_EXPECTED but found $VERSION_ACTUAL." >&2 - exit 1 -} diff --git a/.buildkite/fetch-bazel-bindist b/.buildkite/fetch-bazel-bindist deleted file mode 100755 index 609c41602..000000000 --- a/.buildkite/fetch-bazel-bindist +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -VERSION=$(cat "$DIR/bazel-version") -INSTALL="$(mktemp -d)" -curl -L "https://github.com/bazelbuild/bazel/releases/download/$VERSION/bazel-$VERSION-linux-x86_64" -o "$INSTALL/bazel" >&2 -(cd "$INSTALL" && sha256sum --quiet -c "$DIR/bazel-sha256") >&2 -chmod +x "$INSTALL/bazel" -echo -n "$INSTALL" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 0a9cb03f4..d43f92544 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -6,9 +6,9 @@ steps: nix-shell --arg docTools false --pure --run ' set -e # Ensure that the Nixpkgs bazel version matches the one specified in - # `.buildkite/bazel-version` and fetched in + # `.bazelversion` and fetched in # `.buildkite/bindists-pipeline` for the bindists version. - .buildkite/check-bazel-version + .ci/check-bazel-version ./tests/run-start-script.sh --use-nix bazel build --config ci //tests:run-tests ./bazel-ci-bin/tests/run-tests diff --git a/.ci/bazel-sha256 b/.ci/bazel-sha256 new file mode 100644 index 000000000..2facdeba9 --- /dev/null +++ b/.ci/bazel-sha256 @@ -0,0 +1,3 @@ +cc7b3ff6f4bfd6bc2121a80656afec66ee57713e8b88e9d2fb58b4eddf271268 bazel-2.0.0-windows-x86_64.exe +4df79462c6c3ecdeeee7af99fc269b52ab1aa4828ef3bc359c1837d3fafeeee7 bazel-2.0.0-linux-x86_64 +3eca4c96cfda97a9d5f8d3d0dec4155a5cc5ff339b10d3f35213c398bf13881e bazel-2.0.0-darwin-x86_64 diff --git a/.buildkite/bazel-version b/.ci/bazelversion similarity index 100% rename from .buildkite/bazel-version rename to .ci/bazelversion diff --git a/.ci/check-bazel-version b/.ci/check-bazel-version new file mode 100755 index 000000000..d4cc54b11 --- /dev/null +++ b/.ci/check-bazel-version @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Ideally we would place .bazelversion in the top-level so it could be used by +# tools like bazelisk. However, this breaks the nixpkgs provided Bazel +# installation, see https://github.com/NixOS/nixpkgs/issues/80950. +VERSION_EXPECTED="bazel $(cat "$DIR/bazelversion")" +VERSION_ACTUAL=$(bazel version --gnu_format) +# nixpkgs Bazel version ends on '- (@non-git)'. +[[ "$VERSION_ACTUAL" =~ ^"$VERSION_EXPECTED"("- (@non-git)")?$ ]] || { + echo "Expected Bazel version $VERSION_EXPECTED but found $VERSION_ACTUAL." >&2 + exit 1 +} diff --git a/.ci/fetch-bazel-bindist b/.ci/fetch-bazel-bindist new file mode 100755 index 000000000..fcb9bb5b8 --- /dev/null +++ b/.ci/fetch-bazel-bindist @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail +TOP="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )" + +case "$OSTYPE" in + linux-gnu) OS=linux-x86_64; EXT=;; + darwin) OS=darwin-x86_64; EXT=;; + cygwin|msys|win32) OS=windows-x86_64; EXT=.exe;; + **) echo "Unknown operating system" >&2; exit 1;; +esac +VERSION=$(cat "$TOP/.ci/bazelversion") +INSTALL="$(mktemp -d)" +(cd "$INSTALL" && curl -LO "https://github.com/bazelbuild/bazel/releases/download/$VERSION/bazel-$VERSION-$OS$EXT" >&2) +if [[ $OS = windows-x86_64 ]]; then + # sha256sum -c does not behave as expected on Windows. It produces + # + # sha256sum: /d/a/1/s/.ci/bazel-sha256: no file was verified + # + # even when the hashes match upon manual inspection. + SHA256="$(cd "$INSTALL" && sha256sum --text bazel-$VERSION-$OS$EXT)" + if ! grep -q "$SHA256" "$TOP/.ci/bazel-sha256" >&2; then + echo "sha256 mismatch (compare .ci/bazel-sha256): $SHA256" >&2 + exit 1 + fi +else + (cd "$INSTALL" && sha256sum --quiet --ignore-missing -c "$TOP/.ci/bazel-sha256") >&2 +fi +mv "$INSTALL/bazel-$VERSION-$OS$EXT" "$INSTALL/bazel$EXT" +chmod +x "$INSTALL/bazel$EXT" +echo -n "$INSTALL" diff --git a/.ci/update-bazel-version b/.ci/update-bazel-version new file mode 100755 index 000000000..f9b332a72 --- /dev/null +++ b/.ci/update-bazel-version @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail +TOP="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )" + +usage() { + cat >&2 <"$TOP/.ci/bazelversion" +mkdir -p "$TOP/.ci" +>"$TOP/.ci/bazel-sha256" +for OSEXT in windows-x86_64.exe linux-x86_64 darwin-x86_64; do + URL="https://github.com/bazelbuild/bazel/releases/download/$VERSION/bazel-$VERSION-$OSEXT.sha256" + curl -L "$URL" >>"$TOP/.ci/bazel-sha256" +done diff --git a/.circleci/config.yml b/.circleci/config.yml index f22fd8db2..91686cdcc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,6 +57,12 @@ jobs: - v1-rules_haskell-cache-{{ .Branch }}- - v1-rules_haskell-cache-master- + - run: + name: Check Bazel version + shell: /bin/bash -eilo pipefail + command: | + nix-shell --arg docTools false --pure --run \ + '.ci/check-bazel-version' - run: name: Prefetch Stackage snapshot shell: /bin/bash -eilo pipefail diff --git a/.netlify/install.sh b/.netlify/install.sh index 8d02375d0..3e944ab57 100755 --- a/.netlify/install.sh +++ b/.netlify/install.sh @@ -1,9 +1,4 @@ #!/bin/sh - set -eux - -V=2.0.0 - -curl -LO https://github.com/bazelbuild/bazel/releases/download/$V/bazel-$V-installer-linux-x86_64.sh -chmod +x bazel-$V-installer-linux-x86_64.sh -./bazel-$V-installer-linux-x86_64.sh --user +INSTALL="$(.ci/fetch-bazel-bindist)" +mv "$INSTALL" "$HOME/bin" diff --git a/MAINTAINERS.md b/MAINTAINERS.md index d3216fbf5..4ebf731e2 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -51,7 +51,8 @@ We strive to always test against the latest non-LTS release nonetheless, so bumping bazel regularly is required. -- [ ] Bump bazel download link for bazel in `azure-pipelines.yml` +- [ ] Use `.ci/update-bazel-version` to update `.ci/bazelversion` and + `.ci/bazel-sha256`. - [ ] Update all bazel rules dependencies in `WORKSPACE` (e.g. `io_bazel_stardoc`) - [ ] Update bazel in nixpkgs and bump `nixpkgs/default.nix` @@ -60,4 +61,3 @@ nonetheless, so bumping bazel regularly is required. - Bump `MIN_BAZEL_*` in `start` - TODO - [ ] Add update notice to `CHANGELOG` -- [ ] Bump bazel version in .netlify/install.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 271b662fd..655194cc1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,11 +9,10 @@ jobs: architecture: 'x64' - bash: | set -e - curl -LO https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-windows-x86_64.exe - mv bazel-*.exe bazel.exe - mkdir /c/bazel - mv bazel.exe /c/bazel + INSTALL="$(.ci/fetch-bazel-bindist)" + mv "$INSTALL" /c/bazel /c/bazel/bazel.exe info release + PATH="/c/bazel:$PATH" .ci/check-bazel-version displayName: 'Install Bazel' From 39b10c8a25d04d91951899096d15291e5fd9863e Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 24 Feb 2020 16:40:46 +0100 Subject: [PATCH 2/3] Netlify's sha256 does not support --ignore-missing To work around this we store separate `.sha256` files for each platform. --- .ci/bazel-2.0.0-darwin-x86_64.sha256 | 1 + .ci/bazel-2.0.0-linux-x86_64.sha256 | 1 + .ci/bazel-2.0.0-windows-x86_64.exe.sha256 | 1 + .ci/bazel-sha256 | 3 --- .ci/fetch-bazel-bindist | 4 ++-- .ci/update-bazel-version | 5 ++--- MAINTAINERS.md | 2 +- 7 files changed, 8 insertions(+), 9 deletions(-) create mode 100644 .ci/bazel-2.0.0-darwin-x86_64.sha256 create mode 100644 .ci/bazel-2.0.0-linux-x86_64.sha256 create mode 100644 .ci/bazel-2.0.0-windows-x86_64.exe.sha256 delete mode 100644 .ci/bazel-sha256 diff --git a/.ci/bazel-2.0.0-darwin-x86_64.sha256 b/.ci/bazel-2.0.0-darwin-x86_64.sha256 new file mode 100644 index 000000000..19c361612 --- /dev/null +++ b/.ci/bazel-2.0.0-darwin-x86_64.sha256 @@ -0,0 +1 @@ +3eca4c96cfda97a9d5f8d3d0dec4155a5cc5ff339b10d3f35213c398bf13881e bazel-2.0.0-darwin-x86_64 diff --git a/.ci/bazel-2.0.0-linux-x86_64.sha256 b/.ci/bazel-2.0.0-linux-x86_64.sha256 new file mode 100644 index 000000000..11524a031 --- /dev/null +++ b/.ci/bazel-2.0.0-linux-x86_64.sha256 @@ -0,0 +1 @@ +4df79462c6c3ecdeeee7af99fc269b52ab1aa4828ef3bc359c1837d3fafeeee7 bazel-2.0.0-linux-x86_64 diff --git a/.ci/bazel-2.0.0-windows-x86_64.exe.sha256 b/.ci/bazel-2.0.0-windows-x86_64.exe.sha256 new file mode 100644 index 000000000..ab65f4c49 --- /dev/null +++ b/.ci/bazel-2.0.0-windows-x86_64.exe.sha256 @@ -0,0 +1 @@ +cc7b3ff6f4bfd6bc2121a80656afec66ee57713e8b88e9d2fb58b4eddf271268 bazel-2.0.0-windows-x86_64.exe diff --git a/.ci/bazel-sha256 b/.ci/bazel-sha256 deleted file mode 100644 index 2facdeba9..000000000 --- a/.ci/bazel-sha256 +++ /dev/null @@ -1,3 +0,0 @@ -cc7b3ff6f4bfd6bc2121a80656afec66ee57713e8b88e9d2fb58b4eddf271268 bazel-2.0.0-windows-x86_64.exe -4df79462c6c3ecdeeee7af99fc269b52ab1aa4828ef3bc359c1837d3fafeeee7 bazel-2.0.0-linux-x86_64 -3eca4c96cfda97a9d5f8d3d0dec4155a5cc5ff339b10d3f35213c398bf13881e bazel-2.0.0-darwin-x86_64 diff --git a/.ci/fetch-bazel-bindist b/.ci/fetch-bazel-bindist index fcb9bb5b8..805e8bc6a 100755 --- a/.ci/fetch-bazel-bindist +++ b/.ci/fetch-bazel-bindist @@ -18,12 +18,12 @@ if [[ $OS = windows-x86_64 ]]; then # # even when the hashes match upon manual inspection. SHA256="$(cd "$INSTALL" && sha256sum --text bazel-$VERSION-$OS$EXT)" - if ! grep -q "$SHA256" "$TOP/.ci/bazel-sha256" >&2; then + if ! grep -q "$SHA256" "$TOP/.ci/bazel-$VERSION-$OS$EXT.sha256" >&2; then echo "sha256 mismatch (compare .ci/bazel-sha256): $SHA256" >&2 exit 1 fi else - (cd "$INSTALL" && sha256sum --quiet --ignore-missing -c "$TOP/.ci/bazel-sha256") >&2 + (cd "$INSTALL" && sha256sum --quiet -c "$TOP/.ci/bazel-$VERSION-$OS$EXT.sha256") >&2 fi mv "$INSTALL/bazel-$VERSION-$OS$EXT" "$INSTALL/bazel$EXT" chmod +x "$INSTALL/bazel$EXT" diff --git a/.ci/update-bazel-version b/.ci/update-bazel-version index f9b332a72..40dc6328c 100755 --- a/.ci/update-bazel-version +++ b/.ci/update-bazel-version @@ -5,7 +5,7 @@ TOP="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )" usage() { cat >&2 <"$TOP/.ci/bazelversion" mkdir -p "$TOP/.ci" ->"$TOP/.ci/bazel-sha256" for OSEXT in windows-x86_64.exe linux-x86_64 darwin-x86_64; do URL="https://github.com/bazelbuild/bazel/releases/download/$VERSION/bazel-$VERSION-$OSEXT.sha256" - curl -L "$URL" >>"$TOP/.ci/bazel-sha256" + (cd "$TOP/.ci" && curl -LO "$URL") done diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 4ebf731e2..25c72ff1d 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -52,7 +52,7 @@ We strive to always test against the latest non-LTS release nonetheless, so bumping bazel regularly is required. - [ ] Use `.ci/update-bazel-version` to update `.ci/bazelversion` and - `.ci/bazel-sha256`. + `.ci/bazel-*.sha256`. - [ ] Update all bazel rules dependencies in `WORKSPACE` (e.g. `io_bazel_stardoc`) - [ ] Update bazel in nixpkgs and bump `nixpkgs/default.nix` From 2a297e72ec469dafbcfe2180df9551b1a7b1bc4e Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 24 Feb 2020 16:42:17 +0100 Subject: [PATCH 3/3] Fix sha256sum issue on Windows git was automatically converting `\n` lineendings to `\r\n`, which confused `sha256sum`. By declaring `.sha256` files as binary `.gitattributes` we avoid this issue. --- .ci/fetch-bazel-bindist | 15 +-------------- .gitattributes | 3 +++ 2 files changed, 4 insertions(+), 14 deletions(-) create mode 100644 .gitattributes diff --git a/.ci/fetch-bazel-bindist b/.ci/fetch-bazel-bindist index 805e8bc6a..d116d1c35 100755 --- a/.ci/fetch-bazel-bindist +++ b/.ci/fetch-bazel-bindist @@ -11,20 +11,7 @@ esac VERSION=$(cat "$TOP/.ci/bazelversion") INSTALL="$(mktemp -d)" (cd "$INSTALL" && curl -LO "https://github.com/bazelbuild/bazel/releases/download/$VERSION/bazel-$VERSION-$OS$EXT" >&2) -if [[ $OS = windows-x86_64 ]]; then - # sha256sum -c does not behave as expected on Windows. It produces - # - # sha256sum: /d/a/1/s/.ci/bazel-sha256: no file was verified - # - # even when the hashes match upon manual inspection. - SHA256="$(cd "$INSTALL" && sha256sum --text bazel-$VERSION-$OS$EXT)" - if ! grep -q "$SHA256" "$TOP/.ci/bazel-$VERSION-$OS$EXT.sha256" >&2; then - echo "sha256 mismatch (compare .ci/bazel-sha256): $SHA256" >&2 - exit 1 - fi -else - (cd "$INSTALL" && sha256sum --quiet -c "$TOP/.ci/bazel-$VERSION-$OS$EXT.sha256") >&2 -fi +(cd "$INSTALL" && sha256sum --quiet -c "$TOP/.ci/bazel-$VERSION-$OS$EXT.sha256") >&2 mv "$INSTALL/bazel-$VERSION-$OS$EXT" "$INSTALL/bazel$EXT" chmod +x "$INSTALL/bazel$EXT" echo -n "$INSTALL" diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..cfb0158a3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Prevent git from automatically introducing \r characters in .sha256 files. +# Otherwise msys sha256sum fails. +*.sha256 binary