ci/ is now more reproducible locally

README.md

@@ -129,6 +129,11 @@ Run the test suite:
 $ cargo test
 ```
 
+To emulate all the tests that will run on a Pull Request, run:
+```bash
+$ ./ci/run-local.sh
+```
+
 Debugging
 ---
 
@@ -169,8 +174,10 @@ Code coverage
 To generate code coverage statistics, run kcov via Docker:
 
 ```bash
-$ docker run -it --rm --security-opt seccomp=unconfined --volume "$PWD:/volume" elmtai/docker-rust-kcov
+$ ./ci/coverage.sh
 ```
+The coverage report will be written to `./target/cov/index.html`
+
 
 Why coverage? While most see coverage as a code quality metric, we see it primarily as a developer
 productivity metric. When a developer makes a change to the codebase, presumably it's a *solution* to

ci/.gitignore

@@ -0,0 +1,2 @@
+/node_modules/
+/package-lock.json

ci/buildkite.yml

@@ -1,43 +1,16 @@
 steps:
   - command: "ci/coverage.sh"
-    label: "coverage [public]"
-    # TODO: Run coverage in a docker image rather than assuming kcov/cargo-kcov
-    #       is installed on the build agent...
-    #plugins:
-    #  docker#v1.1.1:
-    #    image: "rust"
-    #    user: "998:997" # buildkite-agent:buildkite-agent
-    #    environment:
-    #      - CODECOV_TOKEN
-  - command: "ci/test-stable.sh"
-    label: "stable [public]"
-    plugins:
-      docker#v1.1.1:
-        image: "rust"
-        user: "998:997" # buildkite-agent:buildkite-agent
-  - command: "ci/test-nightly.sh || true"
-    label: "nightly - FAILURES IGNORED [public]"
-    plugins:
-      docker#v1.1.1:
-        image: "rustlang/rust:nightly"
-        user: "998:997" # buildkite-agent:buildkite-agent
-  - command: "ci/test-ignored.sh"
-    label: "ignored [public]"
-    plugins:
-      docker#v1.1.1:
-        image: "rust"
-        user: "998:997" # buildkite-agent:buildkite-agent
+    name: "coverage [public]"
+  - command: "ci/docker-run.sh rust ci/test-stable.sh"
+    name: "stable [public]"
+  - command: "ci/docker-run.sh rustlang/rust:nightly ci/test-nightly.sh || true"
+    name: "nightly - FAILURES IGNORED [public]"
+  - command: "ci/docker-run.sh rust ci/test-ignored.sh"
+    name: "ignored [public]"
   - command: "ci/test-cuda.sh"
-    label: "cuda"
+    name: "cuda"
   - command: "ci/shellcheck.sh"
-    label: "shellcheck [public]"
+    name: "shellcheck [public]"
   - wait
   - command: "ci/publish.sh"
-    label: "publish release artifacts"
-    plugins:
-      docker#v1.1.1:
-        image: "rust"
-        user: "998:997" # buildkite-agent:buildkite-agent
-        environment:
-          - BUILDKITE_TAG
-          - CRATES_IO_TOKEN
+    name: "publish release artifacts"

ci/coverage.sh

@@ -2,25 +2,20 @@
 
 cd "$(dirname "$0")/.."
 
-if [[ -r ~/.cargo/env ]]; then
-  # Pick up local install of kcov/cargo-kcov
-  # shellcheck disable=SC1090
-  source ~/.cargo/env
-fi
-
-rustc --version
-cargo --version
-kcov --version
-cargo-kcov --version
+ci/docker-run.sh evilmachines/rust-cargo-kcov \
+  bash -exc "\
+    export RUST_BACKTRACE=1; \
+    cargo build --verbose; \
+    cargo kcov --lib --verbose; \
+  "
 
-export RUST_BACKTRACE=1
-cargo build
-cargo kcov --lib
+echo Coverage report:
+ls -l target/cov/index.html
 
 if [[ -z "$CODECOV_TOKEN" ]]; then
   echo CODECOV_TOKEN undefined
-  exit 1
+else
+  bash <(curl -s https://codecov.io/bash)
 fi
 
-bash <(curl -s https://codecov.io/bash)
 exit 0

ci/docker-run.sh

@@ -0,0 +1,41 @@
+#!/bin/bash -e
+
+usage() {
+  echo "Usage: $0 [docker image name] [command]"
+  echo
+  echo Runs command in the specified docker image with
+  echo a CI-appropriate environment
+  echo
+}
+
+cd "$(dirname "$0")/.."
+
+IMAGE="$1"
+if [[ -z "$IMAGE" ]]; then
+  echo Error: image not defined
+  exit 1
+fi
+
+docker pull "$IMAGE"
+shift
+
+ARGS=(--workdir /solana --volume "$PWD:/solana" --rm)
+
+ARGS+=(--env "CARGO_HOME=/solana/.cargo")
+
+# kcov tries to set the personality of the binary which docker
+# doesn't allow by default.
+ARGS+=(--security-opt "seccomp=unconfined")
+
+# Ensure files are created with the current host uid/gid
+ARGS+=(--user "$(id -u):$(id -g)")
+
+# Environment variables to propagate into the container
+ARGS+=(
+  --env BUILDKITE_TAG
+  --env CODECOV_TOKEN
+  --env CRATES_IO_TOKEN
+)
+
+set -x
+docker run "${ARGS[@]}" "$IMAGE" "$@"

ci/publish.sh

@@ -12,8 +12,8 @@ if [[ -z "$CRATES_IO_TOKEN" ]]; then
   exit 1
 fi
 
-cargo package
 # TODO: Ensure the published version matches the contents of BUILDKITE_TAG
-cargo publish --token "$CRATES_IO_TOKEN"
+ci/docker-run.sh rust \
+  bash -exc "cargo package; cargo publish --token $CRATES_IO_TOKEN"
 
 exit 0

ci/run-local.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+#
+# Run the entire buildkite CI pipeline locally for pre-testing before sending a
+# Github pull request
+#
+
+cd "$(dirname "$0")/.."
+BKRUN=ci/node_modules/.bin/bkrun
+
+if [[ ! -x $BKRUN ]]; then
+  (
+    set -x
+    cd ci/
+    npm install bkrun
+  )
+fi
+
+set -x
+./ci/node_modules/.bin/bkrun ci/buildkite.yml

ci/shellcheck.sh

@@ -5,10 +5,7 @@
 cd "$(dirname "$0")/.."
 
 set -x
-docker pull koalaman/shellcheck
-find . -name "*.sh" -not -regex ".*/.cargo/.*" -print0 \
+find . -name "*.sh" -not -regex ".*/.cargo/.*" -not -regex ".*/node_modules/.*" -print0 \
   | xargs -0 \
-      docker run -w /work -v "$PWD:/work" \
-        koalaman/shellcheck --color=always --external-sources --shell=bash
-
+      ci/docker-run.sh koalaman/shellcheck --color=always --external-sources --shell=bash
 exit 0

ci/test-cuda.sh

@@ -2,14 +2,21 @@
 
 cd "$(dirname "$0")/.."
 
-: "${libcuda_verify_ed25519_URL:?environment variable undefined}"
+LIB=libcuda_verify_ed25519.a
+if [[ ! -r $LIB ]]; then
+  if [[ -z "${libcuda_verify_ed25519_URL:-}" ]]; then
+    echo "$0 skipped.  Unable to locate $LIB"
+    exit 0
+  fi
 
-export LD_LIBRARY_PATH=/usr/local/cuda/lib64
-export PATH=$PATH:/usr/local/cuda/bin
-curl -X GET -o libcuda_verify_ed25519.a "$libcuda_verify_ed25519_URL"
+  export LD_LIBRARY_PATH=/usr/local/cuda/lib64
+  export PATH=$PATH:/usr/local/cuda/bin
+  curl -X GET -o $LIB "$libcuda_verify_ed25519_URL"
+fi
 
 # shellcheck disable=SC1090    # <-- shellcheck can't follow ~
 source ~/.cargo/env
+export RUST_BACKTRACE=1
 cargo test --features=cuda
 
 exit 0

ci/test-nightly.sh

@@ -5,6 +5,7 @@ cd "$(dirname "$0")/.."
 rustc --version
 cargo --version
 
+export RUST_BACKTRACE=1
 rustup component add rustfmt-preview
 cargo build --verbose --features unstable
 cargo test --verbose --features unstable

ci/test-stable.sh

@@ -5,6 +5,7 @@ cd "$(dirname "$0")/.."
 rustc --version
 cargo --version
 
+export RUST_BACKTRACE=1
 rustup component add rustfmt-preview
 cargo fmt -- --write-mode=diff
 cargo build --verbose