Overhaul coverage setup

.buildkite/pipeline-upload.sh

@@ -14,7 +14,7 @@ buildkite-agent pipeline upload ci/buildkite.yml
 
 if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
   # Add helpful link back to the corresponding Github Pull Request
-  buildkite-agent annotate --style "info" \
+  buildkite-agent annotate --style info --context pr-backlink \
     "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
 fi
 

README.md

@@ -166,22 +166,13 @@ The release process for this project is described [here](RELEASE.md).
 Code coverage
 ---
 
-To generate code coverage statistics, install cargo-cov. Note: the tool currently only works
-in Rust nightly.
+To generate code coverage statistics:
 
 ```bash
-$ cargo +nightly install cargo-cov
+$ scripts/coverage.sh
+$ open target/cov/lcov-local/index.html
 ```
 
-Run cargo-cov and generate a report:
-
-```bash
-$ cargo +nightly cov test
-$ cargo +nightly cov report --open
-```
-
-The coverage report will be written to `./target/cov/report/index.html`
-
 
 Why coverage? While most see coverage as a code quality metric, we see it primarily as a developer
 productivity metric. When a developer makes a change to the codebase, presumably it's a *solution* to

ci/buildkite.yml

@@ -17,9 +17,9 @@ steps:
   - command: "ci/docker-run.sh solanalabs/rust:1.31.0 ci/test-stable.sh"
     name: "stable"
     timeout_in_minutes: 30
-  #- command: "ci/docker-run.sh solanalabs/rust-nightly:2018-12-05 ci/test-nightly.sh"
-  #  name: "nightly"
-  #  timeout_in_minutes: 30
+  - command: "ci/docker-run.sh solanalabs/rust-nightly:2018-12-18 ci/test-coverage.sh"
+    name: "coverage"
+    timeout_in_minutes: 30
   # TODO: Fix and re-enable test-large-network.sh
   # - command: "ci/test-large-network.sh || true"
   #   name: "large-network [ignored]"

ci/publish-book.sh

@@ -8,18 +8,6 @@ _() {
   "$@"
 }
 
-maybe_install() {
-  for cmd in "$@"; do
-    set +e
-    "$cmd" --help > /dev/null 2>&1
-    declare exitcode=$?
-    set -e
-    if [[ $exitcode -ne 0 ]]; then
-      _ cargo install "$cmd"
-    fi
-  done
-}
-
 book/build.sh
 
 echo --- create book repo

ci/test-coverage.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")/.."
+
+annotate() {
+  ${BUILDKITE:-false} && {
+    buildkite-agent annotate "$@"
+  }
+}
+
+affectedFiles="$(buildkite-agent meta-data get affected_files)"
+echo "Affected files in this PR: $affectedFiles"
+if [[ ! ":$affectedFiles:" =~ \.rs: ]]; then
+  annotate --style info --context coverage-info \
+    "Coverage skipped as no .rs files were modified"
+  exit 0
+fi
+
+source ci/upload-ci-artifact.sh
+ci/version-check-with-upgrade.sh nightly
+
+scripts/coverage.sh
+
+report=coverage-"${BUILDKITE_COMMIT:0:9}".tar.gz
+mv target/cov/report.tar.gz "$report"
+upload-ci-artifact "$report"
+annotate --style success --context lcov-report \
+  "lcov report: <a href=\"artifact://$report\">$report</a>"
+
+echo "--- codecov.io report"
+if [[ -z "$CODECOV_TOKEN" ]]; then
+  echo "^^^ +++"
+  echo CODECOV_TOKEN undefined, codecov.io upload skipped
+else
+  bash <(curl -s https://codecov.io/bash) -X gcov -f target/cov/lcov.info
+
+  annotate --style success --context codecov.io \
+    "CodeCov report: https://codecov.io/github/solana-labs/solana/commit/${BUILDKITE_COMMIT:0:9}"
+fi

scripts/coverage.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+#
+# Runs all tests and collects code coverage
+#
+# Warning: this process is a little slow
+#
+
+set -e
+cd "$(dirname "$0")/.."
+
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+: "${BUILDKITE_COMMIT:=local}"
+reportName="lcov-${BUILDKITE_COMMIT:0:9}"
+
+export RUSTFLAGS="
+  -Zprofile
+  -Zno-landing-pads
+  -Ccodegen-units=1
+  -Cinline-threshold=0
+  -Coverflow-checks=off
+"
+export CARGO_INCREMENTAL=0
+export RUST_BACKTRACE=1
+
+echo "--- remove old coverage results"
+
+if [[ -d target/cov ]]; then
+  find target/cov -name \*.gcda -print0 | xargs -0 rm -f
+fi
+rm -rf target/cov/$reportName
+
+_ cargo +nightly build --target-dir target/cov --all
+_ cargo +nightly test --target-dir target/cov --lib --all
+
+_ scripts/fetch-grcov.sh
+echo "--- grcov"
+./grcov target/cov/debug/deps/ > target/cov/lcov_full.info
+
+echo "--- filter_non_local_files_from_lcov"
+filter_non_local_files_from_lcov() {
+  declare skip=false
+  while read -r line; do
+    if [[ $line =~ ^SF:/ ]]; then
+      skip=true # Skip all absolute paths as these are references into ~/.cargo
+    elif [[ $line =~ ^SF:(.*) ]]; then
+      # Skip relative paths that don't exist
+      declare file="${BASH_REMATCH[1]}"
+      if [[ -r $file ]]; then
+        skip=false
+      else
+        skip=true
+      fi
+    fi
+    [[ $skip = true ]] || echo "$line"
+  done
+}
+
+filter_non_local_files_from_lcov < target/cov/lcov_full.info > target/cov/lcov.info
+
+echo "--- html report"
+# ProTip: genhtml comes from |brew install lcov| or |apt-get install lcov|
+genhtml --output-directory target/cov/$reportName \
+  --show-details \
+  --highlight \
+  --ignore-errors source \
+  --prefix "$PWD" \
+  --legend \
+  target/cov/lcov.info
+
+(
+  cd target/cov
+  tar zcf report.tar.gz $reportName
+)
+
+ls -l target/cov/$reportName/index.html

scripts/fetch-grcov.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ ! -x ./grcov ]]; then
+  uname=$(uname | tr '[:upper:]' '[:lower:]')
+  if [[ $uname = darwin ]]; then
+    uname="osx"
+  fi
+  uname_m=$(uname -m | tr '[:upper:]' '[:lower:]')
+  name=grcov-${uname}-${uname_m}.tar.bz2
+
+  wget "https://github.com/mozilla/grcov/releases/download/v0.3.2/$name"
+  tar xjf "$name"
+fi
+
+ls -lh grcov