From 3ce62714b7e38bcffe4f1279184d734004bc0e9f Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Mon, 17 Dec 2018 15:22:29 -0800 Subject: [PATCH 1/4] Remove dead code --- ci/publish-book.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/ci/publish-book.sh b/ci/publish-book.sh index 620324d9cefbac..c7ea4ec2fe5def 100755 --- a/ci/publish-book.sh +++ b/ci/publish-book.sh @@ -8,18 +8,6 @@ _() { "$@" } -maybe_install() { - for cmd in "$@"; do - set +e - "$cmd" --help > /dev/null 2>&1 - declare exitcode=$? - set -e - if [[ $exitcode -ne 0 ]]; then - _ cargo install "$cmd" - fi - done -} - book/build.sh echo --- create book repo From 7525bb95bc92717c786e11854ceb6fa881433ed3 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Sun, 16 Dec 2018 14:21:49 -0700 Subject: [PATCH 2/4] Upgrade grcov --- ci/test-nightly.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test-nightly.sh b/ci/test-nightly.sh index 3f80d09a3efb15..b3a3295ec04310 100755 --- a/ci/test-nightly.sh +++ b/ci/test-nightly.sh @@ -39,7 +39,7 @@ if [[ ! -f ./grcov ]]; then fi uname_m=$(uname -m | tr '[:upper:]' '[:lower:]') name=grcov-${uname}-${uname_m}.tar.bz2 - _ wget "https://github.com/mozilla/grcov/releases/download/v0.2.3/${name}" + _ wget "https://github.com/mozilla/grcov/releases/download/v0.3.2/${name}" _ tar -xjf "${name}" fi _ ./grcov . -t lcov > lcov.info From 2a0c65303f64c42fe57bebfbe6e3a9b5b7f3c9df Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Mon, 17 Dec 2018 13:07:44 -0800 Subject: [PATCH 3/4] Extract grcov download script --- ci/test-nightly.sh | 13 +------------ scripts/fetch-grcov.sh | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 12 deletions(-) create mode 100755 scripts/fetch-grcov.sh diff --git a/ci/test-nightly.sh b/ci/test-nightly.sh index b3a3295ec04310..329002f6df8286 100755 --- a/ci/test-nightly.sh +++ b/ci/test-nightly.sh @@ -30,18 +30,7 @@ _ cargo cov clean _ cargo cov build --all _ cargo cov test --lib _ cargo cov report - -# Generate a coverage report with grcov via lcov. -if [[ ! -f ./grcov ]]; then - uname=$(uname | tr '[:upper:]' '[:lower:]') - if [[ ${uname} = "darwin" ]]; then - uname="osx" - fi - uname_m=$(uname -m | tr '[:upper:]' '[:lower:]') - name=grcov-${uname}-${uname_m}.tar.bz2 - _ wget "https://github.com/mozilla/grcov/releases/download/v0.3.2/${name}" - _ tar -xjf "${name}" -fi +_ ./scripts/fetch-grcov.sh _ ./grcov . -t lcov > lcov.info _ genhtml -o target/cov/report-lcov --show-details --highlight --ignore-errors source --legend lcov.info diff --git a/scripts/fetch-grcov.sh b/scripts/fetch-grcov.sh new file mode 100755 index 00000000000000..9b14544270aa97 --- /dev/null +++ b/scripts/fetch-grcov.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +if [[ ! -x ./grcov ]]; then + uname=$(uname | tr '[:upper:]' '[:lower:]') + if [[ $uname = darwin ]]; then + uname="osx" + fi + uname_m=$(uname -m | tr '[:upper:]' '[:lower:]') + name=grcov-${uname}-${uname_m}.tar.bz2 + + wget "https://github.com/mozilla/grcov/releases/download/v0.3.2/$name" + tar xjf "$name" +fi + +ls -lh grcov From e65c9d9cb4ea58cda9ec86bcf0bdb28203756e60 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Mon, 17 Dec 2018 10:11:02 -0800 Subject: [PATCH 4/4] Overhaul coverage setup --- .buildkite/pipeline-upload.sh | 2 +- README.md | 15 ++----- ci/buildkite.yml | 6 +-- ci/test-coverage.sh | 40 ++++++++++++++++++ ci/test-nightly.sh | 57 ------------------------- scripts/coverage.sh | 79 +++++++++++++++++++++++++++++++++++ 6 files changed, 126 insertions(+), 73 deletions(-) create mode 100755 ci/test-coverage.sh delete mode 100755 ci/test-nightly.sh create mode 100755 scripts/coverage.sh diff --git a/.buildkite/pipeline-upload.sh b/.buildkite/pipeline-upload.sh index d256c2767ab56f..dd3aa146c76c10 100755 --- a/.buildkite/pipeline-upload.sh +++ b/.buildkite/pipeline-upload.sh @@ -14,7 +14,7 @@ buildkite-agent pipeline upload ci/buildkite.yml if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request - buildkite-agent annotate --style "info" \ + buildkite-agent annotate --style info --context pr-backlink \ "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH" fi diff --git a/README.md b/README.md index 38ab4cc6d80ee4..d04be7e6c77b8d 100644 --- a/README.md +++ b/README.md @@ -166,22 +166,13 @@ The release process for this project is described [here](RELEASE.md). Code coverage --- -To generate code coverage statistics, install cargo-cov. Note: the tool currently only works -in Rust nightly. +To generate code coverage statistics: ```bash -$ cargo +nightly install cargo-cov +$ scripts/coverage.sh +$ open target/cov/lcov-local/index.html ``` -Run cargo-cov and generate a report: - -```bash -$ cargo +nightly cov test -$ cargo +nightly cov report --open -``` - -The coverage report will be written to `./target/cov/report/index.html` - Why coverage? While most see coverage as a code quality metric, we see it primarily as a developer productivity metric. When a developer makes a change to the codebase, presumably it's a *solution* to diff --git a/ci/buildkite.yml b/ci/buildkite.yml index 52e2ca34873ae9..9f499168482636 100644 --- a/ci/buildkite.yml +++ b/ci/buildkite.yml @@ -17,9 +17,9 @@ steps: - command: "ci/docker-run.sh solanalabs/rust:1.31.0 ci/test-stable.sh" name: "stable" timeout_in_minutes: 30 - #- command: "ci/docker-run.sh solanalabs/rust-nightly:2018-12-05 ci/test-nightly.sh" - # name: "nightly" - # timeout_in_minutes: 30 + - command: "ci/docker-run.sh solanalabs/rust-nightly:2018-12-18 ci/test-coverage.sh" + name: "coverage" + timeout_in_minutes: 30 # TODO: Fix and re-enable test-large-network.sh # - command: "ci/test-large-network.sh || true" # name: "large-network [ignored]" diff --git a/ci/test-coverage.sh b/ci/test-coverage.sh new file mode 100755 index 00000000000000..14278bce6d7c4e --- /dev/null +++ b/ci/test-coverage.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -e + +cd "$(dirname "$0")/.." + +annotate() { + ${BUILDKITE:-false} && { + buildkite-agent annotate "$@" + } +} + +affectedFiles="$(buildkite-agent meta-data get affected_files)" +echo "Affected files in this PR: $affectedFiles" +if [[ ! ":$affectedFiles:" =~ \.rs: ]]; then + annotate --style info --context coverage-info \ + "Coverage skipped as no .rs files were modified" + exit 0 +fi + +source ci/upload-ci-artifact.sh +ci/version-check-with-upgrade.sh nightly + +scripts/coverage.sh + +report=coverage-"${BUILDKITE_COMMIT:0:9}".tar.gz +mv target/cov/report.tar.gz "$report" +upload-ci-artifact "$report" +annotate --style success --context lcov-report \ + "lcov report: $report" + +echo "--- codecov.io report" +if [[ -z "$CODECOV_TOKEN" ]]; then + echo "^^^ +++" + echo CODECOV_TOKEN undefined, codecov.io upload skipped +else + bash <(curl -s https://codecov.io/bash) -X gcov -f target/cov/lcov.info + + annotate --style success --context codecov.io \ + "CodeCov report: https://codecov.io/github/solana-labs/solana/commit/${BUILDKITE_COMMIT:0:9}" +fi diff --git a/ci/test-nightly.sh b/ci/test-nightly.sh deleted file mode 100755 index 329002f6df8286..00000000000000 --- a/ci/test-nightly.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -e - -cd "$(dirname "$0")/.." -source ci/upload-ci-artifact.sh - -ci/version-check.sh nightly -export RUST_BACKTRACE=1 - -_() { - echo "--- $*" - "$@" -} - -# Uncomment this to run nightly test suit -# _ cargo test --all --verbose --features=unstable -- --test-threads=1 - -cargo_install_unless() { - declare crate=$1 - shift - - "$@" > /dev/null 2>&1 || \ - _ cargo install "$crate" -} - -cargo_install_unless cargo-cov cargo cov --help - -# Generate coverage data and report via unit-test suite. -_ cargo cov clean -_ cargo cov build --all -_ cargo cov test --lib -_ cargo cov report -_ ./scripts/fetch-grcov.sh -_ ./grcov . -t lcov > lcov.info -_ genhtml -o target/cov/report-lcov --show-details --highlight --ignore-errors source --legend lcov.info - -# Upload to tarballs to buildkite. -_ cd target/cov && tar -cjf cov-report.tar.bz2 report/* && cd - -_ upload-ci-artifact "target/cov/cov-report.tar.bz2" - -_ cd target/cov && tar -cjf lcov-report.tar.bz2 report-lcov/* && cd - -_ upload-ci-artifact "target/cov/lcov-report.tar.bz2" - -# Upload coverage files to buildkite for grcov debugging -_ cd target/cov/build && tar -cjf cov-gcda.tar.bz2 gcda/* && cd - -_ upload-ci-artifact "target/cov/build/cov-gcda.tar.bz2" - -_ cd target/cov/build && tar -cjf cov-gcno.tar.bz2 gcno/* && cd - -_ upload-ci-artifact "target/cov/build/cov-gcno.tar.bz2" - -if [[ -z "$CODECOV_TOKEN" ]]; then - echo CODECOV_TOKEN undefined -else - true - # TODO: Why doesn't codecov grok our lcov files? - #bash <(curl -s https://codecov.io/bash) -X gcov -fi diff --git a/scripts/coverage.sh b/scripts/coverage.sh new file mode 100755 index 00000000000000..2e2d05392deaa9 --- /dev/null +++ b/scripts/coverage.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# +# Runs all tests and collects code coverage +# +# Warning: this process is a little slow +# + +set -e +cd "$(dirname "$0")/.." + +_() { + echo "--- $*" + "$@" +} + +: "${BUILDKITE_COMMIT:=local}" +reportName="lcov-${BUILDKITE_COMMIT:0:9}" + +export RUSTFLAGS=" + -Zprofile + -Zno-landing-pads + -Ccodegen-units=1 + -Cinline-threshold=0 + -Coverflow-checks=off +" +export CARGO_INCREMENTAL=0 +export RUST_BACKTRACE=1 + +echo "--- remove old coverage results" + +if [[ -d target/cov ]]; then + find target/cov -name \*.gcda -print0 | xargs -0 rm -f +fi +rm -rf target/cov/$reportName + +_ cargo +nightly build --target-dir target/cov --all +_ cargo +nightly test --target-dir target/cov --lib --all + +_ scripts/fetch-grcov.sh +echo "--- grcov" +./grcov target/cov/debug/deps/ > target/cov/lcov_full.info + +echo "--- filter_non_local_files_from_lcov" +filter_non_local_files_from_lcov() { + declare skip=false + while read -r line; do + if [[ $line =~ ^SF:/ ]]; then + skip=true # Skip all absolute paths as these are references into ~/.cargo + elif [[ $line =~ ^SF:(.*) ]]; then + # Skip relative paths that don't exist + declare file="${BASH_REMATCH[1]}" + if [[ -r $file ]]; then + skip=false + else + skip=true + fi + fi + [[ $skip = true ]] || echo "$line" + done +} + +filter_non_local_files_from_lcov < target/cov/lcov_full.info > target/cov/lcov.info + +echo "--- html report" +# ProTip: genhtml comes from |brew install lcov| or |apt-get install lcov| +genhtml --output-directory target/cov/$reportName \ + --show-details \ + --highlight \ + --ignore-errors source \ + --prefix "$PWD" \ + --legend \ + target/cov/lcov.info + +( + cd target/cov + tar zcf report.tar.gz $reportName +) + +ls -l target/cov/$reportName/index.html