From 209b0a3f2bed07bfa32a2d95da5fb3cc3383668f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tadej=20Jane=C5=BE?= <tadej.j@nez.si>
Date: Fri, 21 Feb 2020 14:02:31 +0100
Subject: [PATCH] buildkite: Add dynamic pipeline generator for conditional
 skipping

Use Buildkite's dynamic pipelines feature to dynamically choose between
Code/Code-skip pipelines depending on whether the build was triggered
for a pull request and the list of changes files.
---
 .buildkite/code-skip.pipeline.yml             | 10 +++++
 .../{pipeline.yml => code.pipeline.yml}       | 19 +++-----
 .buildkite/pipeline.sh                        | 45 +++++++++++++++++++
 .../scripts/skip_if_only_docs_changes.sh      | 23 ----------
 .changelog/2702.internal.md                   |  6 +++
 5 files changed, 68 insertions(+), 35 deletions(-)
 create mode 100644 .buildkite/code-skip.pipeline.yml
 rename .buildkite/{pipeline.yml => code.pipeline.yml} (94%)
 create mode 100755 .buildkite/pipeline.sh
 delete mode 100644 .buildkite/scripts/skip_if_only_docs_changes.sh
 create mode 100644 .changelog/2702.internal.md

diff --git a/.buildkite/code-skip.pipeline.yml b/.buildkite/code-skip.pipeline.yml
new file mode 100644
index 00000000000..ff08e3119db
--- /dev/null
+++ b/.buildkite/code-skip.pipeline.yml
@@ -0,0 +1,10 @@
+##
+# Code-skip pipeline
+##
+#
+# Buildkite pipeline for skipping running code-related linters and tests.
+#
+
+steps:
+  - label: No code-related changes detected, skipping Code pipeline
+    command: exit 0
diff --git a/.buildkite/pipeline.yml b/.buildkite/code.pipeline.yml
similarity index 94%
rename from .buildkite/pipeline.yml
rename to .buildkite/code.pipeline.yml
index f9b8760bca7..9d15e022bde 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/code.pipeline.yml
@@ -1,6 +1,10 @@
-################
-# Build pipeline
-################
+##
+# Code pipeline
+##
+#
+# Main Buildkite pipeline for running code-related linters and tests.
+#
+
 docker_plugin_default_config: &docker_plugin_default_config
   image: "oasislabs/testing:0.3.0"
   always_pull: true
@@ -90,7 +94,6 @@ steps:
   # .buildkite/scripts/download_utils.sh.
   - label: Build Go node
     command:
-      - . .buildkite/scripts/skip_if_only_docs_changes.sh
       - .buildkite/go/build.sh
 
       # Upload the built artifacts.
@@ -109,7 +112,6 @@ steps:
   # .buildkite/scripts/download_utils.sh.
   - label: Build Rust runtime loader
     command:
-      - . .buildkite/scripts/skip_if_only_docs_changes.sh
       - .buildkite/rust/build_generic.sh /workdir -p oasis-core-runtime-loader
       - .buildkite/rust/build_generic.sh /workdir -p test-long-term-client
       - .buildkite/rust/build_generic.sh /workdir -p simple-keyvalue-client
@@ -136,7 +138,6 @@ steps:
   # .buildkite/rust/test_runtime_and_gateway.sh and .buildkite/scripts/download_utils.sh.
   - label: Build key manager runtime
     command:
-      - . .buildkite/scripts/skip_if_only_docs_changes.sh
       - .buildkite/rust/build_runtime.sh keymanager-runtime
       - .buildkite/rust/build_runtime.sh tests/runtimes/simple-keyvalue
 
@@ -157,7 +158,6 @@ steps:
   ###########
   - label: Test Rust crates
     command:
-      - . .buildkite/scripts/skip_if_only_docs_changes.sh
       # Build storage interoperability test helpers first.
       - make build-helpers
       - export OASIS_STORAGE_PROTOCOL_SERVER_BINARY=$(realpath go/storage/mkvs/urkel/interop/urkel-test-helpers)
@@ -177,7 +177,6 @@ steps:
   - label: Test fuzz builds
     branches: master
     command:
-      - . .buildkite/scripts/skip_if_only_docs_changes.sh
       # TODO: Consider making this a part of the development Docker image.
       - go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
       - make -C go build-fuzz
@@ -189,7 +188,6 @@ steps:
   #####################################
   - label: Test Go node
     command:
-      - . .buildkite/scripts/skip_if_only_docs_changes.sh
       - .buildkite/go/test_and_coverage.sh
     artifact_paths:
       - coverage-misc.txt
@@ -205,7 +203,6 @@ steps:
     parallelism: 6
     timeout_in_minutes: 9
     command:
-      - . .buildkite/scripts/skip_if_only_docs_changes.sh
       - .buildkite/scripts/download_e2e_test_artifacts.sh
       - .buildkite/scripts/test_e2e.sh
     artifact_paths:
@@ -226,7 +223,6 @@ steps:
     parallelism: 5
     timeout_in_minutes: 19
     command:
-      - . .buildkite/scripts/skip_if_only_docs_changes.sh
       - .buildkite/scripts/download_e2e_test_artifacts.sh
       - .buildkite/scripts/test_e2e.sh
     artifact_paths:
@@ -281,7 +277,6 @@ steps:
 
   - label: "Merge and upload coverage"
     command:
-      - . .buildkite/scripts/skip_if_only_docs_changes.sh
       - .buildkite/scripts/merge_coverage.sh
       - .buildkite/scripts/upload_coverage.sh
     artifact_paths:
diff --git a/.buildkite/pipeline.sh b/.buildkite/pipeline.sh
new file mode 100755
index 00000000000..81844744e6c
--- /dev/null
+++ b/.buildkite/pipeline.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+##
+# Dynamic Buildkite pipeline generator.
+##
+#
+# It outputs valid Buildkite pipeline in YAML format.
+#
+# To use it, define the following Steps under your Buildkite's Pipeline Settings:
+#
+# steps:
+#   - command: .buildkite/pipeline.sh | buildkite-agent pipeline upload
+#     label: ":pipeline: Upload"
+#
+# For more details, see:
+# https://buildkite.com/docs/pipelines/defining-steps#dynamic-pipelines.
+#
+
+set -eux
+
+# Helper that ensures the build is triggered for a pull request and that there
+# are no code-related changes compared to the pull request's base branch.
+pr_and_no_code_related_changes() {
+  # Check if the build was triggered for a pull request.
+  if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
+    return 1
+  fi
+  # Get the list of changes files, excluding changes unrelated to code.
+  # NOTE: The exclude patterns below use git's pathspec syntax:
+  # https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefpathspecapathspec.
+  git diff --name-only --exit-code "refs/remotes/origin/$BUILDKITE_PULL_REQUEST_BASE_BRANCH.." 1>&2 -- \
+      ':(exclude)*.md' \
+      ':(exclude).changelog/' \
+      ':(exclude).github/' \
+      ':(exclude).gitlint' \
+      ':(exclude).markdownlint.yml' \
+      ':(exclude).punch_config.py' \
+      ':(exclude)docs/' \
+      ':(exclude)towncrier.toml'
+}
+
+if pr_and_no_code_related_changes; then
+    cat .buildkite/code-skip.pipeline.yml
+else
+    cat .buildkite/code.pipeline.yml
+fi
diff --git a/.buildkite/scripts/skip_if_only_docs_changes.sh b/.buildkite/scripts/skip_if_only_docs_changes.sh
deleted file mode 100644
index 65c8c2459f7..00000000000
--- a/.buildkite/scripts/skip_if_only_docs_changes.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-# Source this file rather than fork-exec-ing it.
-
-only_docs_changes() {
-  # https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefpathspecapathspec
-  # Some non-docs things related to workflows are also in .github/, but this script currently
-  # is specific to buildkite, where we wouldn't care about changes there anyway. If we later
-  # use this in other CI places, we can narrow it down.
-  test -n "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" &&
-    git diff --quiet "refs/remotes/origin/$BUILDKITE_PULL_REQUEST_BASE_BRANCH.." -- \
-      ':(exclude)*.md' \
-      ':(exclude).changelog/' \
-      ':(exclude).github/' \
-      ':(exclude).gitlint' \
-      ':(exclude).markdownlint.yml' \
-      ':(exclude).punch_config.py' \
-      ':(exclude)docs/' \
-      ':(exclude)towncrier.toml'
-}
-
-if only_docs_changes; then
-  echo "Only docs changes. Skipping"
-  exit 0
-fi
diff --git a/.changelog/2702.internal.md b/.changelog/2702.internal.md
new file mode 100644
index 00000000000..65de41d29a4
--- /dev/null
+++ b/.changelog/2702.internal.md
@@ -0,0 +1,6 @@
+ci: Skip some steps for non-code changes
+
+When one makes a pull request that e.g. only adds documentation or
+assembles the Change Log from fragments, all the *heavy* Buildkite
+pipeline steps (e.g. Go/Rust building, Go tests, E2E tests) should be
+skipped.