[Develop] Prevent buildkite clone to speedup pipeline

.cicd/generate-pipeline.sh

@@ -6,6 +6,7 @@ export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas::
 export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.6_6C_14G_40G'}
 export PLATFORMS_JSON_ARRAY='[]'
 [[ -z "$ROUNDS" ]] && export ROUNDS='1'
+DISABLE_CONCURRENCY=${DISABLE_CONCURRENCY:-false}
 LINUX_CONCURRENCY='8'
 MAC_CONCURRENCY='2'
 LINUX_CONCURRENCY_GROUP='eos-scheduled-build'
@@ -134,7 +135,9 @@ EOF
             - 'registry_2'
           pre-execute-sleep: 10
           pre-commands: 
-            - "git clone [email protected]:EOSIO/mac-anka-fleet.git && cd mac-anka-fleet && . ./ensure-tag.bash -u 12 -r 25G -a '-n'"
+            - "rm -rf mac-anka-fleet; git clone [email protected]:EOSIO/mac-anka-fleet.git && cd mac-anka-fleet && . ./ensure-tag.bash -u 12 -r 25G -a '-n'"
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     env:
       REPO: ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO}
       REPO_COMMIT: $BUILDKITE_COMMIT
@@ -149,7 +152,7 @@ EOF
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_BUILD}
 EOF
     fi
-    if [ "$BUILDKITE_SOURCE" = "schedule" ]; then
+    if [ "$BUILDKITE_SOURCE" = "schedule" ] && [[ $DISABLE_CONCURRENCY != true ]]; then
         cat <<EOF
     concurrency: ${CONCURRENCY}
     concurrency_group: ${CONCURRENCY_GROUP}
@@ -206,13 +209,15 @@ EOF
             - 'registry_1'
             - 'registry_2'
           pre-execute-sleep: 10
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     timeout: ${TIMEOUT:-60}
     agents: "queue=mac-anka-node-fleet"
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_UNIT_TESTS}
 
 EOF
         fi
-        if [ "$BUILDKITE_SOURCE" = "schedule" ]; then
+        if [ "$BUILDKITE_SOURCE" = "schedule" ] && [[ $DISABLE_CONCURRENCY != true ]]; then
             cat <<EOF
     concurrency: ${CONCURRENCY}
     concurrency_group: ${CONCURRENCY_GROUP}
@@ -262,13 +267,15 @@ EOF
             - 'registry_1'
             - 'registry_2'
           pre-execute-sleep: 10
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     timeout: ${TIMEOUT:-60}
     agents: "queue=mac-anka-node-fleet"
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_WASM_SPEC_TESTS}
 
 EOF
         fi
-        if [ "$BUILDKITE_SOURCE" = "schedule" ]; then
+        if [ "$BUILDKITE_SOURCE" = "schedule" ] && [[ $DISABLE_CONCURRENCY != true ]]; then
             cat <<EOF
     concurrency: ${CONCURRENCY}
     concurrency_group: ${CONCURRENCY_GROUP}
@@ -288,8 +295,14 @@ EOF
                 cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
     command:
+      - "ssh-keyscan -H github.com >> ~/.ssh/known_hosts"
+      - "git clone \$BUILDKITE_REPO ."
+      - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT"
       - "buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz"
       - "./.cicd/test.sh scripts/serial-test.sh $TEST_NAME"
+    plugins:
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     env:
       IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)
       PLATFORM_TYPE: $PLATFORM_TYPE
@@ -321,12 +334,14 @@ EOF
             - 'registry_1'
             - 'registry_2'
           pre-execute-sleep: 10
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     timeout: ${TIMEOUT:-60}
     agents: "queue=mac-anka-node-fleet"
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_SERIAL_TESTS}
 EOF
             fi
-            if [ "$BUILDKITE_SOURCE" = "schedule" ]; then
+            if [ "$BUILDKITE_SOURCE" = "schedule" ] && [[ $DISABLE_CONCURRENCY != true ]]; then
                 cat <<EOF
     concurrency: ${CONCURRENCY}
     concurrency_group: ${CONCURRENCY_GROUP}
@@ -348,8 +363,14 @@ EOF
                 cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
     command:
+      - "ssh-keyscan -H github.com >> ~/.ssh/known_hosts"
+      - "git clone \$BUILDKITE_REPO ."
+      - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT"
       - "buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' ${BUILD_SOURCE} && tar -xzf build.tar.gz"
       - "./.cicd/test.sh scripts/long-running-test.sh $TEST_NAME"
+    plugins:
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     env:
       IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)
       PLATFORM_TYPE: $PLATFORM_TYPE
@@ -381,12 +402,14 @@ EOF
             - 'registry_1'
             - 'registry_2'
           pre-execute-sleep: 10
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     timeout: ${TIMEOUT:-180}
     agents: "queue=mac-anka-node-fleet"
     skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_LONG_RUNNING_TESTS:-true}
 EOF
             fi
-            if [ "$BUILDKITE_SOURCE" = "schedule" ]; then
+            if [ "$BUILDKITE_SOURCE" = "schedule" ] && [[ $DISABLE_CONCURRENCY != true ]]; then
                 cat <<EOF
     concurrency: ${CONCURRENCY}
     concurrency_group: ${CONCURRENCY_GROUP}
@@ -449,11 +472,17 @@ cat <<EOF
     continue_on_failure: true
 
   - label: ":bar_chart: Test Metrics"
-    command: |
-      echo '+++ :compression: Extracting Test Metrics Code'
-      tar -zxf .cicd/metrics/test-metrics.tar.gz
-      echo '+++ :javascript: Running test-metrics.js'
-      node --max-old-space-size=32768 test-metrics.js
+    command:
+      - "ssh-keyscan -H github.com >> ~/.ssh/known_hosts"
+      - "git clone \$BUILDKITE_REPO ."
+      - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT"
+      - "echo '+++ :compression: Extracting Test Metrics Code'"
+      - "tar -zxf .cicd/metrics/test-metrics.tar.gz"
+      - "echo '+++ :javascript: Running test-metrics.js'"
+      - "node --max-old-space-size=32768 test-metrics.js"
+    plugins:
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     agents:
       queue: "$BUILDKITE_TEST_AGENT_QUEUE"
     timeout: ${TIMEOUT:-10}
@@ -464,8 +493,14 @@ cat <<EOF
     # packaging
   - label: ":centos: CentOS 7.6 - Package Builder"
     command:
+      - "ssh-keyscan -H github.com >> ~/.ssh/known_hosts"
+      - "git clone \$BUILDKITE_REPO ."
+      - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT"
       - "buildkite-agent artifact download build.tar.gz . --step ':centos: CentOS 7.6 - Build' && tar -xzf build.tar.gz"
       - "./.cicd/package.sh"
+    plugins:
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     env:
       IMAGE_TAG: "centos-7.6-$PLATFORM_TYPE"
       PLATFORM_TYPE: $PLATFORM_TYPE
@@ -478,8 +513,14 @@ cat <<EOF
 
   - label: ":ubuntu: Ubuntu 16.04 - Package Builder"
     command:
+      - "ssh-keyscan -H github.com >> ~/.ssh/known_hosts"
+      - "git clone \$BUILDKITE_REPO ."
+      - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT"
       - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 16.04 - Build' && tar -xzf build.tar.gz"
       - "./.cicd/package.sh"
+    plugins:
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     env:
       IMAGE_TAG: "ubuntu-16.04-$PLATFORM_TYPE"
       PLATFORM_TYPE: $PLATFORM_TYPE
@@ -492,8 +533,14 @@ cat <<EOF
 
   - label: ":ubuntu: Ubuntu 18.04 - Package Builder"
     command:
+      - "ssh-keyscan -H github.com >> ~/.ssh/known_hosts"
+      - "git clone \$BUILDKITE_REPO ."
+      - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT"
       - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 18.04 - Build' && tar -xzf build.tar.gz"
       - "./.cicd/package.sh"
+    plugins:
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     env:
       IMAGE_TAG: "ubuntu-18.04-$PLATFORM_TYPE"
       PLATFORM_TYPE: $PLATFORM_TYPE
@@ -522,6 +569,8 @@ cat <<EOF
             - 'registry_1'
             - 'registry_2'
           pre-execute-sleep: 10
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     agents:
       - "queue=mac-anka-node-fleet"
     timeout: ${TIMEOUT:-60}
@@ -546,9 +595,15 @@ cat <<EOF
     timeout: ${TIMEOUT:-5}
 
   - label: ":beer: Brew Updater"
-    command: |
-      buildkite-agent artifact download eosio.rb . --step ':darwin: macOS 10.14 - Package Builder'
-      buildkite-agent artifact upload eosio.rb
+    command:
+      - "ssh-keyscan -H github.com >> ~/.ssh/known_hosts"
+      - "git clone \$BUILDKITE_REPO ."
+      - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT"
+      - "buildkite-agent artifact download eosio.rb . --step ':darwin: macOS 10.14 - Package Builder'"
+      - "buildkite-agent artifact upload eosio.rb"
+    plugins:
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     agents:
       queue: "automation-basic-builder-fleet"
     timeout: "${TIMEOUT:-5}"

.cicd/pipeline.yml

@@ -3,9 +3,16 @@ steps:
 
   - label: ":pipeline: Generate Pipeline Steps"
     command:
+      - "ssh-keyscan -H github.com >> ~/.ssh/known_hosts"
+      - "git clone $BUILDKITE_REPO ."
+      - "[[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]] && git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head"
+      - "git checkout -f $BUILDKITE_COMMIT"
       - "./.cicd/generate-pipeline.sh > generated-pipeline.yml"
       - "buildkite-agent pipeline upload < generated-pipeline.yml"
       - "buildkite-agent artifact upload generated-pipeline.yml"
+    plugins:
+      - thedyrt/skip-checkout#v0.1.1:
+          cd: ~
     agents:
       queue: "automation-basic-builder-fleet"
     timeout: ${TIMEOUT:-10}