From 89fb748572ed9b66fc6e0884e7a06cde898755b3 Mon Sep 17 00:00:00 2001 From: Thomas Wiseman Date: Wed, 18 Jan 2023 15:47:41 -0500 Subject: [PATCH 001/213] workflow trial 1 --- .github/workflows/travis-to-actions-test.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/travis-to-actions-test.yml diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml new file mode 100644 index 00000000000..a95dbd1851c --- /dev/null +++ b/.github/workflows/travis-to-actions-test.yml @@ -0,0 +1,17 @@ +name: travis-to-github-test +run-name: ${{ github.actor }} is testing out workflows. +on: + push: + branches: + -travis_to_workflows_test + +jobs: + check-bats-version: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: '14' + - run: npm install -g bats + - run: bats -v \ No newline at end of file From a7f5fe7b3711854492c87d66e60d81bfd010da95 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 18 Jan 2023 15:52:04 -0500 Subject: [PATCH 002/213] udate --- .github/workflows/travis-to-actions-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index a95dbd1851c..f1f6f497995 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -1,9 +1,10 @@ name: travis-to-github-test run-name: ${{ github.actor }} is testing out workflows. on: + workflow_dispatch: push: branches: - -travis_to_workflows_test + - travis_to_workflows_test jobs: check-bats-version: From c619a6132aec5391b224f5807887e9a9ca4afc8c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 18 Jan 2023 15:56:50 -0500 Subject: [PATCH 003/213] remove trigger --- .github/workflows/travis-to-actions-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index f1f6f497995..295cc3e3cdd 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -1,7 +1,6 @@ name: travis-to-github-test run-name: ${{ github.actor }} is testing out workflows. on: - workflow_dispatch: push: branches: - travis_to_workflows_test From 900da1b464de11ed91df6caaba1bbbbf010b36ec Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 18 Jan 2023 16:15:28 -0500 Subject: [PATCH 004/213] add workflow_dispatch --- .github/workflows/travis-to-actions-test.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 295cc3e3cdd..cbf5ecee1f3 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -1,6 +1,12 @@ name: travis-to-github-test -run-name: ${{ github.actor }} is testing out workflows. + +#This is what shows up in the github workflows page as the title. +run-name: ${{ github.actor }} is testing out workflows. + +#What triggers the workflow to run. +#Currently limited to git push on the travis branch. on: + workflow_dispatcb: push: branches: - travis_to_workflows_test From 41b4fc58f048a983a5332fd3bb917e1620e5565c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 18 Jan 2023 16:16:42 -0500 Subject: [PATCH 005/213] fix spelling mistake --- .github/workflows/travis-to-actions-test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index cbf5ecee1f3..6cd5145f22b 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -4,10 +4,9 @@ name: travis-to-github-test run-name: ${{ github.actor }} is testing out workflows. #What triggers the workflow to run. -#Currently limited to git push on the travis branch. on: - workflow_dispatcb: - push: + workflow_dispatch: #Manual trigger from GitHub UI + push: #On push to specified branches. branches: - travis_to_workflows_test From b742a6b1d304859fe31d0143aa3ac83838a86891 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 18 Jan 2023 16:22:26 -0500 Subject: [PATCH 006/213] echo environment variables --- .github/workflows/travis-to-actions-test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 6cd5145f22b..447fe1db760 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -10,6 +10,10 @@ on: branches: - travis_to_workflows_test +env: + BUILD_TYPE: centaurAws + BUILD_MYSQL: 5.7 + jobs: check-bats-version: runs-on: ubuntu-latest @@ -19,4 +23,5 @@ jobs: with: node-version: '14' - run: npm install -g bats - - run: bats -v \ No newline at end of file + - run: bats -v + - run: echo "Build Type: $BUILD_TYPE\n Build MySQL: $BUILD_MYSQL" \ No newline at end of file From 2c8fe9405b5f876b240939ad607d145f8e0a9ffa Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 18 Jan 2023 16:25:04 -0500 Subject: [PATCH 007/213] typo --- .github/workflows/travis-to-actions-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 447fe1db760..600682c0910 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -24,4 +24,5 @@ jobs: node-version: '14' - run: npm install -g bats - run: bats -v - - run: echo "Build Type: $BUILD_TYPE\n Build MySQL: $BUILD_MYSQL" \ No newline at end of file + - run: echo "Build Type $BUILD_TYPE" + - run: echo "Build MySQL $BUILD_MYSQL" \ No newline at end of file From ef5540f13a5cede8187c88612c267239fbe40595 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 18 Jan 2023 16:28:04 -0500 Subject: [PATCH 008/213] change enviornment variable scope --- .github/workflows/travis-to-actions-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 600682c0910..2ad0e583a51 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -10,12 +10,12 @@ on: branches: - travis_to_workflows_test -env: - BUILD_TYPE: centaurAws - BUILD_MYSQL: 5.7 jobs: check-bats-version: + env: + BUILD_TYPE: centaurAws + BUILD_MYSQL: 5.7 runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 3960e6aad493678174411cfde7147f9f65d857b4 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 18 Jan 2023 16:29:55 -0500 Subject: [PATCH 009/213] it was working the first time --- .github/workflows/travis-to-actions-test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 2ad0e583a51..061ea97fb9b 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -10,12 +10,13 @@ on: branches: - travis_to_workflows_test +#Global environment variables. +env: + BUILD_TYPE: centaurAws + BUILD_MYSQL: 5.7 jobs: check-bats-version: - env: - BUILD_TYPE: centaurAws - BUILD_MYSQL: 5.7 runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 047b9572992c5bdccd2ab99ff1f498c97bf98383 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 23 Jan 2023 12:39:43 -0500 Subject: [PATCH 010/213] here goes nothing --- .github/workflows/travis-to-actions-test.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 061ea97fb9b..805d93b3286 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -1,4 +1,5 @@ -name: travis-to-github-test +name: 'Build Cromwell Test' + #This is what shows up in the github workflows page as the title. run-name: ${{ github.actor }} is testing out workflows. @@ -10,13 +11,16 @@ on: branches: - travis_to_workflows_test +permissions: + contents: read + #Global environment variables. env: BUILD_TYPE: centaurAws BUILD_MYSQL: 5.7 jobs: - check-bats-version: + check-bats-version: #Sanity check not related to Cromwell. Should always pass. runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -26,4 +30,9 @@ jobs: - run: npm install -g bats - run: bats -v - run: echo "Build Type $BUILD_TYPE" - - run: echo "Build MySQL $BUILD_MYSQL" \ No newline at end of file + - run: echo "Build MySQL $BUILD_MYSQL" + run-shell-script: + runs-on: ubuntu-latest + steps: + - run: src/ci/bin/test.sh + \ No newline at end of file From f8f73c72ef3ad883ba2cd73cb14b44815a2c3cdb Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 23 Jan 2023 12:41:37 -0500 Subject: [PATCH 011/213] getting my bearings --- .github/workflows/travis-to-actions-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 805d93b3286..69d8f767732 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -34,5 +34,7 @@ jobs: run-shell-script: runs-on: ubuntu-latest steps: + - run: pwd + - run: bash -c 'echo $BASH_VERSION' - run: src/ci/bin/test.sh \ No newline at end of file From 85e949e7390b69ad197cdecb7256642afd8c9991 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 23 Jan 2023 12:42:40 -0500 Subject: [PATCH 012/213] dotslash --- .github/workflows/travis-to-actions-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 69d8f767732..f7f5490e134 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -36,5 +36,5 @@ jobs: steps: - run: pwd - run: bash -c 'echo $BASH_VERSION' - - run: src/ci/bin/test.sh + - run: ./src/ci/bin/test.sh \ No newline at end of file From a0285509ce65cbcb043e3204d8f4910e2b4a203a Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 23 Jan 2023 12:44:28 -0500 Subject: [PATCH 013/213] are you suuuuure git? --- .github/workflows/travis-to-actions-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index f7f5490e134..d1faad12b41 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -35,6 +35,7 @@ jobs: runs-on: ubuntu-latest steps: - run: pwd + - run: ls - run: bash -c 'echo $BASH_VERSION' - run: ./src/ci/bin/test.sh \ No newline at end of file From 92075529b0fc349a8bd3db5da027b97ebc72513d Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 23 Jan 2023 12:47:05 -0500 Subject: [PATCH 014/213] checkout first --- .github/workflows/travis-to-actions-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index d1faad12b41..0464838f03c 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -34,6 +34,10 @@ jobs: run-shell-script: runs-on: ubuntu-latest steps: + - + name: Checkout + uses: actions/checkout@v3 + - - run: pwd - run: ls - run: bash -c 'echo $BASH_VERSION' From 49c45bf93f511f873982d89cf8906c4c9c764d22 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 23 Jan 2023 12:49:20 -0500 Subject: [PATCH 015/213] formatting --- .github/workflows/travis-to-actions-test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 0464838f03c..b6c9b5136ac 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -34,10 +34,8 @@ jobs: run-shell-script: runs-on: ubuntu-latest steps: - - - name: Checkout - uses: actions/checkout@v3 - - + - name: Checkout + - uses: actions/checkout@v3 - run: pwd - run: ls - run: bash -c 'echo $BASH_VERSION' From 2daf2d6f82218e94520a5d33c945fb0b0e9822d1 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 23 Jan 2023 12:52:47 -0500 Subject: [PATCH 016/213] try this --- .github/workflows/travis-to-actions-test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index b6c9b5136ac..97169419ceb 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -35,9 +35,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - - uses: actions/checkout@v3 - - run: pwd - - run: ls - - run: bash -c 'echo $BASH_VERSION' - - run: ./src/ci/bin/test.sh + uses: actions/checkout@v3 + run: pwd + run: ls + run: bash -c 'echo $BASH_VERSION' + run: ./src/ci/bin/test.sh \ No newline at end of file From 6fc8470607e7a86d8597a3e721159e40034de372 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 23 Jan 2023 13:08:53 -0500 Subject: [PATCH 017/213] steal from adam --- .github/workflows/travis-to-actions-test.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 97169419ceb..6a9b3d13e68 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -31,13 +31,14 @@ jobs: - run: bats -v - run: echo "Build Type $BUILD_TYPE" - run: echo "Build MySQL $BUILD_MYSQL" - run-shell-script: - runs-on: ubuntu-latest + build-and-test: + name: SBT unit tests + runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners steps: - - name: Checkout - uses: actions/checkout@v3 - run: pwd - run: ls - run: bash -c 'echo $BASH_VERSION' - run: ./src/ci/bin/test.sh + - name: Clone Cromwell + uses: actions/checkout@v2 + with: + repository: broadinstitute/cromwell + token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work + path: cromwell \ No newline at end of file From 776e4a91983bdf9e69b98a5528af05629208fa4e Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 23 Jan 2023 13:11:08 -0500 Subject: [PATCH 018/213] with scala --- .github/workflows/travis-to-actions-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 6a9b3d13e68..0bab2646c2e 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -41,4 +41,7 @@ jobs: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work path: cromwell + - uses: olafurpg/setup-scala@v10 + with: + java-version: adopt@1.11 \ No newline at end of file From 426044ae8ed534c9fdc48c7d9e5946e248d7dab4 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 23 Jan 2023 13:13:49 -0500 Subject: [PATCH 019/213] full steal --- .github/workflows/travis-to-actions-test.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 0bab2646c2e..3e92dc33803 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -44,4 +44,12 @@ jobs: - uses: olafurpg/setup-scala@v10 with: java-version: adopt@1.11 - \ No newline at end of file + - name: Run tests + run: | + set -e + cd cromwell + sbt \ + -Dsbt.supershell=false \ + -Dakka.test.timefactor=10 \ + -Dbackend.providers.Local.config.filesystems.local.localization.0=copy \ + test \ No newline at end of file From 6b30ef42b3aee43c1ab454d4fbcdc7335bb6faf7 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 24 Jan 2023 13:23:19 -0500 Subject: [PATCH 020/213] v3 newer than v2 --- .github/workflows/travis-to-actions-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 3e92dc33803..c8672a587ba 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -36,7 +36,7 @@ jobs: runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners steps: - name: Clone Cromwell - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work From 61704c812312ab43cf366f76c87090836d346fbc Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 24 Jan 2023 15:27:14 -0500 Subject: [PATCH 021/213] moment of truth --- .github/workflows/travis-to-actions-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index c8672a587ba..5855bbc12cb 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -52,4 +52,4 @@ jobs: -Dsbt.supershell=false \ -Dakka.test.timefactor=10 \ -Dbackend.providers.Local.config.filesystems.local.localization.0=copy \ - test \ No newline at end of file + testOnly common.util.VersionUtilSpec \ No newline at end of file From 4c70191c647fddfcdc4e6e1cc61dafcf297b49fd Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 24 Jan 2023 15:41:40 -0500 Subject: [PATCH 022/213] will this work? --- .github/workflows/travis-to-actions-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 5855bbc12cb..39c545d1e44 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -51,5 +51,5 @@ jobs: sbt \ -Dsbt.supershell=false \ -Dakka.test.timefactor=10 \ - -Dbackend.providers.Local.config.filesystems.local.localization.0=copy \ - testOnly common.util.VersionUtilSpec \ No newline at end of file + -Dbackend.providers.Local.config.filesystems.local.localization.0=copy + testOnly common.util.VersionUtilSpec \ No newline at end of file From da89730bacb72248c1f4debd7477c094311f3ac3 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 24 Jan 2023 16:14:11 -0500 Subject: [PATCH 023/213] this is the secret sauce --- .github/workflows/travis-to-actions-test.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 39c545d1e44..076e4191026 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -48,8 +48,4 @@ jobs: run: | set -e cd cromwell - sbt \ - -Dsbt.supershell=false \ - -Dakka.test.timefactor=10 \ - -Dbackend.providers.Local.config.filesystems.local.localization.0=copy - testOnly common.util.VersionUtilSpec \ No newline at end of file + sbt "test:testOnly common.util.VersionUtilSpec" \ No newline at end of file From 6c623ed9b0e08af4cc57900f7d3cc78d4a32322f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 12:48:14 -0500 Subject: [PATCH 024/213] This should still be working --- .github/workflows/travis-to-actions-test.yml | 23 +++++--------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 076e4191026..63308507107 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -1,15 +1,13 @@ -name: 'Build Cromwell Test' - +name: 'Basic Unit Test' #This is what shows up in the github workflows page as the title. -run-name: ${{ github.actor }} is testing out workflows. +run-name: ${{ github.actor }} running Basic Unit Test. #What triggers the workflow to run. on: workflow_dispatch: #Manual trigger from GitHub UI - push: #On push to specified branches. - branches: - - travis_to_workflows_test + push: + pull_request: permissions: contents: read @@ -20,19 +18,8 @@ env: BUILD_MYSQL: 5.7 jobs: - check-bats-version: #Sanity check not related to Cromwell. Should always pass. - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: '14' - - run: npm install -g bats - - run: bats -v - - run: echo "Build Type $BUILD_TYPE" - - run: echo "Build MySQL $BUILD_MYSQL" build-and-test: - name: SBT unit tests + name: SBT VersionUtilSpec test runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners steps: - name: Clone Cromwell From e6cd8756c70b4216c2dd9b4b9132a1df4f641e6a Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 12:58:46 -0500 Subject: [PATCH 025/213] one more test --- .github/workflows/travis-to-actions-test.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index 63308507107..da21de478a3 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -1,5 +1,7 @@ name: 'Basic Unit Test' +#The goal of this Github Action is to verify that + #This is what shows up in the github workflows page as the title. run-name: ${{ github.actor }} running Basic Unit Test. @@ -14,12 +16,12 @@ permissions: #Global environment variables. env: - BUILD_TYPE: centaurAws - BUILD_MYSQL: 5.7 + #BUILD_TYPE: centaurAws + #BUILD_MYSQL: 5.7 jobs: build-and-test: - name: SBT VersionUtilSpec test + name: SBT VersionUtilSpec testgig runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners steps: - name: Clone Cromwell @@ -28,7 +30,7 @@ jobs: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work path: cromwell - - uses: olafurpg/setup-scala@v10 + - uses: olafurpg/setup-scala@v11 with: java-version: adopt@1.11 - name: Run tests From 23b5f539854ec7b744a44b54b1e4e71959272960 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 13:03:16 -0500 Subject: [PATCH 026/213] try again --- .github/workflows/travis-to-actions-test.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index da21de478a3..c1ca0cfb3cd 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -14,11 +14,6 @@ on: permissions: contents: read -#Global environment variables. -env: - #BUILD_TYPE: centaurAws - #BUILD_MYSQL: 5.7 - jobs: build-and-test: name: SBT VersionUtilSpec testgig @@ -33,7 +28,7 @@ jobs: - uses: olafurpg/setup-scala@v11 with: java-version: adopt@1.11 - - name: Run tests + - name: Run tests #Invoke SBT with an argument asking it to run a specific test. run: | set -e cd cromwell From 079b3c62cc5cfea9f9bb8874fa7c0255efbec284 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 13:12:17 -0500 Subject: [PATCH 027/213] spelling --- .github/workflows/travis-to-actions-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/travis-to-actions-test.yml index c1ca0cfb3cd..1649630edbb 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/travis-to-actions-test.yml @@ -16,7 +16,7 @@ permissions: jobs: build-and-test: - name: SBT VersionUtilSpec testgig + name: SBT VersionUtilSpec Unit Test runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners steps: - name: Clone Cromwell From b435c7add8e5ca06d215ccf1ad45dd5557526f76 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 13:20:07 -0500 Subject: [PATCH 028/213] rename file --- .../{travis-to-actions-test.yml => basic_unit_test.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{travis-to-actions-test.yml => basic_unit_test.yml} (89%) diff --git a/.github/workflows/travis-to-actions-test.yml b/.github/workflows/basic_unit_test.yml similarity index 89% rename from .github/workflows/travis-to-actions-test.yml rename to .github/workflows/basic_unit_test.yml index 1649630edbb..822121339ae 100644 --- a/.github/workflows/travis-to-actions-test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -23,7 +23,7 @@ jobs: uses: actions/checkout@v3 with: repository: broadinstitute/cromwell - token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work + token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} path: cromwell - uses: olafurpg/setup-scala@v11 with: From d1e7629b69b42573a49dddabb0fb8204e02f3424 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 13:25:31 -0500 Subject: [PATCH 029/213] comments --- .github/workflows/basic_unit_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 822121339ae..71a192e6f97 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -1,11 +1,11 @@ name: 'Basic Unit Test' -#The goal of this Github Action is to verify that +#The goal of this Github Action is to verify that is is possible to compile cromwell (enough) to run a simple unit test. #This is what shows up in the github workflows page as the title. run-name: ${{ github.actor }} running Basic Unit Test. -#What triggers the workflow to run. +#What will trigger the workflow to run. on: workflow_dispatch: #Manual trigger from GitHub UI push: From 2da9d834f663df28fb6bf58960a41ad9f5c0c717 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 13:31:41 -0500 Subject: [PATCH 030/213] optimize command --- .github/workflows/basic_unit_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 71a192e6f97..e04a69f50c9 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -28,8 +28,8 @@ jobs: - uses: olafurpg/setup-scala@v11 with: java-version: adopt@1.11 - - name: Run tests #Invoke SBT with an argument asking it to run a specific test. + - name: Run tests #Invoke SBT with an argument asking it to run a specific test in a specific sbt project. run: | set -e cd cromwell - sbt "test:testOnly common.util.VersionUtilSpec" \ No newline at end of file + sbt "common/testOnly common.util.VersionUtilSpec" \ No newline at end of file From 9d12de0650bd87d4dd17e501c254701148236149 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 15:28:19 -0500 Subject: [PATCH 031/213] test with matrix --- .github/workflows/basic_unit_test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index e04a69f50c9..fb39cd1d2be 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -18,6 +18,9 @@ jobs: build-and-test: name: SBT VersionUtilSpec Unit Test runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners + strategy: + matrix: + project: [common, wom, engine] steps: - name: Clone Cromwell uses: actions/checkout@v3 @@ -32,4 +35,4 @@ jobs: run: | set -e cd cromwell - sbt "common/testOnly common.util.VersionUtilSpec" \ No newline at end of file + sbt "${{ matrix.project }}/testOnly common.util.VersionUtilSpec" \ No newline at end of file From 076ed6a308e961fe2e97586afd16bcb5c14e1379 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 15:30:12 -0500 Subject: [PATCH 032/213] try something else --- .github/workflows/basic_unit_test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index fb39cd1d2be..cbaa0f1bbba 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -18,7 +18,6 @@ jobs: build-and-test: name: SBT VersionUtilSpec Unit Test runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners - strategy: matrix: project: [common, wom, engine] steps: @@ -35,4 +34,4 @@ jobs: run: | set -e cd cromwell - sbt "${{ matrix.project }}/testOnly common.util.VersionUtilSpec" \ No newline at end of file + sbt "${{ matrix.project }}/testOnly common.util.VersionUtilSpec"g \ No newline at end of file From 75c0955542ef4bf1691e88fa85b8f76b73fddef4 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 15:33:26 -0500 Subject: [PATCH 033/213] tab --- .github/workflows/basic_unit_test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index cbaa0f1bbba..f6505474c1a 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -18,8 +18,9 @@ jobs: build-and-test: name: SBT VersionUtilSpec Unit Test runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners - matrix: - project: [common, wom, engine] + strategy: + matrix: + project: [common, wom, engine] steps: - name: Clone Cromwell uses: actions/checkout@v3 From efbc02c1a67e0ceae2e653a6d166bc659e087c12 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 15:37:03 -0500 Subject: [PATCH 034/213] more realistic workflow --- .github/workflows/basic_unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index f6505474c1a..eaeef3d554b 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -35,4 +35,4 @@ jobs: run: | set -e cd cromwell - sbt "${{ matrix.project }}/testOnly common.util.VersionUtilSpec"g \ No newline at end of file + sbt "${{ matrix.project }}/test" \ No newline at end of file From 8bb541c442fd08f5b16f75ee09b73e13b0a380c6 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 15:42:25 -0500 Subject: [PATCH 035/213] why fail fast if you can prolong your failure --- .github/workflows/basic_unit_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index eaeef3d554b..b46cff10264 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -20,6 +20,7 @@ jobs: runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners strategy: matrix: + fail-fast: false project: [common, wom, engine] steps: - name: Clone Cromwell From 347679658d4373749336cc8509a0b98bf70d831d Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 15:45:03 -0500 Subject: [PATCH 036/213] continue on error --- .github/workflows/basic_unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index b46cff10264..46c731baf66 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -20,7 +20,7 @@ jobs: runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners strategy: matrix: - fail-fast: false + continue-on-error: true project: [common, wom, engine] steps: - name: Clone Cromwell From 6b8afdbd5a1d63f3db6d4e72a974f4d5264c9d34 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 25 Jan 2023 15:47:35 -0500 Subject: [PATCH 037/213] syntax --- .github/workflows/basic_unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 46c731baf66..9258fd71bbf 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -19,8 +19,8 @@ jobs: name: SBT VersionUtilSpec Unit Test runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners strategy: + fail-fast: false matrix: - continue-on-error: true project: [common, wom, engine] steps: - name: Clone Cromwell From e9018d8fe4fe04071a3cb3efa8ce99cc7a568300 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 14:33:01 -0500 Subject: [PATCH 038/213] updated yaml --- .github/workflows/basic_unit_test.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 9258fd71bbf..f9de9922557 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -17,23 +17,24 @@ permissions: jobs: build-and-test: name: SBT VersionUtilSpec Unit Test - runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners - strategy: - fail-fast: false - matrix: - project: [common, wom, engine] + runs-on: ubuntu-latest #Note: This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have SBT installed by default. steps: + - uses: actions/checkout@v3 + - uses: coursier/cache-action@v6 - name: Clone Cromwell uses: actions/checkout@v3 with: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} path: cromwell - - uses: olafurpg/setup-scala@v11 + - name: Setup JDK + uses: actions/setup-java@v3 with: - java-version: adopt@1.11 + distribution: temurin + java-version: 11 + cache: sbt - name: Run tests #Invoke SBT with an argument asking it to run a specific test in a specific sbt project. run: | set -e cd cromwell - sbt "${{ matrix.project }}/test" \ No newline at end of file + sbt "common/testOnly common.util.VersionUtilSpec" \ No newline at end of file From f6cce2fc8c5f6f4a480ef596e72cb5b4dd18a790 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 14:35:12 -0500 Subject: [PATCH 039/213] git secrets attempt --- .github/workflows/basic_unit_test.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index f9de9922557..4569d2ff7a3 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -21,6 +21,17 @@ jobs: steps: - uses: actions/checkout@v3 - uses: coursier/cache-action@v6 + - name: Git secrets setup + run: | + git clone https://github.com/awslabs/git-secrets.git ~/git-secrets + cd ~/git-secrets + git checkout ad82d68ee924906a0401dfd48de5057731a9bc84 + sudo make install + - name: Secrets check + run: | + sudo ln -s "$(which echo)" /usr/local/bin/say + ./minnie-kenny.sh --force + git secrets --scan-history - name: Clone Cromwell uses: actions/checkout@v3 with: From b727570b08d3b70ed3f0b09eb4d338fa99abec34 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 14:38:09 -0500 Subject: [PATCH 040/213] bigger test suite --- .github/workflows/basic_unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 4569d2ff7a3..e20f43287ba 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -48,4 +48,4 @@ jobs: run: | set -e cd cromwell - sbt "common/testOnly common.util.VersionUtilSpec" \ No newline at end of file + sbt "common/test" \ No newline at end of file From a399d1267c5d0a526285c453c725c88eb1c05827 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 14:48:51 -0500 Subject: [PATCH 041/213] path stuff --- .github/workflows/basic_unit_test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index e20f43287ba..a74b1d34629 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -37,7 +37,6 @@ jobs: with: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} - path: cromwell - name: Setup JDK uses: actions/setup-java@v3 with: @@ -47,5 +46,4 @@ jobs: - name: Run tests #Invoke SBT with an argument asking it to run a specific test in a specific sbt project. run: | set -e - cd cromwell sbt "common/test" \ No newline at end of file From c36fd9140017707a5c31f2a720043b1cd63b517d Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 14:51:41 -0500 Subject: [PATCH 042/213] yolo --- .github/workflows/basic_unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index a74b1d34629..4369687bef8 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -46,4 +46,4 @@ jobs: - name: Run tests #Invoke SBT with an argument asking it to run a specific test in a specific sbt project. run: | set -e - sbt "common/test" \ No newline at end of file + sbt "test" \ No newline at end of file From 0349d6d758c419b29b88539450c6ea12d97546b7 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 15:42:09 -0500 Subject: [PATCH 043/213] name cache setup --- .github/workflows/basic_unit_test.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 4369687bef8..7e283fe7e45 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -17,11 +17,13 @@ permissions: jobs: build-and-test: name: SBT VersionUtilSpec Unit Test - runs-on: ubuntu-latest #Note: This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have SBT installed by default. + runs-on: ubuntu-latest #Note: This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. steps: - - uses: actions/checkout@v3 - - uses: coursier/cache-action@v6 - - name: Git secrets setup + - name: Coursier Cache Setup #Allow sbt to cache setup stuff between github acitons + uses: actions/checkout@v3 + uses: coursier/cache-action@v6 + + - name: Git secrets setup #The Broad requires git-secrets be setup. The Secrets check below veritfies that the setup worked. run: | git clone https://github.com/awslabs/git-secrets.git ~/git-secrets cd ~/git-secrets @@ -32,6 +34,7 @@ jobs: sudo ln -s "$(which echo)" /usr/local/bin/say ./minnie-kenny.sh --force git secrets --scan-history + - name: Clone Cromwell uses: actions/checkout@v3 with: @@ -46,4 +49,4 @@ jobs: - name: Run tests #Invoke SBT with an argument asking it to run a specific test in a specific sbt project. run: | set -e - sbt "test" \ No newline at end of file + sbt "services/testOnly *WriteMetadataActorSpec" \ No newline at end of file From d84cbabe14cfcb9ab6ee39faeb98354ae8329d2c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 15:43:38 -0500 Subject: [PATCH 044/213] hyphens --- .github/workflows/basic_unit_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 7e283fe7e45..638b4ef4b9b 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -20,8 +20,8 @@ jobs: runs-on: ubuntu-latest #Note: This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. steps: - name: Coursier Cache Setup #Allow sbt to cache setup stuff between github acitons - uses: actions/checkout@v3 - uses: coursier/cache-action@v6 + - uses: actions/checkout@v3 + - uses: coursier/cache-action@v6 - name: Git secrets setup #The Broad requires git-secrets be setup. The Secrets check below veritfies that the setup worked. run: | From aee9df5877dd2339b516532325937f9653376532 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 15:49:06 -0500 Subject: [PATCH 045/213] steps --- .github/workflows/basic_unit_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 638b4ef4b9b..331ddf28f21 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -19,9 +19,10 @@ jobs: name: SBT VersionUtilSpec Unit Test runs-on: ubuntu-latest #Note: This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. steps: - - name: Coursier Cache Setup #Allow sbt to cache setup stuff between github acitons - uses: actions/checkout@v3 + name: Checkout Coursier Cache - uses: coursier/cache-action@v6 + name: Enable Coursier Cache - name: Git secrets setup #The Broad requires git-secrets be setup. The Secrets check below veritfies that the setup worked. run: | From aaeee6f5eb17f38e3b2f27fb18a2c9d748b606ce Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 15:55:49 -0500 Subject: [PATCH 046/213] do we even need that? --- .github/workflows/basic_unit_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 331ddf28f21..fb664b5110f 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -37,7 +37,6 @@ jobs: git secrets --scan-history - name: Clone Cromwell - uses: actions/checkout@v3 with: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} From adcfae889319e18f84e04aa01003301419fa4447 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 15:57:43 -0500 Subject: [PATCH 047/213] yes we need that --- .github/workflows/basic_unit_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index fb664b5110f..331ddf28f21 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -37,6 +37,7 @@ jobs: git secrets --scan-history - name: Clone Cromwell + uses: actions/checkout@v3 with: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} From eda3f89cf9abe9b33eac06a5e19bc2f233e7d985 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 16:01:57 -0500 Subject: [PATCH 048/213] please don't break the internet --- .github/workflows/basic_unit_test.yml | 55 ++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 331ddf28f21..dcbd5bfc95e 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -16,7 +16,58 @@ permissions: jobs: build-and-test: - name: SBT VersionUtilSpec Unit Test + strategy: + matrix: + sbtSubProject: + [awsBackend, + awsS3FileSystem, + azureBlobFileSystem, + backend, + centaur, + cloud-nio-impl-drs, + cloud-nio-impl-ftp, + cloud-nio-spi, + Cloud-nio-util, + cloudSupport, + common, + core, + cromiam, + cromwell-drs-localizer, + cromwellApiClient, + databaseMigration, + databaseSql, + dockerHashing, + drsFileSystem, + engine, + ftpFileSystem, + gcsFileSystem, + googlePipelinesCommon, + googlePipelinesV2Alpha1, + googlePipelinesV2Beta, + httpFileSystem, + languageFactoryCore, + perf, + server, + services, + sfsBackend, + sraFileSystem, + tesBackend, + wdlBiscayneLanguageFactory, + wdlDraft2LanguageFactory, + wdlDraft3LanguageFactory, + wdlModelBiscayne, + wdlModelDraft2, + wdlModelDraft3, + wdlNewBaseTransforms, + wdlSharedModel, + wdlSharedTransforms, + wdlTransformsBiscayne, + wdlTransformsDraft2, + wdlTransformsDraft3, + wes2cromwell, + wom, + womtool] + name: ${{ matrix.sbtSubProject }} Unit Tests runs-on: ubuntu-latest #Note: This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. steps: - uses: actions/checkout@v3 @@ -50,4 +101,4 @@ jobs: - name: Run tests #Invoke SBT with an argument asking it to run a specific test in a specific sbt project. run: | set -e - sbt "services/testOnly *WriteMetadataActorSpec" \ No newline at end of file + sbt "${{ matrix.sbtSubProject }}/test" \ No newline at end of file From 81ce1da31e5fcb75ee13a5b718629701f25bd8bd Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 16:04:09 -0500 Subject: [PATCH 049/213] syntax error this --- .github/workflows/basic_unit_test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index dcbd5bfc95e..e0be075181c 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -18,8 +18,7 @@ jobs: build-and-test: strategy: matrix: - sbtSubProject: - [awsBackend, + sbtSubProject: [awsBackend, awsS3FileSystem, azureBlobFileSystem, backend, From a985a646b52336b3bc6b05986ac8a60b3c77d064 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 16:09:59 -0500 Subject: [PATCH 050/213] desparate --- .github/workflows/basic_unit_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index e0be075181c..5dccfadafd3 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -16,6 +16,7 @@ permissions: jobs: build-and-test: + runs-on: ubuntu-latest #Note: This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. strategy: matrix: sbtSubProject: [awsBackend, @@ -66,8 +67,7 @@ jobs: wes2cromwell, wom, womtool] - name: ${{ matrix.sbtSubProject }} Unit Tests - runs-on: ubuntu-latest #Note: This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. + #name: ${{ matrix.sbtSubProject }} Unit Tests steps: - uses: actions/checkout@v3 name: Checkout Coursier Cache From d4da61f28b7f3cfe71e6d0e57d55e26014e2fc28 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 16:12:54 -0500 Subject: [PATCH 051/213] no more --- .github/workflows/basic_unit_test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 5dccfadafd3..1506e26fca0 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -16,10 +16,10 @@ permissions: jobs: build-and-test: - runs-on: ubuntu-latest #Note: This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. strategy: - matrix: - sbtSubProject: [awsBackend, + fail-fast: false + matrix: + sbtSubProject: [awsBackend, awsS3FileSystem, azureBlobFileSystem, backend, @@ -67,6 +67,7 @@ jobs: wes2cromwell, wom, womtool] + runs-on: ubuntu-latest #Note: This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. #name: ${{ matrix.sbtSubProject }} Unit Tests steps: - uses: actions/checkout@v3 From 3132a19dc52d62fe60624b1d912258a96f98a8b6 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 16:18:11 -0500 Subject: [PATCH 052/213] case sensitive and so am I --- .github/workflows/basic_unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 1506e26fca0..65ce396069c 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -27,7 +27,7 @@ jobs: cloud-nio-impl-drs, cloud-nio-impl-ftp, cloud-nio-spi, - Cloud-nio-util, + cloud-nio-util, cloudSupport, common, core, From ae82349b55d77b0b1e3e9e5aad2a5b8f717ad3a8 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 16:31:04 -0500 Subject: [PATCH 053/213] commentos --- .github/workflows/basic_unit_test.yml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/basic_unit_test.yml index 65ce396069c..4bde51e6db9 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/basic_unit_test.yml @@ -1,9 +1,10 @@ name: 'Basic Unit Test' -#The goal of this Github Action is to verify that is is possible to compile cromwell (enough) to run a simple unit test. +#This github action runs all of Cromwell's unit tests. +#A matrix strategy is used so that each subproject can be unit tested in parallel with all other subprojects. #This is what shows up in the github workflows page as the title. -run-name: ${{ github.actor }} running Basic Unit Test. +run-name: ${{ github.actor }} running Cromwell sbt Unit Tests. #What will trigger the workflow to run. on: @@ -15,10 +16,10 @@ permissions: contents: read jobs: - build-and-test: + build-and-test: #This job is spawned once for each subproject in the matrix below. Each iteration will set up a cache, git secrets, and java before running unit tests for that subproject. strategy: - fail-fast: false - matrix: + fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. + matrix: #this array is the output of the command `sbt projects` from the cromwell root directory, minus a bunch of superflous formatting that sbt adds. sbtSubProject: [awsBackend, awsS3FileSystem, azureBlobFileSystem, @@ -67,15 +68,19 @@ jobs: wes2cromwell, wom, womtool] + runs-on: ubuntu-latest #Note: This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. - #name: ${{ matrix.sbtSubProject }} Unit Tests + steps: + + #Allows this github action to use a cache to store stuff like Java and SBT files between runs. - uses: actions/checkout@v3 name: Checkout Coursier Cache - uses: coursier/cache-action@v6 name: Enable Coursier Cache - - name: Git secrets setup #The Broad requires git-secrets be setup. The Secrets check below veritfies that the setup worked. + #Cromwell requires git-secrets be setup. Here, we set up secrets and verify success with a script. + - name: Git secrets setup run: | git clone https://github.com/awslabs/git-secrets.git ~/git-secrets cd ~/git-secrets @@ -87,18 +92,23 @@ jobs: ./minnie-kenny.sh --force git secrets --scan-history + #Clone the cromwell repo to this VM. - name: Clone Cromwell uses: actions/checkout@v3 with: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + + #Install Java to this VM. This Java version and distribution is compatible with Cromwell. - name: Setup JDK uses: actions/setup-java@v3 with: distribution: temurin java-version: 11 cache: sbt - - name: Run tests #Invoke SBT with an argument asking it to run a specific test in a specific sbt project. + + #Invoke SBT on a specific subproject, asking it to run all unit tests contained within that subproject. + - name: Run tests run: | set -e sbt "${{ matrix.sbtSubProject }}/test" \ No newline at end of file From 3a158eaded488a5387c0244c97e76cd1f377a2be Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 26 Jan 2023 17:04:12 -0500 Subject: [PATCH 054/213] rename and polish --- .../{basic_unit_test.yml => cromwell_unit_tests.yml} | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) rename .github/workflows/{basic_unit_test.yml => cromwell_unit_tests.yml} (86%) diff --git a/.github/workflows/basic_unit_test.yml b/.github/workflows/cromwell_unit_tests.yml similarity index 86% rename from .github/workflows/basic_unit_test.yml rename to .github/workflows/cromwell_unit_tests.yml index 4bde51e6db9..896cfce6908 100644 --- a/.github/workflows/basic_unit_test.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -1,10 +1,10 @@ -name: 'Basic Unit Test' +name: 'Cromwell unit tests' #This github action runs all of Cromwell's unit tests. #A matrix strategy is used so that each subproject can be unit tested in parallel with all other subprojects. #This is what shows up in the github workflows page as the title. -run-name: ${{ github.actor }} running Cromwell sbt Unit Tests. +run-name: ${{ github.actor }} running Cromwell sbt unit tests. #What will trigger the workflow to run. on: @@ -15,8 +15,9 @@ on: permissions: contents: read +#This job is spawned once for each subproject in the matrix below. Each iteration will set up a cache, git secrets, and java before running unit tests for that subproject. jobs: - build-and-test: #This job is spawned once for each subproject in the matrix below. Each iteration will set up a cache, git secrets, and java before running unit tests for that subproject. + build-and-test: strategy: fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. matrix: #this array is the output of the command `sbt projects` from the cromwell root directory, minus a bunch of superflous formatting that sbt adds. @@ -69,11 +70,12 @@ jobs: wom, womtool] - runs-on: ubuntu-latest #Note: This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. + #This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. + runs-on: ubuntu-latest steps: - #Allows this github action to use a cache to store stuff like Java and SBT files between runs. + #Allows this github action to use a cache to store stuff like Java and sbt files between runs. - uses: actions/checkout@v3 name: Checkout Coursier Cache - uses: coursier/cache-action@v6 From 4e0821c77a0a36950c26a3e07fbfd903f7b5ee8e Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:06:31 -0500 Subject: [PATCH 055/213] this will not work --- .github/workflows/cromwell_unit_tests.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 896cfce6908..a0b95dbbf39 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -17,6 +17,15 @@ permissions: #This job is spawned once for each subproject in the matrix below. Each iteration will set up a cache, git secrets, and java before running unit tests for that subproject. jobs: + generate-sbt-project-array: + runs-on: ubuntu-latest + steps: + -id: build-array + -run: output=$(sbt projects \ + | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" \ + | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') + -run: echo $output + build-and-test: strategy: fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. From a1b4f4bb028facb98aaadefd01ad494ccbc0d974 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:08:19 -0500 Subject: [PATCH 056/213] newlines --- .github/workflows/cromwell_unit_tests.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index a0b95dbbf39..d05a58e76a0 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -21,9 +21,7 @@ jobs: runs-on: ubuntu-latest steps: -id: build-array - -run: output=$(sbt projects \ - | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" \ - | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') + -run: output=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') -run: echo $output build-and-test: From 8076020e5e7a646b8258496ab479b9aceab65f0a Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:09:38 -0500 Subject: [PATCH 057/213] yaml --- .github/workflows/cromwell_unit_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index d05a58e76a0..a40a9a7d8e4 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -21,8 +21,8 @@ jobs: runs-on: ubuntu-latest steps: -id: build-array - -run: output=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') - -run: echo $output + run: output=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') + run: echo $output build-and-test: strategy: From d00fe600ce9b134429e1954072bd46af11be4029 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:11:27 -0500 Subject: [PATCH 058/213] error lol --- .github/workflows/cromwell_unit_tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index a40a9a7d8e4..bc0003901ac 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -22,8 +22,6 @@ jobs: steps: -id: build-array run: output=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') - run: echo $output - build-and-test: strategy: fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. From 9272636629ae3733448dbce771235c33f98d1514 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:12:38 -0500 Subject: [PATCH 059/213] pipe --- .github/workflows/cromwell_unit_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index bc0003901ac..9a140269f50 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -21,7 +21,8 @@ jobs: runs-on: ubuntu-latest steps: -id: build-array - run: output=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') + run: | + output=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') build-and-test: strategy: fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. From 2ccbc76bf9982e46151c0f4bb3908759500db2e0 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:15:10 -0500 Subject: [PATCH 060/213] one step closer --- .github/workflows/cromwell_unit_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 9a140269f50..a5c6b681c4e 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -20,8 +20,8 @@ jobs: generate-sbt-project-array: runs-on: ubuntu-latest steps: - -id: build-array - run: | + - name: build-array + run: | output=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') build-and-test: strategy: From 602c5eee9e78fdddcd5eece4a9dc322837f43000 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:16:51 -0500 Subject: [PATCH 061/213] idk --- .github/workflows/cromwell_unit_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index a5c6b681c4e..d581d715631 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -21,7 +21,8 @@ jobs: runs-on: ubuntu-latest steps: - name: build-array - run: | + id: step_one + run: | output=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') build-and-test: strategy: From 53df19c38b26f2d83da7b73af3efa77318bc5529 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:18:40 -0500 Subject: [PATCH 062/213] syntax why --- .github/workflows/cromwell_unit_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index d581d715631..4ee6cb6e637 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -18,8 +18,11 @@ permissions: #This job is spawned once for each subproject in the matrix below. Each iteration will set up a cache, git secrets, and java before running unit tests for that subproject. jobs: generate-sbt-project-array: + runs-on: ubuntu-latest + steps: + - name: build-array id: step_one run: | From cd78e005202513c8ebeae7da06a9c183d95e071f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:21:35 -0500 Subject: [PATCH 063/213] yeye --- .github/workflows/cromwell_unit_tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 4ee6cb6e637..8b4e6f39aed 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -22,11 +22,10 @@ jobs: runs-on: ubuntu-latest steps: - - - name: build-array - id: step_one + - name: Do the thing run: | output=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') + echo $output build-and-test: strategy: fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. From 9a93df0e21d4a69e51e2c7ed472812909db319a5 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:23:17 -0500 Subject: [PATCH 064/213] back to basics --- .github/workflows/cromwell_unit_tests.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 8b4e6f39aed..0a9538a737b 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -17,15 +17,6 @@ permissions: #This job is spawned once for each subproject in the matrix below. Each iteration will set up a cache, git secrets, and java before running unit tests for that subproject. jobs: - generate-sbt-project-array: - - runs-on: ubuntu-latest - - steps: - - name: Do the thing - run: | - output=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') - echo $output build-and-test: strategy: fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. @@ -119,6 +110,12 @@ jobs: cache: sbt #Invoke SBT on a specific subproject, asking it to run all unit tests contained within that subproject. + + - name: Do the thing + run: | + output=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') + echo $output + - name: Run tests run: | set -e From b6916810dd66cd7b4d2dc545645d76ffff513c4f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:32:37 -0500 Subject: [PATCH 065/213] advanced techniques --- .github/workflows/cromwell_unit_tests.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 0a9538a737b..7f4fc097d05 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,9 +112,11 @@ jobs: #Invoke SBT on a specific subproject, asking it to run all unit tests contained within that subproject. - name: Do the thing - run: | - output=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') - echo $output + run: | + echo "SBT_PROJECTS=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + + - name: Test the thing + run: echo ${{ env.SBT_PROJECTS }} - name: Run tests run: | From 897bb7a39c71f89c99a2991360c5c89c0009ebb9 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:35:02 -0500 Subject: [PATCH 066/213] this wont work either --- .github/workflows/cromwell_unit_tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 7f4fc097d05..64ade9fcc84 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -109,8 +109,6 @@ jobs: java-version: 11 cache: sbt - #Invoke SBT on a specific subproject, asking it to run all unit tests contained within that subproject. - - name: Do the thing run: | echo "SBT_PROJECTS=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV @@ -118,6 +116,7 @@ jobs: - name: Test the thing run: echo ${{ env.SBT_PROJECTS }} + #Invoke SBT on a specific subproject, asking it to run all unit tests contained within that subproject. - name: Run tests run: | set -e From 025eea83ad8e3a7c24ae372507433c2922a83bce Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:36:18 -0500 Subject: [PATCH 067/213] nope --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 64ade9fcc84..920e871e4ac 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -110,7 +110,7 @@ jobs: cache: sbt - name: Do the thing - run: | + run: | echo "SBT_PROJECTS=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing From 89a560781375ab3899e7171670095f83dae6cecb Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:41:07 -0500 Subject: [PATCH 068/213] try again --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 920e871e4ac..46f28e69fec 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -111,7 +111,7 @@ jobs: - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed -e "s/^\[info\]\s:*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects| sed -e "s/^\[info\][[:blank:]]*\([-_A-Za-z0-9]*\)$/\1/g" grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing run: echo ${{ env.SBT_PROJECTS }} From f155b4982b2321358404d1a024af0faa9cd68b9c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:44:09 -0500 Subject: [PATCH 069/213] aisd --- .github/workflows/cromwell_unit_tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 46f28e69fec..8053e8d40e3 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -111,10 +111,11 @@ jobs: - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects| sed -e "s/^\[info\][[:blank:]]*\([-_A-Za-z0-9]*\)$/\1/g" grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects| sed -e "s/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g" grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing - run: echo ${{ env.SBT_PROJECTS }} + run: | + echo ${{ env.SBT_PROJECTS }} #Invoke SBT on a specific subproject, asking it to run all unit tests contained within that subproject. - name: Run tests From f28381ed52a8264516e7ae5569bd775583013e71 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:45:45 -0500 Subject: [PATCH 070/213] pipe got lost --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 8053e8d40e3..5ea92299536 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -111,7 +111,7 @@ jobs: - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects| sed -e "s/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g" grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects| sed -e "s/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing run: | From e978ad49f854c64a1aa10bf02b87dc4e43b3f7e0 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:56:49 -0500 Subject: [PATCH 071/213] quotatsions --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 5ea92299536..3ce32a57985 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -111,7 +111,7 @@ jobs: - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects| sed -e "s/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g" | grep -v "^\[info\].*" | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects| sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*'' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing run: | From e4f6f825c8416103317985645fe3706f25ada954 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 14:59:26 -0500 Subject: [PATCH 072/213] trying --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 3ce32a57985..d609d9f444d 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -111,7 +111,7 @@ jobs: - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects| sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*'' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\][[:blank:]]*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing run: | From 8d702dea2940154b68025bfd629b6b7c4e28096a Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 15:02:04 -0500 Subject: [PATCH 073/213] \s --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index d609d9f444d..369750ff16f 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -111,7 +111,7 @@ jobs: - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\][[:blank:]]*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing run: | From 778af442e1722afa7ee787ac7ba89fb054c5000f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 15:07:49 -0500 Subject: [PATCH 074/213] rome was built from the command line --- .github/workflows/cromwell_unit_tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 369750ff16f..408dd71607a 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -109,10 +109,11 @@ jobs: java-version: 11 cache: sbt + # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - + echo "SBT_PROJECTS=$(sbt projects)" + - name: Test the thing run: | echo ${{ env.SBT_PROJECTS }} From 7eff6891764dc2564caf5ab9884ac2034f232eb2 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 15:09:03 -0500 Subject: [PATCH 075/213] i cant keep coming up with commit messages --- .github/workflows/cromwell_unit_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 408dd71607a..fc43b7ff64e 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,8 +112,8 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects)" - + echo "SBT_PROJECTS=$(sbt projects)" >> $GITHUB_ENV + - name: Test the thing run: | echo ${{ env.SBT_PROJECTS }} From a62a7549c916a92c225ad6f53124dda9ec5e4faf Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 30 Jan 2023 15:29:33 -0500 Subject: [PATCH 076/213] working? --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index fc43b7ff64e..e5ce2ba6a51 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects)" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed -e 's/\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing run: | From 11101fb588ec868a735354e97a99d587be70f5bb Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 15:47:04 -0500 Subject: [PATCH 077/213] Escape asterisks --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index e5ce2ba6a51..60e9abafd8a 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed -e 's/\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed -e 's/\[info\]\s\*\([-_A-Za-z0-9]\*\)$/\1/g' | grep -v '\[info\].\*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing run: | From c4341daf0c2729e04586953a5c3becfcece8f4bf Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 15:55:19 -0500 Subject: [PATCH 078/213] Does this work? --- .github/workflows/cromwell_unit_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 60e9abafd8a..5f204bc1d96 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,8 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed -e 's/\[info\]\s\*\([-_A-Za-z0-9]\*\)$/\1/g' | grep -v '\[info\].\*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + SBT_PROJ=$(sbt projects | sed 's/\[info\]\s\*\([-_A-Za-z0-9]\*\)$/\1/g' | grep -v '\[info\]' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') + echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing run: | From 39e62706066968151b392ca8d2459393faaef07e Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 16:00:37 -0500 Subject: [PATCH 079/213] Also escape dollar sign --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 5f204bc1d96..f665d155aaf 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\[info\]\s\*\([-_A-Za-z0-9]\*\)$/\1/g' | grep -v '\[info\]' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') + SBT_PROJ=$(sbt projects | sed 's/\[info\]\s\*\([-_A-Za-z0-9]\*\)\$/\1/g' | grep -v '\[info\]' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From ceed50eb1979fc605860106a98025a0eed78999b Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 16:14:30 -0500 Subject: [PATCH 080/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index f665d155aaf..56b03f5c08e 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\[info\]\s\*\([-_A-Za-z0-9]\*\)\$/\1/g' | grep -v '\[info\]' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') + SBT_PROJ=$(sbt projects | sed 's/\[info\]/INFO/g' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From df46ef6f0d8a09efc2e9109c68265b2bcda556ac Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 16:16:44 -0500 Subject: [PATCH 081/213] More escaping --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 56b03f5c08e..3fdee572999 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\[info\]/INFO/g' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') + SBT_PROJ=$(sbt projects | sed 's/\\[info\\]/INFO/g' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From de6704baa659fbe69a72c831064da93a294afc76 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 16:19:51 -0500 Subject: [PATCH 082/213] Less escaping --- .github/workflows/cromwell_unit_tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 3fdee572999..894d3e4eddf 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\\[info\\]/INFO/g' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') + SBT_PROJ=$(sbt projects | sed 's/[info]/INFO/g' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing @@ -120,7 +120,7 @@ jobs: echo ${{ env.SBT_PROJECTS }} #Invoke SBT on a specific subproject, asking it to run all unit tests contained within that subproject. - - name: Run tests - run: | - set -e - sbt "${{ matrix.sbtSubProject }}/test" \ No newline at end of file +# - name: Run tests +# run: | +# set -e +# sbt "${{ matrix.sbtSubProject }}/test" \ No newline at end of file From 9afc8c98e0644940ac5da2850af6d9a1b10707db Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 16:26:52 -0500 Subject: [PATCH 083/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 98 +++++++++++------------ 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 894d3e4eddf..2836d73d5ed 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -21,54 +21,54 @@ jobs: strategy: fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. matrix: #this array is the output of the command `sbt projects` from the cromwell root directory, minus a bunch of superflous formatting that sbt adds. - sbtSubProject: [awsBackend, - awsS3FileSystem, - azureBlobFileSystem, - backend, - centaur, - cloud-nio-impl-drs, - cloud-nio-impl-ftp, - cloud-nio-spi, - cloud-nio-util, - cloudSupport, - common, - core, - cromiam, - cromwell-drs-localizer, - cromwellApiClient, - databaseMigration, - databaseSql, - dockerHashing, - drsFileSystem, - engine, - ftpFileSystem, - gcsFileSystem, - googlePipelinesCommon, - googlePipelinesV2Alpha1, - googlePipelinesV2Beta, - httpFileSystem, - languageFactoryCore, - perf, - server, - services, - sfsBackend, - sraFileSystem, - tesBackend, - wdlBiscayneLanguageFactory, - wdlDraft2LanguageFactory, - wdlDraft3LanguageFactory, - wdlModelBiscayne, - wdlModelDraft2, - wdlModelDraft3, - wdlNewBaseTransforms, - wdlSharedModel, - wdlSharedTransforms, - wdlTransformsBiscayne, - wdlTransformsDraft2, - wdlTransformsDraft3, - wes2cromwell, - wom, - womtool] + sbtSubProject: [awsBackend] +# awsS3FileSystem, +# azureBlobFileSystem, +# backend, +# centaur, +# cloud-nio-impl-drs, +# cloud-nio-impl-ftp, +# cloud-nio-spi, +# cloud-nio-util, +# cloudSupport, +# common, +# core, +# cromiam, +# cromwell-drs-localizer, +# cromwellApiClient, +# databaseMigration, +# databaseSql, +# dockerHashing, +# drsFileSystem, +# engine, +# ftpFileSystem, +# gcsFileSystem, +# googlePipelinesCommon, +# googlePipelinesV2Alpha1, +# googlePipelinesV2Beta, +# httpFileSystem, +# languageFactoryCore, +# perf, +# server, +# services, +# sfsBackend, +# sraFileSystem, +# tesBackend, +# wdlBiscayneLanguageFactory, +# wdlDraft2LanguageFactory, +# wdlDraft3LanguageFactory, +# wdlModelBiscayne, +# wdlModelDraft2, +# wdlModelDraft3, +# wdlNewBaseTransforms, +# wdlSharedModel, +# wdlSharedTransforms, +# wdlTransformsBiscayne, +# wdlTransformsDraft2, +# wdlTransformsDraft3, +# wes2cromwell, +# wom, +# womtool] #This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. runs-on: ubuntu-latest @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/[info]/INFO/g' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/') + SBT_PROJ=$(sbt projects | sed 's/\[info\]/INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From c9640c607bdc3ffcf455972610b2427e6da89ed0 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 16:31:34 -0500 Subject: [PATCH 084/213] EVEN MORE ESCAPING --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 2836d73d5ed..0f4262cd343 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\[info\]/INFO/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/\\\[info\\\]/INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From 29cc772d55d0dbcd41d8a3a92fc4a07fa35c4f9b Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 16:42:17 -0500 Subject: [PATCH 085/213] What if we didn't use brackets at all --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 0f4262cd343..bf89cc8278b 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\\\[info\\\]/INFO/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/^.info./INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From fc01ca7909639f1d419c847378b731630a860887 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 16:47:30 -0500 Subject: [PATCH 086/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index bf89cc8278b..5fcc2f47de7 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/^.info./INFO/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/.info./INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From 76290cd8dc2cbb25867999560fac3654a9faf80d Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 16:54:32 -0500 Subject: [PATCH 087/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 5fcc2f47de7..b498fc2e743 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/.info./INFO/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/^.info.\s*/INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From c924572ef4de85e3bfc72e937ada5afe9253ebf0 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 16:56:10 -0500 Subject: [PATCH 088/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index b498fc2e743..9467ab299b5 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/^.info.\s*/INFO/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/\^\.info\.\\s*/INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From 997f5a7155176559e613c8c1f13af0cb5114b848 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 16:58:19 -0500 Subject: [PATCH 089/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 9467ab299b5..fbaabeda996 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\^\.info\.\\s*/INFO/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/\\^\\.info\\.\\\\s*/INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From 3cfee4ad8c1ea540c230dd3875cf34000e563e6b Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 17:00:49 -0500 Subject: [PATCH 090/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index fbaabeda996..0504759f273 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\\^\\.info\\.\\\\s*/INFO/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/info/INFO/g') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From 04709630e306c7182e19ffb87d9293520b96f221 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 17:05:46 -0500 Subject: [PATCH 091/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 0504759f273..33b6548c147 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/info/INFO/g') + SBT_PROJ=$(sbt projects | sed 's/\.info\./INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From 5d11931eb0b557abab0f9d46ee788e62581bc336 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 17:07:27 -0500 Subject: [PATCH 092/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 33b6548c147..5fcc2f47de7 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\.info\./INFO/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/.info./INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From e83e7a4a6073f5318d23b1fd79e46cfa4e3eea54 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 19:07:47 -0500 Subject: [PATCH 093/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 5fcc2f47de7..f7632dd52ee 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/.info./INFO/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/^\[.*info.*\]/INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From 678f77a456e11d5a5bfec1aeaff4eb83e9584b82 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 19:11:21 -0500 Subject: [PATCH 094/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index f7632dd52ee..4f5f4afff7f 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/^\[.*info.*\]/INFO/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/^\\\[.*info.*\\\]/INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From cc209c4398bafad5246ad7eacba558d2d2cffb41 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 19:14:28 -0500 Subject: [PATCH 095/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 4f5f4afff7f..7ffe1fcc8f7 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/^\\\[.*info.*\\\]/INFO/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/\[.*info.*\]/INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From 069c1a6de4f1f1df82e3764e5225a50146ee250d Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 19:17:21 -0500 Subject: [PATCH 096/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 7ffe1fcc8f7..34e7088af79 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\[.*info.*\]/INFO/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/\[.*info.*\]\s*(.*)/\1/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From 8368318174831cc9875bab49ebb8be2bba1dffd1 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 19:18:24 -0500 Subject: [PATCH 097/213] Testing --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 34e7088af79..9493c0182d1 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\[.*info.*\]\s*(.*)/\1/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/\[.*info.*\]\s*\(.*\)/\1/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From 9c8b38ce8091b3558528fae18bf48dc35717d23f Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 19:31:09 -0500 Subject: [PATCH 098/213] Remove color codes --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 9493c0182d1..3f9c5d1fb2a 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\[.*info.*\]\s*\(.*\)/\1/g' | tr '\n', ',') + SBT_PROJ=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed 's/\[info\]/INFO/g' | tr '\n', ',') echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV - name: Test the thing From db128afb3a6d5a05772693e29f4bad797f3cf965 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 19:34:22 -0500 Subject: [PATCH 099/213] Try the whole thing --- .github/workflows/cromwell_unit_tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 3f9c5d1fb2a..325a392533d 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,8 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJ=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed 's/\[info\]/INFO/g' | tr '\n', ',') - echo "SBT_PROJECTS='${SBT_PROJ}'" >> $GITHUB_ENV + SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing run: | From 461c0aa49e7448dd27947568f4d553e3ed717caf Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 19:36:50 -0500 Subject: [PATCH 100/213] Oops --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 325a392533d..e5556a05bc8 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -112,7 +112,7 @@ jobs: # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing run: | From 6358c75e43135bd4af00cdd006abdb79164a5c1d Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 19:41:02 -0500 Subject: [PATCH 101/213] Revert test changes, remove trailing comma --- .github/workflows/cromwell_unit_tests.yml | 107 +++++++++++----------- 1 file changed, 53 insertions(+), 54 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index e5556a05bc8..9e9e8387e52 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -21,54 +21,54 @@ jobs: strategy: fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. matrix: #this array is the output of the command `sbt projects` from the cromwell root directory, minus a bunch of superflous formatting that sbt adds. - sbtSubProject: [awsBackend] -# awsS3FileSystem, -# azureBlobFileSystem, -# backend, -# centaur, -# cloud-nio-impl-drs, -# cloud-nio-impl-ftp, -# cloud-nio-spi, -# cloud-nio-util, -# cloudSupport, -# common, -# core, -# cromiam, -# cromwell-drs-localizer, -# cromwellApiClient, -# databaseMigration, -# databaseSql, -# dockerHashing, -# drsFileSystem, -# engine, -# ftpFileSystem, -# gcsFileSystem, -# googlePipelinesCommon, -# googlePipelinesV2Alpha1, -# googlePipelinesV2Beta, -# httpFileSystem, -# languageFactoryCore, -# perf, -# server, -# services, -# sfsBackend, -# sraFileSystem, -# tesBackend, -# wdlBiscayneLanguageFactory, -# wdlDraft2LanguageFactory, -# wdlDraft3LanguageFactory, -# wdlModelBiscayne, -# wdlModelDraft2, -# wdlModelDraft3, -# wdlNewBaseTransforms, -# wdlSharedModel, -# wdlSharedTransforms, -# wdlTransformsBiscayne, -# wdlTransformsDraft2, -# wdlTransformsDraft3, -# wes2cromwell, -# wom, -# womtool] + sbtSubProject: [awsBackend, + awsS3FileSystem, + azureBlobFileSystem, + backend, + centaur, + cloud-nio-impl-drs, + cloud-nio-impl-ftp, + cloud-nio-spi, + cloud-nio-util, + cloudSupport, + common, + core, + cromiam, + cromwell-drs-localizer, + cromwellApiClient, + databaseMigration, + databaseSql, + dockerHashing, + drsFileSystem, + engine, + ftpFileSystem, + gcsFileSystem, + googlePipelinesCommon, + googlePipelinesV2Alpha1, + googlePipelinesV2Beta, + httpFileSystem, + languageFactoryCore, + perf, + server, + services, + sfsBackend, + sraFileSystem, + tesBackend, + wdlBiscayneLanguageFactory, + wdlDraft2LanguageFactory, + wdlDraft3LanguageFactory, + wdlModelBiscayne, + wdlModelDraft2, + wdlModelDraft3, + wdlNewBaseTransforms, + wdlSharedModel, + wdlSharedTransforms, + wdlTransformsBiscayne, + wdlTransformsDraft2, + wdlTransformsDraft3, + wes2cromwell, + wom, + womtool] #This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. runs-on: ubuntu-latest @@ -109,17 +109,16 @@ jobs: java-version: 11 cache: sbt - # echo "SBT_PROJECTS=$(sbt projects | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.,$/]\n/')" >> $GITHUB_ENV - name: Test the thing run: | echo ${{ env.SBT_PROJECTS }} #Invoke SBT on a specific subproject, asking it to run all unit tests contained within that subproject. -# - name: Run tests -# run: | -# set -e -# sbt "${{ matrix.sbtSubProject }}/test" \ No newline at end of file + - name: Run tests + run: | + set -e + sbt "${{ matrix.sbtSubProject }}/test" \ No newline at end of file From 52ee24fe3384528423e03b04311487fd611c626b Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 30 Jan 2023 19:45:47 -0500 Subject: [PATCH 102/213] Leave comma to deal with later, add comment --- .github/workflows/cromwell_unit_tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 9e9e8387e52..fb04d5eacd0 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -109,9 +109,11 @@ jobs: java-version: 11 cache: sbt + #Parse the sbt-generated list of projects into a YAML array. Note that we strip + #invisible color codes out of the `sbt projects` output. - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.,$/]\n/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing run: | From 15d93027a4daa161739585598dda2e84f786db0d Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 10:45:37 -0500 Subject: [PATCH 103/213] here goes nothing --- .github/set_up_cromwell_action/action.yml | 43 +++++++++++++++++++++++ .github/workflows/cromwell_unit_tests.yml | 36 ++----------------- 2 files changed, 45 insertions(+), 34 deletions(-) create mode 100644 .github/set_up_cromwell_action/action.yml diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml new file mode 100644 index 00000000000..55ead7a09e9 --- /dev/null +++ b/.github/set_up_cromwell_action/action.yml @@ -0,0 +1,43 @@ +name: 'Set Up Cromwell Steps' +#This is a series of steps that will setup Java/sbt/Cromwell on the local runner. +#These steps are meant to be re-used and invoked by other Github Action Workflows that wish to have cromwell up-and-running before they do their thing. + +runs: + using: "composite" # <-- this allows these steps to be used by other workflows. + steps: + #Allows this github action to use a cache to store stuff like Java and sbt files between runs. + - uses: actions/checkout@v3 + name: Checkout Coursier Cache + - uses: coursier/cache-action@v6 + name: Enable Coursier Cache + + #Cromwell requires git-secrets be setup. Here, we set up secrets and verify success with a script. + - name: Git secrets setup + run: | + git clone https://github.com/awslabs/git-secrets.git ~/git-secrets + cd ~/git-secrets + git checkout ad82d68ee924906a0401dfd48de5057731a9bc84 + sudo make install + shell: bash + + - name: Secrets check + run: | + sudo ln -s "$(which echo)" /usr/local/bin/say + ./minnie-kenny.sh --force + git secrets --scan-history + shell: bash + + #Clone the cromwell repo to this VM. + - name: Clone Cromwell + uses: actions/checkout@v3 + with: + repository: broadinstitute/cromwell + token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + + #Install Java to this VM. This Java version and distribution is compatible with Cromwell. + - name: Setup JDK + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 11 + cache: sbt diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index fb04d5eacd0..de59a3ecf77 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -74,40 +74,8 @@ jobs: runs-on: ubuntu-latest steps: - - #Allows this github action to use a cache to store stuff like Java and sbt files between runs. - - uses: actions/checkout@v3 - name: Checkout Coursier Cache - - uses: coursier/cache-action@v6 - name: Enable Coursier Cache - - #Cromwell requires git-secrets be setup. Here, we set up secrets and verify success with a script. - - name: Git secrets setup - run: | - git clone https://github.com/awslabs/git-secrets.git ~/git-secrets - cd ~/git-secrets - git checkout ad82d68ee924906a0401dfd48de5057731a9bc84 - sudo make install - - name: Secrets check - run: | - sudo ln -s "$(which echo)" /usr/local/bin/say - ./minnie-kenny.sh --force - git secrets --scan-history - - #Clone the cromwell repo to this VM. - - name: Clone Cromwell - uses: actions/checkout@v3 - with: - repository: broadinstitute/cromwell - token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} - - #Install Java to this VM. This Java version and distribution is compatible with Cromwell. - - name: Setup JDK - uses: actions/setup-java@v3 - with: - distribution: temurin - java-version: 11 - cache: sbt + - uses: actions/checkout@v3 # required first to checkout the repo + - uses: ./.github/set_up_cromwell_action #Parse the sbt-generated list of projects into a YAML array. Note that we strip #invisible color codes out of the `sbt projects` output. From bf91a067a9c0f5f244af1198f220b9201b9ca66c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 10:47:31 -0500 Subject: [PATCH 104/213] whitespace --- .github/workflows/cromwell_unit_tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index de59a3ecf77..5b5b2401037 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -74,9 +74,8 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 # required first to checkout the repo - - uses: ./.github/set_up_cromwell_action - + - uses: actions/checkout@v3 # required first to checkout the repo + - uses: ./.github/set_up_cromwell_action #Parse the sbt-generated list of projects into a YAML array. Note that we strip #invisible color codes out of the `sbt projects` output. - name: Do the thing From 74796d45f948d7d3825927c3245fd6c918c85986 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 10:54:59 -0500 Subject: [PATCH 105/213] using secrets --- .github/set_up_cromwell_action/action.yml | 6 +++++- .github/workflows/cromwell_unit_tests.yml | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml index 55ead7a09e9..67944c59c44 100644 --- a/.github/set_up_cromwell_action/action.yml +++ b/.github/set_up_cromwell_action/action.yml @@ -1,4 +1,8 @@ name: 'Set Up Cromwell Steps' +description: Specific steps that will set up git secrets, java, sbt, and Cromwell on the local machine. +inputs: + cromwell_repo_token: + required: true #This is a series of steps that will setup Java/sbt/Cromwell on the local runner. #These steps are meant to be re-used and invoked by other Github Action Workflows that wish to have cromwell up-and-running before they do their thing. @@ -32,7 +36,7 @@ runs: uses: actions/checkout@v3 with: repository: broadinstitute/cromwell - token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + token: ${{ inputs.cromwell_repo_token }} #Install Java to this VM. This Java version and distribution is compatible with Cromwell. - name: Setup JDK diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 5b5b2401037..2d2cb028739 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -76,6 +76,9 @@ jobs: steps: - uses: actions/checkout@v3 # required first to checkout the repo - uses: ./.github/set_up_cromwell_action + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + #Parse the sbt-generated list of projects into a YAML array. Note that we strip #invisible color codes out of the `sbt projects` output. - name: Do the thing From 247f412423ee007d58f38c60aecdb6fb2b9d649f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:00:54 -0500 Subject: [PATCH 106/213] comments --- .github/set_up_cromwell_action/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml index 67944c59c44..038aa2b0346 100644 --- a/.github/set_up_cromwell_action/action.yml +++ b/.github/set_up_cromwell_action/action.yml @@ -1,10 +1,10 @@ +#This is a series of steps that will setup Java/sbt/Cromwell on the local runner. +#These steps are meant to be re-used and invoked by other Github Action Workflows name: 'Set Up Cromwell Steps' description: Specific steps that will set up git secrets, java, sbt, and Cromwell on the local machine. inputs: - cromwell_repo_token: + cromwell_repo_token: #As an input to this action, you are required to pass in a token that can be used to authenticate while checking out Cromwell. required: true -#This is a series of steps that will setup Java/sbt/Cromwell on the local runner. -#These steps are meant to be re-used and invoked by other Github Action Workflows that wish to have cromwell up-and-running before they do their thing. runs: using: "composite" # <-- this allows these steps to be used by other workflows. From d184b6f735efe21c37e59d01aea33a7ab15e8430 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:15:30 -0500 Subject: [PATCH 107/213] yolo --- .github/workflows/cromwell_unit_tests.yml | 76 +++++++---------------- 1 file changed, 24 insertions(+), 52 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 2d2cb028739..0459d837cfa 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -15,60 +15,34 @@ on: permissions: contents: read -#This job is spawned once for each subproject in the matrix below. Each iteration will set up a cache, git secrets, and java before running unit tests for that subproject. jobs: + generate-subproject-matrix: #This job will invoke SBT to get a list of all subprojects in Cromwell. Later, this list will be used to teach each subproject in parallel. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 # required first to checkout the repo + - uses: ./.github/set_up_cromwell_action + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + + #Parse the sbt-generated list of projects into a YAML array. Note that we strip + #invisible color codes out of the `sbt projects` output. + - name: Do the thing + run: | + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + + - name: Test the thing + run: | + echo ${{ env.SBT_PROJECTS }} + + outputs: + subproject-matrix: ${{ env.SBT_PROJECTS }} + build-and-test: + needs: generate-subproject-matrix strategy: fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. matrix: #this array is the output of the command `sbt projects` from the cromwell root directory, minus a bunch of superflous formatting that sbt adds. - sbtSubProject: [awsBackend, - awsS3FileSystem, - azureBlobFileSystem, - backend, - centaur, - cloud-nio-impl-drs, - cloud-nio-impl-ftp, - cloud-nio-spi, - cloud-nio-util, - cloudSupport, - common, - core, - cromiam, - cromwell-drs-localizer, - cromwellApiClient, - databaseMigration, - databaseSql, - dockerHashing, - drsFileSystem, - engine, - ftpFileSystem, - gcsFileSystem, - googlePipelinesCommon, - googlePipelinesV2Alpha1, - googlePipelinesV2Beta, - httpFileSystem, - languageFactoryCore, - perf, - server, - services, - sfsBackend, - sraFileSystem, - tesBackend, - wdlBiscayneLanguageFactory, - wdlDraft2LanguageFactory, - wdlDraft3LanguageFactory, - wdlModelBiscayne, - wdlModelDraft2, - wdlModelDraft3, - wdlNewBaseTransforms, - wdlSharedModel, - wdlSharedTransforms, - wdlTransformsBiscayne, - wdlTransformsDraft2, - wdlTransformsDraft3, - wes2cromwell, - wom, - womtool] + sbtSubProject: ${{ needs.generate-subproject-matrix.outputs.subproject-matrix }} #This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. runs-on: ubuntu-latest @@ -85,9 +59,7 @@ jobs: run: | echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - - name: Test the thing - run: | - echo ${{ env.SBT_PROJECTS }} + #Invoke SBT on a specific subproject, asking it to run all unit tests contained within that subproject. - name: Run tests From 68fc894f375d6f6aaf939c6d5cccccc2c1e8ba8c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:17:52 -0500 Subject: [PATCH 108/213] indent --- .github/workflows/cromwell_unit_tests.yml | 32 +++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 0459d837cfa..c0be60a5b12 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -17,25 +17,25 @@ permissions: jobs: generate-subproject-matrix: #This job will invoke SBT to get a list of all subprojects in Cromwell. Later, this list will be used to teach each subproject in parallel. - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 # required first to checkout the repo - - uses: ./.github/set_up_cromwell_action - with: - cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 # required first to checkout the repo + - uses: ./.github/set_up_cromwell_action + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} - #Parse the sbt-generated list of projects into a YAML array. Note that we strip - #invisible color codes out of the `sbt projects` output. - - name: Do the thing - run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + #Parse the sbt-generated list of projects into a YAML array. Note that we strip + #invisible color codes out of the `sbt projects` output. + - name: Do the thing + run: | + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - - name: Test the thing - run: | - echo ${{ env.SBT_PROJECTS }} + - name: Test the thing + run: | + echo ${{ env.SBT_PROJECTS }} - outputs: - subproject-matrix: ${{ env.SBT_PROJECTS }} + outputs: + subproject-matrix: ${{ env.SBT_PROJECTS }} build-and-test: needs: generate-subproject-matrix From 962c04f2846e53a238bbc326d1caf57e0c4c1277 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:20:31 -0500 Subject: [PATCH 109/213] idgetit --- .github/workflows/cromwell_unit_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index c0be60a5b12..ae29152f51b 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -18,6 +18,7 @@ permissions: jobs: generate-subproject-matrix: #This job will invoke SBT to get a list of all subprojects in Cromwell. Later, this list will be used to teach each subproject in parallel. runs-on: ubuntu-latest + steps: - uses: actions/checkout@v3 # required first to checkout the repo - uses: ./.github/set_up_cromwell_action From 0302bc6878853938874a5f8291dd284dbec4c3fe Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:22:50 -0500 Subject: [PATCH 110/213] indent --- .github/workflows/cromwell_unit_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index ae29152f51b..c0be60a5b12 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -18,7 +18,6 @@ permissions: jobs: generate-subproject-matrix: #This job will invoke SBT to get a list of all subprojects in Cromwell. Later, this list will be used to teach each subproject in parallel. runs-on: ubuntu-latest - steps: - uses: actions/checkout@v3 # required first to checkout the repo - uses: ./.github/set_up_cromwell_action From afe4d3b0ad135be035926146ab0ece61c4f29ebe Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:25:25 -0500 Subject: [PATCH 111/213] this is dumb --- .github/workflows/cromwell_unit_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index c0be60a5b12..1eea6ef6ae2 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -17,6 +17,7 @@ permissions: jobs: generate-subproject-matrix: #This job will invoke SBT to get a list of all subprojects in Cromwell. Later, this list will be used to teach each subproject in parallel. + name: Get sbt subprojects runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 # required first to checkout the repo From 01fbecec520953f8e6b3b48e5dc7045a7ba52a89 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:26:33 -0500 Subject: [PATCH 112/213] copypasta --- .github/workflows/cromwell_unit_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 1eea6ef6ae2..ae9ef77e87a 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -19,6 +19,7 @@ jobs: generate-subproject-matrix: #This job will invoke SBT to get a list of all subprojects in Cromwell. Later, this list will be used to teach each subproject in parallel. name: Get sbt subprojects runs-on: ubuntu-latest + steps: - uses: actions/checkout@v3 # required first to checkout the repo - uses: ./.github/set_up_cromwell_action From 37bca130ed7138aa101deaf573abb7bb8410dee1 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:28:55 -0500 Subject: [PATCH 113/213] asd --- .github/workflows/cromwell_unit_tests.yml | 27 ++++++++++------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index ae9ef77e87a..41c69f3e9ad 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -19,25 +19,24 @@ jobs: generate-subproject-matrix: #This job will invoke SBT to get a list of all subprojects in Cromwell. Later, this list will be used to teach each subproject in parallel. name: Get sbt subprojects runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 # required first to checkout the repo - - uses: ./.github/set_up_cromwell_action - with: - cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + - uses: actions/checkout@v3 # required first to checkout the repo + - uses: ./.github/set_up_cromwell_action + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} #Parse the sbt-generated list of projects into a YAML array. Note that we strip #invisible color codes out of the `sbt projects` output. - - name: Do the thing - run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + - name: Do the thing + run: | + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - - name: Test the thing - run: | - echo ${{ env.SBT_PROJECTS }} + - name: Test the thing + un: | + echo ${{ env.SBT_PROJECTS }} - outputs: - subproject-matrix: ${{ env.SBT_PROJECTS }} + outputs: + subproject-matrix: ${{ env.SBT_PROJECTS }} build-and-test: needs: generate-subproject-matrix @@ -61,8 +60,6 @@ jobs: run: | echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - - #Invoke SBT on a specific subproject, asking it to run all unit tests contained within that subproject. - name: Run tests run: | From 019f33323c0512c3b431796a3f2901704927a22b Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:29:39 -0500 Subject: [PATCH 114/213] actionable --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 41c69f3e9ad..1da6957df84 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -32,7 +32,7 @@ jobs: echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing - un: | + run: | echo ${{ env.SBT_PROJECTS }} outputs: From 5aa1271461543ba9a4d31e172470b2937434839c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:31:07 -0500 Subject: [PATCH 115/213] whitespace --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 1da6957df84..5bacb811975 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -32,7 +32,7 @@ jobs: echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - name: Test the thing - run: | + run: | echo ${{ env.SBT_PROJECTS }} outputs: From e9dc5f631d4977f0430114ffb07ad432785e1c6f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:41:09 -0500 Subject: [PATCH 116/213] remove newline: --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 5bacb811975..1e746d661fa 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -29,7 +29,7 @@ jobs: #invisible color codes out of the `sbt projects` output. - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]/')" >> $GITHUB_ENV - name: Test the thing run: | From 857ad95da51adb6f99111c31bc0210abecd14f58 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:50:57 -0500 Subject: [PATCH 117/213] lets seeeeee --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 1e746d661fa..8c9351df436 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -29,7 +29,7 @@ jobs: #invisible color codes out of the `sbt projects` output. - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/,[^,]*\]$/\]/g')" >> $GITHUB_ENV - name: Test the thing run: | From 97b3666554044093501ead2e6b48b99f95dd0c5f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 11:53:20 -0500 Subject: [PATCH 118/213] what is even in there: --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 8c9351df436..c6013fda23a 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -29,7 +29,7 @@ jobs: #invisible color codes out of the `sbt projects` output. - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/,[^,]*\]$/\]/g')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/')" >> $GITHUB_ENV - name: Test the thing run: | From 7fb8069ce6dbad2215ee4251ba052fc1cbacf372 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 12:01:52 -0500 Subject: [PATCH 119/213] the ol doubble commar --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index c6013fda23a..db97dc7b83c 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -29,7 +29,7 @@ jobs: #invisible color codes out of the `sbt projects` output. - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/,,$/]/')" >> $GITHUB_ENV - name: Test the thing run: | From a78c30fb9143e4dcc3a2505b36c1fe43f73d9797 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 12:05:24 -0500 Subject: [PATCH 120/213] the double dot --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index db97dc7b83c..257a9cd2dab 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -29,7 +29,7 @@ jobs: #invisible color codes out of the `sbt projects` output. - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/,,$/]/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/..$/]/')" >> $GITHUB_ENV - name: Test the thing run: | From f4d72160d600b70f9bc2ce20af9e3ce75ef682f3 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 12:11:16 -0500 Subject: [PATCH 121/213] dont ask questions --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 257a9cd2dab..0c5640a72e1 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -29,7 +29,7 @@ jobs: #invisible color codes out of the `sbt projects` output. - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/..$/]/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/,{2,}/]/')" >> $GITHUB_ENV - name: Test the thing run: | From 54aaee3292e0bc10b885522a0e58ada08fecf16d Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 12:14:12 -0500 Subject: [PATCH 122/213] y --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 0c5640a72e1..af7baa835d9 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -29,7 +29,7 @@ jobs: #invisible color codes out of the `sbt projects` output. - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/,{2,}/]/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/,\{2,\}/]/g')" >> $GITHUB_ENV - name: Test the thing run: | From 8403b9f9b5bbb4c22346291c2ca041593b7f817f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 12:19:14 -0500 Subject: [PATCH 123/213] 2 --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index af7baa835d9..691c5f52106 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -29,7 +29,7 @@ jobs: #invisible color codes out of the `sbt projects` output. - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/,\{2,\}/]/g')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' |sed 's/.\{2\}$//')" >> $GITHUB_ENV - name: Test the thing run: | From fcf9c8f2a1f9259a9aefc1a90aa09b6b503902bc Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 12:21:35 -0500 Subject: [PATCH 124/213] 5 --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 691c5f52106..639669d0ffa 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -29,7 +29,7 @@ jobs: #invisible color codes out of the `sbt projects` output. - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' |sed 's/.\{2\}$//')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' |sed 's/.\{5\}$//')" >> $GITHUB_ENV - name: Test the thing run: | From eaa48520b878ee4cea5aea22dbb471d9ba509b8d Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 12:24:58 -0500 Subject: [PATCH 125/213] lol --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 639669d0ffa..c832c58de4d 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -29,7 +29,7 @@ jobs: #invisible color codes out of the `sbt projects` output. - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' |sed 's/.\{5\}$//')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' |sed 's/.\{6\}$/]/')" >> $GITHUB_ENV - name: Test the thing run: | From 26091461d252dbfcacca656d6d8e758fe2cc8286 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 12:34:29 -0500 Subject: [PATCH 126/213] whitespoace --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index c832c58de4d..e301bfa3c87 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -29,7 +29,7 @@ jobs: #invisible color codes out of the `sbt projects` output. - name: Do the thing run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' |sed 's/.\{6\}$/]/')" >> $GITHUB_ENV + echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' |sed 's/.\{6\}$/]/' | sed ':a; N; s/[[:space:]]//g; ta')" >> $GITHUB_ENV - name: Test the thing run: | From f4837f80f6a2d69eba6f83f2af2f1408658f1e37 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 13:07:38 -0500 Subject: [PATCH 127/213] a --- .github/workflows/cromwell_unit_tests.yml | 74 +++++++++++++++-------- 1 file changed, 50 insertions(+), 24 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index e301bfa3c87..7e7d684f399 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -16,34 +16,60 @@ permissions: contents: read jobs: - generate-subproject-matrix: #This job will invoke SBT to get a list of all subprojects in Cromwell. Later, this list will be used to teach each subproject in parallel. - name: Get sbt subprojects - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 # required first to checkout the repo - - uses: ./.github/set_up_cromwell_action - with: - cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} - - #Parse the sbt-generated list of projects into a YAML array. Note that we strip - #invisible color codes out of the `sbt projects` output. - - name: Do the thing - run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' |sed 's/.\{6\}$/]/' | sed ':a; N; s/[[:space:]]//g; ta')" >> $GITHUB_ENV - - - name: Test the thing - run: | - echo ${{ env.SBT_PROJECTS }} - - outputs: - subproject-matrix: ${{ env.SBT_PROJECTS }} - build-and-test: - needs: generate-subproject-matrix strategy: fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. matrix: #this array is the output of the command `sbt projects` from the cromwell root directory, minus a bunch of superflous formatting that sbt adds. - sbtSubProject: ${{ needs.generate-subproject-matrix.outputs.subproject-matrix }} + sbtSubProject: [ + awsBackend + awsS3FileSystem + azureBlobFileSystem + backend + centaur + cloud-nio-impl-drs + cloud-nio-impl-ftp + cloud-nio-spi + Cloud-nio-util + cloudSupport + common + core + cromiam + cromwell-drs-localizer + cromwellApiClient + databaseMigration + databaseSql + dockerHashing + drsFileSystem + engine + ftpFileSystem + gcsFileSystem + googlePipelinesCommon + googlePipelinesV2Alpha1 + googlePipelinesV2Beta + httpFileSystem + languageFactoryCore + perf + root + server + services + sfsBackend + sraFileSystem + tesBackend + wdlBiscayneLanguageFactory + wdlDraft2LanguageFactory + wdlDraft3LanguageFactory + wdlModelBiscayne + wdlModelDraft2 + wdlModelDraft3 + wdlNewBaseTransforms + wdlSharedModel + wdlSharedTransforms + wdlTransformsBiscayne + wdlTransformsDraft2 + wdlTransformsDraft3 + wes2cromwell + wom + womtool] #This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. runs-on: ubuntu-latest From 33fe63cf072b099130032a887d40803a0568a47b Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 13:09:30 -0500 Subject: [PATCH 128/213] whoops --- .github/workflows/cromwell_unit_tests.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 7e7d684f399..f83798fa5cb 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -79,13 +79,7 @@ jobs: - uses: ./.github/set_up_cromwell_action with: cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} - - #Parse the sbt-generated list of projects into a YAML array. Note that we strip - #invisible color codes out of the `sbt projects` output. - - name: Do the thing - run: | - echo "SBT_PROJECTS=$(sbt projects | sed 's/\x1b\[[0-9;]*m//g' | sed -e 's/^\[info\]\s*\([-_A-Za-z0-9]*\)$/\1/g' | grep -v '^\[info\].*' | tr '\n', ',' | sed 's/^/[/' | sed 's/.$/]\n/')" >> $GITHUB_ENV - + #Invoke SBT on a specific subproject, asking it to run all unit tests contained within that subproject. - name: Run tests run: | From df0fbc7c5ebf50b663e1208ffe0b0b51e5db4fca Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 13:12:10 -0500 Subject: [PATCH 129/213] better formatting --- .github/workflows/cromwell_unit_tests.yml | 98 +++++++++++------------ 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index f83798fa5cb..ed704e960be 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -20,56 +20,54 @@ jobs: strategy: fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. matrix: #this array is the output of the command `sbt projects` from the cromwell root directory, minus a bunch of superflous formatting that sbt adds. - sbtSubProject: [ - awsBackend - awsS3FileSystem - azureBlobFileSystem - backend - centaur - cloud-nio-impl-drs - cloud-nio-impl-ftp - cloud-nio-spi - Cloud-nio-util - cloudSupport - common - core - cromiam - cromwell-drs-localizer - cromwellApiClient - databaseMigration - databaseSql - dockerHashing - drsFileSystem - engine - ftpFileSystem - gcsFileSystem - googlePipelinesCommon - googlePipelinesV2Alpha1 - googlePipelinesV2Beta - httpFileSystem - languageFactoryCore - perf - root - server - services - sfsBackend - sraFileSystem - tesBackend - wdlBiscayneLanguageFactory - wdlDraft2LanguageFactory - wdlDraft3LanguageFactory - wdlModelBiscayne - wdlModelDraft2 - wdlModelDraft3 - wdlNewBaseTransforms - wdlSharedModel - wdlSharedTransforms - wdlTransformsBiscayne - wdlTransformsDraft2 - wdlTransformsDraft3 - wes2cromwell - wom - womtool] + sbtSubProject: [awsBackend, + awsS3FileSystem, + azureBlobFileSystem, + backend, + centaur, + cloud-nio-impl-drs, + cloud-nio-impl-ftp, + cloud-nio-spi, + cloud-nio-util, + cloudSupport, + common, + core, + cromiam, + cromwell-drs-localizer, + cromwellApiClient, + databaseMigration, + databaseSql, + dockerHashing, + drsFileSystem, + engine, + ftpFileSystem, + gcsFileSystem, + googlePipelinesCommon, + googlePipelinesV2Alpha1, + googlePipelinesV2Beta, + httpFileSystem, + languageFactoryCore, + perf, + server, + services, + sfsBackend, + sraFileSystem, + tesBackend, + wdlBiscayneLanguageFactory, + wdlDraft2LanguageFactory, + wdlDraft3LanguageFactory, + wdlModelBiscayne, + wdlModelDraft2, + wdlModelDraft3, + wdlNewBaseTransforms, + wdlSharedModel, + wdlSharedTransforms, + wdlTransformsBiscayne, + wdlTransformsDraft2, + wdlTransformsDraft3, + wes2cromwell, + wom, + womtool] #This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. runs-on: ubuntu-latest From 05ff151b42385c9516eb5752c707813b3e3c85e1 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 13:14:52 -0500 Subject: [PATCH 130/213] commento --- .github/workflows/cromwell_unit_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index ed704e960be..a27c4051575 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -73,8 +73,8 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 # required first to checkout the repo - - uses: ./.github/set_up_cromwell_action + - uses: actions/checkout@v3 # checkout the cromwell repo + - uses: ./.github/set_up_cromwell_action #Exectute this reusable github action. It will set up java/sbt/git-secrets/cromwell. with: cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} From 4e338c40d47376127b2b32c9808ed2c12319eec7 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 13:17:21 -0500 Subject: [PATCH 131/213] eof --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index a27c4051575..07cd26ef33a 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -82,4 +82,4 @@ jobs: - name: Run tests run: | set -e - sbt "${{ matrix.sbtSubProject }}/test" \ No newline at end of file + sbt "${{ matrix.sbtSubProject }}/test" From 58e8f80ab7bf41109179b38aa8186803556af97f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 13:35:41 -0500 Subject: [PATCH 132/213] length --- .github/workflows/cromwell_unit_tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 07cd26ef33a..ff0eacd48b4 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -19,7 +19,7 @@ jobs: build-and-test: strategy: fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. - matrix: #this array is the output of the command `sbt projects` from the cromwell root directory, minus a bunch of superflous formatting that sbt adds. + matrix: #this array is the output of the command `sbt projects` from the cromwell root directory (minus lots of formatting) sbtSubProject: [awsBackend, awsS3FileSystem, azureBlobFileSystem, @@ -69,7 +69,8 @@ jobs: wom, womtool] - #This action is using a Github free runner, rather than a Broad self-hosted one. This is because the Broad ones don't have sbt installed by default. + #This action is using a Github free runner, rather than a Broad self-hosted one. + #This is because the Broad ones don't have sbt installed by default. runs-on: ubuntu-latest steps: From 13b0062f7bee0b9e4cdef7588eac8dc48c2e5ac6 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 14:41:11 -0500 Subject: [PATCH 133/213] speedtest without cache --- .github/set_up_cromwell_action/action.yml | 2 +- .github/workflows/cromwell_unit_tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml index 038aa2b0346..46a67870f4d 100644 --- a/.github/set_up_cromwell_action/action.yml +++ b/.github/set_up_cromwell_action/action.yml @@ -44,4 +44,4 @@ runs: with: distribution: temurin java-version: 11 - cache: sbt + #cache: sbt diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index ff0eacd48b4..0dbb40fc7fe 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -70,7 +70,7 @@ jobs: womtool] #This action is using a Github free runner, rather than a Broad self-hosted one. - #This is because the Broad ones don't have sbt installed by default. + This is because the Broad ones don't have sbt installed by default. runs-on: ubuntu-latest steps: From d9da2ed6631add2b1ad570a5e768e22b213a7f03 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 14:42:50 -0500 Subject: [PATCH 134/213] cacheless speedtest --- .github/set_up_cromwell_action/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml index 46a67870f4d..13cb11601db 100644 --- a/.github/set_up_cromwell_action/action.yml +++ b/.github/set_up_cromwell_action/action.yml @@ -44,4 +44,4 @@ runs: with: distribution: temurin java-version: 11 - #cache: sbt + From e00aad3f17e39958c9553293123df1fddab5af17 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 14:44:33 -0500 Subject: [PATCH 135/213] kids call these things hashtags --- .github/workflows/cromwell_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 0dbb40fc7fe..ff0eacd48b4 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -70,7 +70,7 @@ jobs: womtool] #This action is using a Github free runner, rather than a Broad self-hosted one. - This is because the Broad ones don't have sbt installed by default. + #This is because the Broad ones don't have sbt installed by default. runs-on: ubuntu-latest steps: From 83b8dafdc3faafefb6823e811211737acd435a68 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 31 Jan 2023 14:55:19 -0500 Subject: [PATCH 136/213] address PR feedback with liberal use of the delete key --- .github/workflows/cromwell_unit_tests.yml | 55 +---------------------- 1 file changed, 1 insertion(+), 54 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index ff0eacd48b4..122a40d1e1a 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -10,65 +10,12 @@ run-name: ${{ github.actor }} running Cromwell sbt unit tests. on: workflow_dispatch: #Manual trigger from GitHub UI push: - pull_request: permissions: contents: read jobs: build-and-test: - strategy: - fail-fast: false #Disabling fail fast means that even if one job in the matrix fails, the others will still try to complete. - matrix: #this array is the output of the command `sbt projects` from the cromwell root directory (minus lots of formatting) - sbtSubProject: [awsBackend, - awsS3FileSystem, - azureBlobFileSystem, - backend, - centaur, - cloud-nio-impl-drs, - cloud-nio-impl-ftp, - cloud-nio-spi, - cloud-nio-util, - cloudSupport, - common, - core, - cromiam, - cromwell-drs-localizer, - cromwellApiClient, - databaseMigration, - databaseSql, - dockerHashing, - drsFileSystem, - engine, - ftpFileSystem, - gcsFileSystem, - googlePipelinesCommon, - googlePipelinesV2Alpha1, - googlePipelinesV2Beta, - httpFileSystem, - languageFactoryCore, - perf, - server, - services, - sfsBackend, - sraFileSystem, - tesBackend, - wdlBiscayneLanguageFactory, - wdlDraft2LanguageFactory, - wdlDraft3LanguageFactory, - wdlModelBiscayne, - wdlModelDraft2, - wdlModelDraft3, - wdlNewBaseTransforms, - wdlSharedModel, - wdlSharedTransforms, - wdlTransformsBiscayne, - wdlTransformsDraft2, - wdlTransformsDraft3, - wes2cromwell, - wom, - womtool] - #This action is using a Github free runner, rather than a Broad self-hosted one. #This is because the Broad ones don't have sbt installed by default. runs-on: ubuntu-latest @@ -83,4 +30,4 @@ jobs: - name: Run tests run: | set -e - sbt "${{ matrix.sbtSubProject }}/test" + sbt "test" From 4bc0bc89597352178dfa320727ac59b68980eefc Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 1 Feb 2023 09:17:34 -0500 Subject: [PATCH 137/213] fix up some comment --- .github/workflows/cromwell_unit_tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index 122a40d1e1a..d0927f8b954 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -1,7 +1,6 @@ name: 'Cromwell unit tests' #This github action runs all of Cromwell's unit tests. -#A matrix strategy is used so that each subproject can be unit tested in parallel with all other subprojects. #This is what shows up in the github workflows page as the title. run-name: ${{ github.actor }} running Cromwell sbt unit tests. @@ -26,7 +25,7 @@ jobs: with: cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} - #Invoke SBT on a specific subproject, asking it to run all unit tests contained within that subproject. + #Invoke SBT to run all unit tests for Cromwell. - name: Run tests run: | set -e From 9d187373336c3f7e6677f6f8800edcc091409742 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 2 Feb 2023 11:18:25 -0500 Subject: [PATCH 138/213] new yaml --- .github/workflows/centaur_local_test.yml | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/centaur_local_test.yml diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml new file mode 100644 index 00000000000..ebc328234d2 --- /dev/null +++ b/.github/workflows/centaur_local_test.yml @@ -0,0 +1,34 @@ +name: 'Centaur Integration Tests' + +#This github action runs all of Cromwell's unit tests. + +#This is what shows up in the github workflows page as the title. +run-name: ${{ github.actor }} running Centaur Integration Tests. + +#What will trigger the workflow to run. +on: + workflow_dispatch: #Manual trigger from GitHub UI + push: + +permissions: + contents: read + +jobs: + build-and-test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 # checkout the cromwell repo + - uses: ./.github/set_up_cromwell_action #Exectute this reusable github action. It will set up java/sbt/git-secrets/cromwell. + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + + #Invoke SBT to build a cromwell jar + - name: Compile Cromwell + run: | + sbt assembly + + #The name of the cromwell jar depends on its version number. Find the path to the jar we'd like to test + - name: Find Cromwell Jar + run: | + echo find server/target/scala-2.13/*.jar \ No newline at end of file From 50853a2a0c5f380b3541d0b8d60e019d5e0a19f6 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 2 Feb 2023 11:31:08 -0500 Subject: [PATCH 139/213] testing --- .github/workflows/centaur_local_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index ebc328234d2..b14ad9b220f 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -29,6 +29,6 @@ jobs: sbt assembly #The name of the cromwell jar depends on its version number. Find the path to the jar we'd like to test - - name: Find Cromwell Jar + - name: Launch Cromwell jar run: | - echo find server/target/scala-2.13/*.jar \ No newline at end of file + java -jar "$(server/target/scala-2.13/*.jar -print -quit)" server \ No newline at end of file From 4103cab9988740a9a9f624a045394a3a50f046f4 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 2 Feb 2023 11:42:03 -0500 Subject: [PATCH 140/213] testing --- .github/workflows/centaur_local_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index b14ad9b220f..7578dbd9a3d 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -31,4 +31,4 @@ jobs: #The name of the cromwell jar depends on its version number. Find the path to the jar we'd like to test - name: Launch Cromwell jar run: | - java -jar "$(server/target/scala-2.13/*.jar -print -quit)" server \ No newline at end of file + java -jar "$(server/target/scala-2.13/*.jar -print -quit)" server From da3a5af3a1eb9c90826dc581e328afffeb2ba719 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 2 Feb 2023 11:43:44 -0500 Subject: [PATCH 141/213] testing --- .github/workflows/centaur_local_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 7578dbd9a3d..039dec75107 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -31,4 +31,4 @@ jobs: #The name of the cromwell jar depends on its version number. Find the path to the jar we'd like to test - name: Launch Cromwell jar run: | - java -jar "$(server/target/scala-2.13/*.jar -print -quit)" server + java -jar "$(find server/target/scala-2.13/*.jar -print -quit)" server From 048f05243da8437d2025d82b517ee49c3be37839 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 2 Feb 2023 14:51:58 -0500 Subject: [PATCH 142/213] testCentaurLocal --- .github/workflows/centaur_local_test.yml | 29 ++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 039dec75107..f2936937f2f 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -28,7 +28,32 @@ jobs: run: | sbt assembly + #Start the database that cromwell depends on + #- name: Launch Docker MySQL Database + # run: | + # ./processes/release_processes/scripts/start_publish_mysql_docker.sh + #The name of the cromwell jar depends on its version number. Find the path to the jar we'd like to test - - name: Launch Cromwell jar + - name: Run Integration Test run: | - java -jar "$(find server/target/scala-2.13/*.jar -print -quit)" server + exec "${CROMWELL_BUILD_SCRIPTS_DIRECTORY}/testCentaurLocal.sh" + env: + CROMWELL_BUILD_PROVIDER: GITHUB_ACTIONS + CROMWELL_BUILD_HOME_DIRECTORY: "$(pwd)" + CROMWELL_BUILD_ROOT_DIRECTORY: "$(pwd)" + CROMWELL_BUILD_LOG_DIRECTORY: "${CROMWELL_BUILD_ROOT_DIRECTORY}/target/ci/logs" + CROMWELL_BUILD_CROMWELL_LOG: "${CROMWELL_BUILD_LOG_DIRECTORY}/cromwell.log" + CROMWELL_BUILD_DOCKER_DIRECTORY: "${CROMWELL_BUILD_ROOT_DIRECTORY}/src/ci/docker-compose" + CROMWELL_BUILD_SCRIPTS_DIRECTORY: "${CROMWELL_BUILD_ROOT_DIRECTORY}/src/ci/bin" + CROMWELL_BUILD_RESOURCES_SOURCES: "${CROMWELL_BUILD_ROOT_DIRECTORY}/src/ci/resources" + CROMWELL_BUILD_RESOURCES_DIRECTORY: "${CROMWELL_BUILD_ROOT_DIRECTORY}/target/ci/resources" + CROMWELL_BUILD_GIT_SECRETS_DIRECTORY: "${CROMWELL_BUILD_RESOURCES_DIRECTORY}/git-secrets" + CROMWELL_BUILD_GIT_SECRETS_COMMIT: "ad82d68ee924906a0401dfd48de5057731a9bc84" + CROMWELL_BUILD_WAIT_FOR_IT_FILENAME: "wait-for-it.sh" + CROMWELL_BUILD_WAIT_FOR_IT_BRANCH: "db049716e42767d39961e95dd9696103dca813f1" + CROMWELL_BUILD_WAIT_FOR_IT_URL: "https://raw.githubusercontent.com/vishnubob/wait-for-it/${CROMWELL_BUILD_WAIT_FOR_IT_BRANCH}/${CROMWELL_BUILD_WAIT_FOR_IT_FILENAME}" + CROMWELL_BUILD_WAIT_FOR_IT_SCRIPT: "${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_BUILD_WAIT_FOR_IT_FILENAME}" + CROMWELL_BUILD_VAULT_ZIP: "${CROMWELL_BUILD_RESOURCES_DIRECTORY}/vault.zip" + CROMWELL_BUILD_VAULT_EXECUTABLE: "${CROMWELL_BUILD_RESOURCES_DIRECTORY}/vault" + CROMWELL_BUILD_EXIT_FUNCTIONS: "${CROMWELL_BUILD_RESOURCES_DIRECTORY}/cromwell_build_exit_functions.$$" + From 684455e9a391cc55ab42b72c635efa7f2974f7c6 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 2 Feb 2023 15:05:45 -0500 Subject: [PATCH 143/213] hardcodedpath --- .github/workflows/centaur_local_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index f2936937f2f..9247878302d 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -36,7 +36,7 @@ jobs: #The name of the cromwell jar depends on its version number. Find the path to the jar we'd like to test - name: Run Integration Test run: | - exec "${CROMWELL_BUILD_SCRIPTS_DIRECTORY}/testCentaurLocal.sh" + exec "./src/ci/bin/testCentaurLocal.sh" env: CROMWELL_BUILD_PROVIDER: GITHUB_ACTIONS CROMWELL_BUILD_HOME_DIRECTORY: "$(pwd)" From 07e54f6c30869b89e275fc9ec66bfd69e7c915c2 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 2 Feb 2023 15:06:25 -0500 Subject: [PATCH 144/213] testing --- .github/workflows/centaur_local_test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 9247878302d..89decca8c96 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -24,9 +24,9 @@ jobs: cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} #Invoke SBT to build a cromwell jar - - name: Compile Cromwell - run: | - sbt assembly + #- name: Compile Cromwell + # run: | + # sbt assembly #Start the database that cromwell depends on #- name: Launch Docker MySQL Database From c76cb78e0d9df816bc1e505b555a6dc92a1cde3a Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 2 Feb 2023 15:17:59 -0500 Subject: [PATCH 145/213] testing --- .github/workflows/centaur_local_test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 89decca8c96..d63d354c468 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -36,18 +36,18 @@ jobs: #The name of the cromwell jar depends on its version number. Find the path to the jar we'd like to test - name: Run Integration Test run: | - exec "./src/ci/bin/testCentaurLocal.sh" + exec "./src/ci/bin/test.sh" env: CROMWELL_BUILD_PROVIDER: GITHUB_ACTIONS - CROMWELL_BUILD_HOME_DIRECTORY: "$(pwd)" - CROMWELL_BUILD_ROOT_DIRECTORY: "$(pwd)" - CROMWELL_BUILD_LOG_DIRECTORY: "${CROMWELL_BUILD_ROOT_DIRECTORY}/target/ci/logs" + CROMWELL_BUILD_HOME_DIRECTORY: "." + CROMWELL_BUILD_ROOT_DIRECTORY: "." + CROMWELL_BUILD_LOG_DIRECTORY: "target/ci/logs" CROMWELL_BUILD_CROMWELL_LOG: "${CROMWELL_BUILD_LOG_DIRECTORY}/cromwell.log" CROMWELL_BUILD_DOCKER_DIRECTORY: "${CROMWELL_BUILD_ROOT_DIRECTORY}/src/ci/docker-compose" - CROMWELL_BUILD_SCRIPTS_DIRECTORY: "${CROMWELL_BUILD_ROOT_DIRECTORY}/src/ci/bin" - CROMWELL_BUILD_RESOURCES_SOURCES: "${CROMWELL_BUILD_ROOT_DIRECTORY}/src/ci/resources" - CROMWELL_BUILD_RESOURCES_DIRECTORY: "${CROMWELL_BUILD_ROOT_DIRECTORY}/target/ci/resources" - CROMWELL_BUILD_GIT_SECRETS_DIRECTORY: "${CROMWELL_BUILD_RESOURCES_DIRECTORY}/git-secrets" + CROMWELL_BUILD_SCRIPTS_DIRECTORY: "src/ci/bin" + CROMWELL_BUILD_RESOURCES_SOURCES: "src/ci/resources" + CROMWELL_BUILD_RESOURCES_DIRECTORY: "target/ci/resources" + CROMWELL_BUILD_GIT_SECRETS_DIRECTORY: "target/ci/resources/git-secrets" CROMWELL_BUILD_GIT_SECRETS_COMMIT: "ad82d68ee924906a0401dfd48de5057731a9bc84" CROMWELL_BUILD_WAIT_FOR_IT_FILENAME: "wait-for-it.sh" CROMWELL_BUILD_WAIT_FOR_IT_BRANCH: "db049716e42767d39961e95dd9696103dca813f1" From 1c14d290030b5eb3ad4392e74e495d4b5773b6fa Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 2 Feb 2023 15:27:24 -0500 Subject: [PATCH 146/213] testing --- .github/workflows/centaur_local_test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index d63d354c468..2d2de773b82 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -36,14 +36,16 @@ jobs: #The name of the cromwell jar depends on its version number. Find the path to the jar we'd like to test - name: Run Integration Test run: | + echo "${CROMWELL_BUILD_PROVIDER}" + echo "${CROMWELL_BUILD_HOME_DIRECTORY}" exec "./src/ci/bin/test.sh" env: CROMWELL_BUILD_PROVIDER: GITHUB_ACTIONS CROMWELL_BUILD_HOME_DIRECTORY: "." CROMWELL_BUILD_ROOT_DIRECTORY: "." CROMWELL_BUILD_LOG_DIRECTORY: "target/ci/logs" - CROMWELL_BUILD_CROMWELL_LOG: "${CROMWELL_BUILD_LOG_DIRECTORY}/cromwell.log" - CROMWELL_BUILD_DOCKER_DIRECTORY: "${CROMWELL_BUILD_ROOT_DIRECTORY}/src/ci/docker-compose" + CROMWELL_BUILD_CROMWELL_LOG: "target/ci/logs/cromwell.log" + CROMWELL_BUILD_DOCKER_DIRECTORY: "src/ci/docker-compose" CROMWELL_BUILD_SCRIPTS_DIRECTORY: "src/ci/bin" CROMWELL_BUILD_RESOURCES_SOURCES: "src/ci/resources" CROMWELL_BUILD_RESOURCES_DIRECTORY: "target/ci/resources" From cd7da8cf552d2d2dc41d6fc1071659a15eba2f54 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 2 Feb 2023 15:49:36 -0500 Subject: [PATCH 147/213] testing --- .github/workflows/centaur_local_test.yml | 52 +++++------------------- 1 file changed, 10 insertions(+), 42 deletions(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 2d2de773b82..5f1fbfd02fc 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -13,49 +13,17 @@ on: permissions: contents: read +env: + VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }} + jobs: - build-and-test: + centaur-tests: + env: + BUILD_NAME: centaurPapiUpgradePapiV2alpha1 + BUILD_TYPE: centaurPapiUpgradePapiV2alpha1 + BUILD_MYSQL: 5.7 runs-on: ubuntu-latest - + timeout-minutes: 120 steps: - uses: actions/checkout@v3 # checkout the cromwell repo - - uses: ./.github/set_up_cromwell_action #Exectute this reusable github action. It will set up java/sbt/git-secrets/cromwell. - with: - cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} - - #Invoke SBT to build a cromwell jar - #- name: Compile Cromwell - # run: | - # sbt assembly - - #Start the database that cromwell depends on - #- name: Launch Docker MySQL Database - # run: | - # ./processes/release_processes/scripts/start_publish_mysql_docker.sh - - #The name of the cromwell jar depends on its version number. Find the path to the jar we'd like to test - - name: Run Integration Test - run: | - echo "${CROMWELL_BUILD_PROVIDER}" - echo "${CROMWELL_BUILD_HOME_DIRECTORY}" - exec "./src/ci/bin/test.sh" - env: - CROMWELL_BUILD_PROVIDER: GITHUB_ACTIONS - CROMWELL_BUILD_HOME_DIRECTORY: "." - CROMWELL_BUILD_ROOT_DIRECTORY: "." - CROMWELL_BUILD_LOG_DIRECTORY: "target/ci/logs" - CROMWELL_BUILD_CROMWELL_LOG: "target/ci/logs/cromwell.log" - CROMWELL_BUILD_DOCKER_DIRECTORY: "src/ci/docker-compose" - CROMWELL_BUILD_SCRIPTS_DIRECTORY: "src/ci/bin" - CROMWELL_BUILD_RESOURCES_SOURCES: "src/ci/resources" - CROMWELL_BUILD_RESOURCES_DIRECTORY: "target/ci/resources" - CROMWELL_BUILD_GIT_SECRETS_DIRECTORY: "target/ci/resources/git-secrets" - CROMWELL_BUILD_GIT_SECRETS_COMMIT: "ad82d68ee924906a0401dfd48de5057731a9bc84" - CROMWELL_BUILD_WAIT_FOR_IT_FILENAME: "wait-for-it.sh" - CROMWELL_BUILD_WAIT_FOR_IT_BRANCH: "db049716e42767d39961e95dd9696103dca813f1" - CROMWELL_BUILD_WAIT_FOR_IT_URL: "https://raw.githubusercontent.com/vishnubob/wait-for-it/${CROMWELL_BUILD_WAIT_FOR_IT_BRANCH}/${CROMWELL_BUILD_WAIT_FOR_IT_FILENAME}" - CROMWELL_BUILD_WAIT_FOR_IT_SCRIPT: "${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_BUILD_WAIT_FOR_IT_FILENAME}" - CROMWELL_BUILD_VAULT_ZIP: "${CROMWELL_BUILD_RESOURCES_DIRECTORY}/vault.zip" - CROMWELL_BUILD_VAULT_EXECUTABLE: "${CROMWELL_BUILD_RESOURCES_DIRECTORY}/vault" - CROMWELL_BUILD_EXIT_FUNCTIONS: "${CROMWELL_BUILD_RESOURCES_DIRECTORY}/cromwell_build_exit_functions.$$" - + - run: src/ci/bin/test.sh From f0a10887041e32fe9f323ac7b2b18e45f54d21b2 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 3 Feb 2023 12:46:11 -0500 Subject: [PATCH 148/213] testing --- .github/workflows/centaur_local_test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 5f1fbfd02fc..3402ca2b3f9 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -13,9 +13,6 @@ on: permissions: contents: read -env: - VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }} - jobs: centaur-tests: env: From 9dac3d768560e5bd663ef646eaee9930131707f1 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 3 Feb 2023 12:48:28 -0500 Subject: [PATCH 149/213] testing --- .github/workflows/centaur_local_test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 3402ca2b3f9..bbffe4691a2 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -23,4 +23,6 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@v3 # checkout the cromwell repo - - run: src/ci/bin/test.sh + - run: | + echo Running test.sh + ./src/ci/bin/test.sh From 25ff7e60ebfb74179ab516254b6f1b4a82d67efa Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 3 Feb 2023 12:50:03 -0500 Subject: [PATCH 150/213] testing --- .github/workflows/centaur_local_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index bbffe4691a2..81c4977c123 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -25,4 +25,6 @@ jobs: - uses: actions/checkout@v3 # checkout the cromwell repo - run: | echo Running test.sh + echo $BUILD_NAME + echo $BUILD_TYPE ./src/ci/bin/test.sh From 2e7b8796e947cbf26ab859bd04ea2d68df75b794 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 3 Feb 2023 12:51:12 -0500 Subject: [PATCH 151/213] testing --- .github/workflows/centaur_local_test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 81c4977c123..5c97db48fe2 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v3 # checkout the cromwell repo - run: | - echo Running test.sh - echo $BUILD_NAME - echo $BUILD_TYPE - ./src/ci/bin/test.sh + echo Running test.sh + echo $BUILD_NAME + echo $BUILD_TYPE + ./src/ci/bin/test.sh From 3b071bb13330bb49d543e08093b24c6b332d7958 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 3 Feb 2023 12:53:07 -0500 Subject: [PATCH 152/213] syntax --- .github/workflows/centaur_local_test.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 5c97db48fe2..66c058b9f09 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -23,8 +23,9 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@v3 # checkout the cromwell repo - - run: | - echo Running test.sh - echo $BUILD_NAME - echo $BUILD_TYPE - ./src/ci/bin/test.sh + - name: Run Centaur Test + run: | + echo Running test.sh + echo $BUILD_NAME + echo $BUILD_TYPE + ./src/ci/bin/test.sh From b31b72ccfb5f5d121e7b2e619aee4ec3665a28b3 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 3 Feb 2023 12:55:42 -0500 Subject: [PATCH 153/213] testing --- .github/workflows/centaur_local_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 66c058b9f09..c4d06935f96 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v3 # checkout the cromwell repo - name: Run Centaur Test run: | + set -e echo Running test.sh echo $BUILD_NAME echo $BUILD_TYPE - ./src/ci/bin/test.sh From 30f3efb9bbaeddb9448e73f15bb889dffd4d8e63 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 3 Feb 2023 13:40:22 -0500 Subject: [PATCH 154/213] testing --- .github/workflows/centaur_local_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index c4d06935f96..063367700f7 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -29,3 +29,4 @@ jobs: echo Running test.sh echo $BUILD_NAME echo $BUILD_TYPE + ls From e6b095ba7e45fffcb363a68343dc1d0831999739 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 3 Feb 2023 14:08:23 -0500 Subject: [PATCH 155/213] tesitng --- .github/workflows/centaur_local_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 063367700f7..423585f6933 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -29,4 +29,4 @@ jobs: echo Running test.sh echo $BUILD_NAME echo $BUILD_TYPE - ls + ./src/ci/bin/testSingleWorkflowRunner.sh From 832adc3adb729431e3c0c6603fb898833e400708 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 3 Feb 2023 14:11:54 -0500 Subject: [PATCH 156/213] tesitng --- .github/workflows/centaur_local_test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 423585f6933..23e59ad929d 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -23,6 +23,9 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@v3 # checkout the cromwell repo + - uses: ./.github/set_up_cromwell_action #Exectute this reusable github action. It will set up java/sbt/git-secrets/cromwell. + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} - name: Run Centaur Test run: | set -e From bc12969e26f93183ca90a755a6fd5d6762a97ceb Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 6 Feb 2023 16:12:55 -0500 Subject: [PATCH 157/213] added github case --- src/ci/bin/test.inc.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 95820513dfe..34429541a70 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -83,7 +83,9 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_PROVIDER_TRAVIS="travis" CROMWELL_BUILD_PROVIDER_JENKINS="jenkins" CROMWELL_BUILD_PROVIDER_CIRCLE="circle" + CROMWELL_BUILD_PROVIDER_GITHUB="github" CROMWELL_BUILD_PROVIDER_UNKNOWN="unknown" + echo setting environment variables if [[ "${TRAVIS-false}" == "true" ]]; then CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_TRAVIS}" @@ -91,6 +93,8 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_JENKINS}" elif [[ "${CIRCLECI-false}" == "true" ]]; then CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_CIRCLE}" + elif [[ "${GITHUB_ACTIONS-false}" == "true" ]]; then + CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_GITHUB}" else CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_UNKNOWN}" fi @@ -300,6 +304,22 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_RUN_TESTS=true fi ;; + "${CROMWELL_BUILD_PROVIDER_GITHUB}") + CROMWELL_BUILD_IS_CI=true + CROMWELL_BUILD_IS_SECURE=true + echo githubSpecific + + CROMWELL_BUILD_TYPE="${BUILD_TYPE}" + CROMWELL_BUILD_BRANCH="test-build-branch-string" + CROMWELL_BUILD_EVENT="test-build-event-string" + CROMWELL_BUILD_TAG="test-build-tag-string" + CROMWELL_BUILD_NUMBER="test-build-number-string" + CROMWELL_BUILD_URL="www.testBuildURL.com" + CROMWELL_BUILD_GIT_USER_EMAIL="testGitEmail" + CROMWELL_BUILD_GIT_USER_NAME="testGitUsername" + CROMWELL_BUILD_HEARTBEAT_PATTERN="…" + CROMWELL_BUILD_GENERATE_COVERAGE=true + ;; *) CROMWELL_BUILD_IS_CI=false CROMWELL_BUILD_IS_SECURE=true From bbde442089b8965d377a5f7b2b99d64f7a23df21 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 6 Feb 2023 16:18:49 -0500 Subject: [PATCH 158/213] testing --- .github/workflows/centaur_local_test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 23e59ad929d..23980475cfd 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -16,8 +16,8 @@ permissions: jobs: centaur-tests: env: - BUILD_NAME: centaurPapiUpgradePapiV2alpha1 - BUILD_TYPE: centaurPapiUpgradePapiV2alpha1 + BUILD_NAME: singleWorkflowRunner + BUILD_TYPE: singleWorkflowRunner BUILD_MYSQL: 5.7 runs-on: ubuntu-latest timeout-minutes: 120 @@ -32,4 +32,4 @@ jobs: echo Running test.sh echo $BUILD_NAME echo $BUILD_TYPE - ./src/ci/bin/testSingleWorkflowRunner.sh + ./src/ci/bin/test.sh From 83e39b36f6739d34263696276f8c0be4519e85fd Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 6 Feb 2023 16:21:06 -0500 Subject: [PATCH 159/213] testing --- src/ci/bin/test.inc.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 34429541a70..2b135242852 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -319,6 +319,7 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_GIT_USER_NAME="testGitUsername" CROMWELL_BUILD_HEARTBEAT_PATTERN="…" CROMWELL_BUILD_GENERATE_COVERAGE=true + CROMWELL_BUILD_RUN_TESTS=true ;; *) CROMWELL_BUILD_IS_CI=false From b42b36bb94a1a8dd1fa5fb713c91494184841ae4 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 6 Feb 2023 16:26:47 -0500 Subject: [PATCH 160/213] testing --- src/ci/bin/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/bin/test.sh b/src/ci/bin/test.sh index d148b1f808c..1e203e56840 100755 --- a/src/ci/bin/test.sh +++ b/src/ci/bin/test.sh @@ -4,5 +4,5 @@ set -o errexit -o nounset -o pipefail # import in shellcheck / CI / IntelliJ compatible ways # shellcheck source=/dev/null source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh - +echo "in the root script" cromwell::build::exec_test_script From 15accf963afa963087253e08891daec2f6d5004b Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 6 Feb 2023 16:31:02 -0500 Subject: [PATCH 161/213] testing --- src/ci/bin/testSingleWorkflowRunner.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ci/bin/testSingleWorkflowRunner.sh b/src/ci/bin/testSingleWorkflowRunner.sh index fcf0f626822..7a13cc90cd6 100755 --- a/src/ci/bin/testSingleWorkflowRunner.sh +++ b/src/ci/bin/testSingleWorkflowRunner.sh @@ -5,10 +5,13 @@ set -o errexit -o nounset -o pipefail +x # shellcheck source=/dev/null source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh +echo "about to set up common environment" cromwell::build::setup_common_environment +echo "about to assemble jars" cromwell::build::assemble_jars +echo "running java command" # Test 1: basic hello world java \ -jar "${CROMWELL_BUILD_CROMWELL_JAR}" \ From 6ab676ef6d1e0a06bc8fbb1b93613dd3acbc693e Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 6 Feb 2023 16:46:47 -0500 Subject: [PATCH 162/213] testing --- src/ci/bin/test.inc.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 2b135242852..e2dee21cef6 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -85,7 +85,7 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_PROVIDER_CIRCLE="circle" CROMWELL_BUILD_PROVIDER_GITHUB="github" CROMWELL_BUILD_PROVIDER_UNKNOWN="unknown" - echo setting environment variables + echo "setting environment variables" if [[ "${TRAVIS-false}" == "true" ]]; then CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_TRAVIS}" @@ -307,7 +307,7 @@ cromwell::private::create_build_variables() { "${CROMWELL_BUILD_PROVIDER_GITHUB}") CROMWELL_BUILD_IS_CI=true CROMWELL_BUILD_IS_SECURE=true - echo githubSpecific + echo "githubSpecific" CROMWELL_BUILD_TYPE="${BUILD_TYPE}" CROMWELL_BUILD_BRANCH="test-build-branch-string" @@ -1457,6 +1457,21 @@ cromwell::build::setup_common_environment() { cromwell::private::setup_pyenv_python_latest cromwell::private::start_docker_databases ;; + "${CROMWELL_BUILD_PROVIDER_GITHUB}") + cromwell::private::stop_travis_defaults + # Try to login to vault, and if successful then use vault creds to login to docker. + # For those committers with vault access this avoids pull rate limits reported in BT-143. + cromwell::private::install_vault + cromwell::private::login_vault + cromwell::private::login_docker + cromwell::private::install_adoptopenjdk + cromwell::private::install_sbt_launcher + cromwell::private::install_docker_compose + cromwell::private::delete_boto_config + cromwell::private::delete_sbt_boot + cromwell::private::upgrade_pip + cromwell::private::start_docker_databases + ;; "${CROMWELL_BUILD_PROVIDER_JENKINS}"|\ *) ;; From 25145754771b6d75e514673cf97b3fc73bb6410b Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 7 Feb 2023 13:54:02 -0500 Subject: [PATCH 163/213] updated gitignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 571a12c5873..35e8e4f9301 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,12 @@ tags target /site +#from running tests +actual.json +console_output.txt +expected.json +run_mode_metadata.json + # custom config cromwell-executions cromwell-test-executions From adcee04ae99fdbf26fb13f8aa8a99ab8feaecf90 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 7 Feb 2023 14:02:00 -0500 Subject: [PATCH 164/213] testing --- src/ci/bin/test.inc.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index e2dee21cef6..ca747d3dc8d 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1457,7 +1457,7 @@ cromwell::build::setup_common_environment() { cromwell::private::setup_pyenv_python_latest cromwell::private::start_docker_databases ;; - "${CROMWELL_BUILD_PROVIDER_GITHUB}") + "${CROMWELL_BUILD_PROVIDER_GITHUBg}") cromwell::private::stop_travis_defaults # Try to login to vault, and if successful then use vault creds to login to docker. # For those committers with vault access this avoids pull rate limits reported in BT-143. @@ -1479,6 +1479,7 @@ cromwell::build::setup_common_environment() { cromwell::private::setup_secure_resources cromwell::private::start_build_heartbeat + echo "end of setup common environment" } From 10b87e8982156fb1e220f403abf4925cb62f49ea Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 7 Feb 2023 14:05:09 -0500 Subject: [PATCH 165/213] testing --- src/ci/bin/test.inc.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index ca747d3dc8d..7adf173b936 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -470,6 +470,7 @@ cromwell::private::create_build_variables() { export CROMWELL_BUILD_WAIT_FOR_IT_FILENAME export CROMWELL_BUILD_WAIT_FOR_IT_SCRIPT export CROMWELL_BUILD_WAIT_FOR_IT_URL + echo "end creating build varaibles" } cromwell::private::echo_build_variables() { From f2e6e9c596ae2abed1627f394c18a7a9155e56c3 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 7 Feb 2023 14:07:42 -0500 Subject: [PATCH 166/213] testing --- src/ci/bin/test.inc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 7adf173b936..c0940319d15 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1458,7 +1458,7 @@ cromwell::build::setup_common_environment() { cromwell::private::setup_pyenv_python_latest cromwell::private::start_docker_databases ;; - "${CROMWELL_BUILD_PROVIDER_GITHUBg}") + "${CROMWELL_BUILD_PROVIDER_GITHUB}") cromwell::private::stop_travis_defaults # Try to login to vault, and if successful then use vault creds to login to docker. # For those committers with vault access this avoids pull rate limits reported in BT-143. From 27601ca8b386534eb22395ffe06c3d4572405886 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 7 Feb 2023 14:24:47 -0500 Subject: [PATCH 167/213] testing --- src/ci/bin/test.inc.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index c0940319d15..8039595ab2a 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -898,6 +898,7 @@ cromwell::private::install_vault() { --output "${CROMWELL_BUILD_VAULT_ZIP}" \ "https://releases.hashicorp.com/vault/1.6.3/vault_1.6.3_${CROMWELL_BUILD_OS}_amd64.zip" unzip "${CROMWELL_BUILD_VAULT_ZIP}" -d "$(dirname "${CROMWELL_BUILD_VAULT_EXECUTABLE}")" + echo "Installed Vault" } cromwell::private::install_git_secrets() { From 2d2b45f3ec9d6f5f7cc0781b5735b5f993513009 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 7 Feb 2023 15:51:31 -0500 Subject: [PATCH 168/213] testing --- .github/workflows/centaur_local_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 23980475cfd..18d4d642561 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -19,6 +19,7 @@ jobs: BUILD_NAME: singleWorkflowRunner BUILD_TYPE: singleWorkflowRunner BUILD_MYSQL: 5.7 + VAULT_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }} runs-on: ubuntu-latest timeout-minutes: 120 steps: From a78cfb29db6bc23d9f7de746ecb4bfeca96077b4 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 7 Feb 2023 16:08:58 -0500 Subject: [PATCH 169/213] testing --- src/ci/bin/test.inc.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 8039595ab2a..da22f91c8c8 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1084,8 +1084,10 @@ cromwell::private::login_vault() { write -field=token \ auth/approle/login role_id="${VAULT_ROLE_ID}" secret_id="${VAULT_SECRET_ID}" )" + echo "Trying to login to vault" else vault_token="${VAULT_TOKEN:-}" + echo "Using Env Vault Token" fi if [[ -n "${vault_token}" ]]; then @@ -1095,6 +1097,7 @@ cromwell::private::login_vault() { login "${vault_token}" < /dev/null > /dev/null \ && echo vault login success \ || true + echo "Vault login" fi fi } From 42427ace7c1e198b5261b7b834e587c8f2409900 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 9 Feb 2023 13:57:07 -0500 Subject: [PATCH 170/213] testing --- .github/workflows/centaur_local_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 18d4d642561..daf5e97cde3 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -20,6 +20,8 @@ jobs: BUILD_TYPE: singleWorkflowRunner BUILD_MYSQL: 5.7 VAULT_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} + VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} runs-on: ubuntu-latest timeout-minutes: 120 steps: From 947d2dc45b7c99b935f77c87cee7f2254810a746 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 9 Feb 2023 14:01:07 -0500 Subject: [PATCH 171/213] testing --- .github/workflows/centaur_local_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index daf5e97cde3..4a4936945dd 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -19,7 +19,6 @@ jobs: BUILD_NAME: singleWorkflowRunner BUILD_TYPE: singleWorkflowRunner BUILD_MYSQL: 5.7 - VAULT_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }} VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} runs-on: ubuntu-latest From 540649240a2655f8b9e4c972dcdb92c5d2d1370a Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 9 Feb 2023 14:16:58 -0500 Subject: [PATCH 172/213] remove java --- .github/set_up_cromwell_action/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml index 13cb11601db..b12d3e82d94 100644 --- a/.github/set_up_cromwell_action/action.yml +++ b/.github/set_up_cromwell_action/action.yml @@ -39,9 +39,9 @@ runs: token: ${{ inputs.cromwell_repo_token }} #Install Java to this VM. This Java version and distribution is compatible with Cromwell. - - name: Setup JDK - uses: actions/setup-java@v3 - with: - distribution: temurin - java-version: 11 + #- name: Setup JDK + # uses: actions/setup-java@v3 + # with: + # distribution: temurin + # java-version: 11 From 7f53170d678a27c3561a37a0b77c1cf4c62947c0 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 9 Feb 2023 14:21:52 -0500 Subject: [PATCH 173/213] testing --- .github/set_up_cromwell_action/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml index b12d3e82d94..514e8a62db6 100644 --- a/.github/set_up_cromwell_action/action.yml +++ b/.github/set_up_cromwell_action/action.yml @@ -10,10 +10,10 @@ runs: using: "composite" # <-- this allows these steps to be used by other workflows. steps: #Allows this github action to use a cache to store stuff like Java and sbt files between runs. - - uses: actions/checkout@v3 - name: Checkout Coursier Cache - - uses: coursier/cache-action@v6 - name: Enable Coursier Cache + #- uses: actions/checkout@v3 + # name: Checkout Coursier Cache + #- uses: coursier/cache-action@v6 + # name: Enable Coursier Cache #Cromwell requires git-secrets be setup. Here, we set up secrets and verify success with a script. - name: Git secrets setup From f094e2b879f0c82c4db67171fb42e8e158448231 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 9 Feb 2023 14:31:36 -0500 Subject: [PATCH 174/213] testing --- src/ci/bin/test.inc.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index da22f91c8c8..4e9b51ce9da 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -835,8 +835,11 @@ cromwell::private::install_adoptopenjdk() { grep UBUNTU_CODENAME /etc/os-release | cut -d = -f 2 ) main" | sudo tee /etc/apt/sources.list.d/adoptopenjdk.list + echo "before update" sudo apt-get update + echo "before get install" sudo apt-get install -y adoptopenjdk-11-hotspot + echo "before alternatives" sudo update-java-alternatives --set adoptopenjdk-11-hotspot-amd64 } From 2a3c41a95f9a16c1718e121bd750779ae72d7d44 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 9 Feb 2023 14:35:03 -0500 Subject: [PATCH 175/213] testing --- src/ci/bin/test.inc.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 4e9b51ce9da..e3f1f5f2a5b 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -835,6 +835,7 @@ cromwell::private::install_adoptopenjdk() { grep UBUNTU_CODENAME /etc/os-release | cut -d = -f 2 ) main" | sudo tee /etc/apt/sources.list.d/adoptopenjdk.list + lsb_release -a echo "before update" sudo apt-get update echo "before get install" From 50b92b55a60584f3266720dceb5c953f051f13fc Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 9 Feb 2023 14:46:08 -0500 Subject: [PATCH 176/213] testing --- .github/set_up_cromwell_action/action.yml | 12 ++++++------ src/ci/bin/test.inc.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml index 514e8a62db6..d889a9c14bb 100644 --- a/.github/set_up_cromwell_action/action.yml +++ b/.github/set_up_cromwell_action/action.yml @@ -38,10 +38,10 @@ runs: repository: broadinstitute/cromwell token: ${{ inputs.cromwell_repo_token }} - #Install Java to this VM. This Java version and distribution is compatible with Cromwell. - #- name: Setup JDK - # uses: actions/setup-java@v3 - # with: - # distribution: temurin - # java-version: 11 + Install Java to this VM. This Java version and distribution is compatible with Cromwell. + - name: Setup JDK + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 11 diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index e3f1f5f2a5b..7c4e4949e4a 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1473,7 +1473,7 @@ cromwell::build::setup_common_environment() { cromwell::private::install_vault cromwell::private::login_vault cromwell::private::login_docker - cromwell::private::install_adoptopenjdk + #cromwell::private::install_adoptopenjdk cromwell::private::install_sbt_launcher cromwell::private::install_docker_compose cromwell::private::delete_boto_config From d35e7885c090d8ef146d3418f22c1774b04cf992 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 9 Feb 2023 14:48:29 -0500 Subject: [PATCH 177/213] testing --- .github/set_up_cromwell_action/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml index d889a9c14bb..13cb11601db 100644 --- a/.github/set_up_cromwell_action/action.yml +++ b/.github/set_up_cromwell_action/action.yml @@ -10,10 +10,10 @@ runs: using: "composite" # <-- this allows these steps to be used by other workflows. steps: #Allows this github action to use a cache to store stuff like Java and sbt files between runs. - #- uses: actions/checkout@v3 - # name: Checkout Coursier Cache - #- uses: coursier/cache-action@v6 - # name: Enable Coursier Cache + - uses: actions/checkout@v3 + name: Checkout Coursier Cache + - uses: coursier/cache-action@v6 + name: Enable Coursier Cache #Cromwell requires git-secrets be setup. Here, we set up secrets and verify success with a script. - name: Git secrets setup @@ -38,7 +38,7 @@ runs: repository: broadinstitute/cromwell token: ${{ inputs.cromwell_repo_token }} - Install Java to this VM. This Java version and distribution is compatible with Cromwell. + #Install Java to this VM. This Java version and distribution is compatible with Cromwell. - name: Setup JDK uses: actions/setup-java@v3 with: From c9674dcec0fd1f7f231f80d7dfcf733c3e5b3695 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 9 Feb 2023 14:51:55 -0500 Subject: [PATCH 178/213] testing --- src/ci/bin/test.inc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 7c4e4949e4a..7981e93ef5a 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1474,7 +1474,7 @@ cromwell::build::setup_common_environment() { cromwell::private::login_vault cromwell::private::login_docker #cromwell::private::install_adoptopenjdk - cromwell::private::install_sbt_launcher + #cromwell::private::install_sbt_launcher cromwell::private::install_docker_compose cromwell::private::delete_boto_config cromwell::private::delete_sbt_boot From 45d64f383331874b9a638cc1803ba42e51b91ef9 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 9 Feb 2023 15:10:16 -0500 Subject: [PATCH 179/213] testing --- .github/workflows/centaur_local_test.yml | 4 ++-- src/ci/bin/test.inc.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 4a4936945dd..295e701ba6f 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -16,8 +16,8 @@ permissions: jobs: centaur-tests: env: - BUILD_NAME: singleWorkflowRunner - BUILD_TYPE: singleWorkflowRunner + BUILD_NAME: centaurPapiV2beta + BUILD_TYPE: centaurPapiV2beta BUILD_MYSQL: 5.7 VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 7981e93ef5a..597efb39754 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1473,8 +1473,8 @@ cromwell::build::setup_common_environment() { cromwell::private::install_vault cromwell::private::login_vault cromwell::private::login_docker - #cromwell::private::install_adoptopenjdk - #cromwell::private::install_sbt_launcher + #Note: Unlike with other CI providers, we are using Github Actions to install Java and sbt for us. + #This is automatically handled in the set_up_cromwell Github Action, which can be found in [cromwell root]/.github/set_up_cromwell_aciton. cromwell::private::install_docker_compose cromwell::private::delete_boto_config cromwell::private::delete_sbt_boot From a9327ea751b8b90794e536af678d1bd66179b3c5 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 9 Feb 2023 15:54:20 -0500 Subject: [PATCH 180/213] testing --- .github/workflows/centaur_local_test.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 295e701ba6f..16f13f7c387 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -14,10 +14,14 @@ permissions: contents: read jobs: - centaur-tests: + centaur-tests: + strategy: + fail-fast: false + matrix: + build_type: [centaurPapiV2beta, testSingleWorkflowRunner, centaurLocal] env: - BUILD_NAME: centaurPapiV2beta - BUILD_TYPE: centaurPapiV2beta + BUILD_NAME: ${{ matrix.build_type }} + BUILD_TYPE: ${{ matrix.build_type }} BUILD_MYSQL: 5.7 VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} @@ -32,6 +36,4 @@ jobs: run: | set -e echo Running test.sh - echo $BUILD_NAME - echo $BUILD_TYPE ./src/ci/bin/test.sh From 7b3351c1ff81d238daa27eee78899716ce52d1e5 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 9 Feb 2023 15:57:47 -0500 Subject: [PATCH 181/213] spelling --- .github/workflows/centaur_local_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_local_test.yml index 16f13f7c387..b0af584f655 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_local_test.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - build_type: [centaurPapiV2beta, testSingleWorkflowRunner, centaurLocal] + build_type: [centaurPapiV2beta, singleWorkflowRunner, centaurLocal] env: BUILD_NAME: ${{ matrix.build_type }} BUILD_TYPE: ${{ matrix.build_type }} From 335104ab5ae3f21472c7ac79b71e3cf85a4fae04 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 13:37:03 -0500 Subject: [PATCH 182/213] testing --- ...taur_local_test.yml => centaur_integration_tests.yml} | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) rename .github/workflows/{centaur_local_test.yml => centaur_integration_tests.yml} (83%) diff --git a/.github/workflows/centaur_local_test.yml b/.github/workflows/centaur_integration_tests.yml similarity index 83% rename from .github/workflows/centaur_local_test.yml rename to .github/workflows/centaur_integration_tests.yml index b0af584f655..c0fd27c3e20 100644 --- a/.github/workflows/centaur_local_test.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -18,11 +18,16 @@ jobs: strategy: fail-fast: false matrix: - build_type: [centaurPapiV2beta, singleWorkflowRunner, centaurLocal] + include: + -build_type: centaurPapiV2Beta + build_mysql: 5.7 + -build_type: dbms + -build_type: centaurTes + build_mysql: 5.7 env: BUILD_NAME: ${{ matrix.build_type }} BUILD_TYPE: ${{ matrix.build_type }} - BUILD_MYSQL: 5.7 + BUILD_MYSQL: matrix.build_mysql VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} runs-on: ubuntu-latest From 4c6e4ee7d852fcf6060c6b58299332c1fff19df7 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 13:38:00 -0500 Subject: [PATCH 183/213] testing --- .github/workflows/centaur_integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index c0fd27c3e20..771059a5a10 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -20,10 +20,10 @@ jobs: matrix: include: -build_type: centaurPapiV2Beta - build_mysql: 5.7 + build_mysql: 5.7 -build_type: dbms -build_type: centaurTes - build_mysql: 5.7 + build_mysql: 5.7 env: BUILD_NAME: ${{ matrix.build_type }} BUILD_TYPE: ${{ matrix.build_type }} From a9f1bc7276d3b46ac0f94469442f69fa38b1926c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 13:39:51 -0500 Subject: [PATCH 184/213] testing --- .github/workflows/centaur_integration_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index 771059a5a10..6993e2d705e 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -16,7 +16,6 @@ permissions: jobs: centaur-tests: strategy: - fail-fast: false matrix: include: -build_type: centaurPapiV2Beta From 21de2ec84992700b15cc27ab2818e3f41826aa45 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 13:42:17 -0500 Subject: [PATCH 185/213] testing --- .github/workflows/centaur_integration_tests.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index 6993e2d705e..af4f0f8bca3 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -16,12 +16,13 @@ permissions: jobs: centaur-tests: strategy: + fail-fast: false matrix: include: - -build_type: centaurPapiV2Beta + - build_type: centaurPapiV2Beta build_mysql: 5.7 - -build_type: dbms - -build_type: centaurTes + - build_type: dbms + - build_type: centaurTes build_mysql: 5.7 env: BUILD_NAME: ${{ matrix.build_type }} From 283d84b4b3df708ab856781f5f1d1cc938495540 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 13:43:37 -0500 Subject: [PATCH 186/213] testing --- .github/workflows/centaur_integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index af4f0f8bca3..b23d3d0f0ad 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -20,10 +20,10 @@ jobs: matrix: include: - build_type: centaurPapiV2Beta - build_mysql: 5.7 + build_mysql: 5.7 - build_type: dbms - build_type: centaurTes - build_mysql: 5.7 + build_mysql: 5.7 env: BUILD_NAME: ${{ matrix.build_type }} BUILD_TYPE: ${{ matrix.build_type }} From 6e8a5d3b0c0a1eb33858abc881415792499d3116 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 13:46:39 -0500 Subject: [PATCH 187/213] testing --- .github/workflows/centaur_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index b23d3d0f0ad..f354cb66294 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: include: - - build_type: centaurPapiV2Beta + - build_type: centaurPapiV2beta build_mysql: 5.7 - build_type: dbms - build_type: centaurTes From e6ff2274149e50b66f9caf3888a9a5950c2a8d69 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 13:49:41 -0500 Subject: [PATCH 188/213] testing --- .github/workflows/centaur_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index f354cb66294..822c0680aa0 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -27,7 +27,7 @@ jobs: env: BUILD_NAME: ${{ matrix.build_type }} BUILD_TYPE: ${{ matrix.build_type }} - BUILD_MYSQL: matrix.build_mysql + BUILD_MYSQL: ${{ matrix.build_mysql }} VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} runs-on: ubuntu-latest From b8a6a0d890ecaad7f0e6636352e60bf7c1a609dd Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 13:52:54 -0500 Subject: [PATCH 189/213] testing --- .github/workflows/centaur_integration_tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index 822c0680aa0..7b9fe121d63 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -24,10 +24,18 @@ jobs: - build_type: dbms - build_type: centaurTes build_mysql: 5.7 + - build_type: centaurEngineUpgradeLocal + - build_type: centaurLocal + build_mysql: 5.7 + - build_type: centaurLocal + build_postgresql: 11.3 + - build_type: checkPublish + env: BUILD_NAME: ${{ matrix.build_type }} BUILD_TYPE: ${{ matrix.build_type }} BUILD_MYSQL: ${{ matrix.build_mysql }} + BUILD_POSTGRESQL: ${{ matrix.build_postgresql }} VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} runs-on: ubuntu-latest From 70e3701c09adcd5318f9ced9ad8a888f8c5d5071 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 13:58:21 -0500 Subject: [PATCH 190/213] testing --- .github/workflows/centaur_integration_tests.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index 7b9fe121d63..fedfc33bbd3 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -30,12 +30,25 @@ jobs: - build_type: centaurLocal build_postgresql: 11.3 - build_type: checkPublish - + - build_type: centaurAws + build_mysql: 5.7 + - build_type: centaurDummy + build_mysql: 5.7 + - build_type: centaurHoricromtalPapiV2beta + build_mysql: 5.7 + - build_type: centaurSlurm + build_mysql: 5.7 + - build_type: horicromtalDeadlock + - build_type: singleWorkflowRunner + - build_type: referenceDiskManifestBuilderApp + - build_type: centaurLocal + build_mariadb: 10.3 env: BUILD_NAME: ${{ matrix.build_type }} BUILD_TYPE: ${{ matrix.build_type }} BUILD_MYSQL: ${{ matrix.build_mysql }} BUILD_POSTGRESQL: ${{ matrix.build_postgresql }} + BUILD_MATRIADB: ${{ matrix.build_mariadb }} VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} runs-on: ubuntu-latest From 591c55aa13f48d5a8989b1e158f664454dfde589 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 14:16:05 -0500 Subject: [PATCH 191/213] testing --- src/ci/bin/test.inc.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 597efb39754..aefe9a2019a 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -85,7 +85,6 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_PROVIDER_CIRCLE="circle" CROMWELL_BUILD_PROVIDER_GITHUB="github" CROMWELL_BUILD_PROVIDER_UNKNOWN="unknown" - echo "setting environment variables" if [[ "${TRAVIS-false}" == "true" ]]; then CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_TRAVIS}" @@ -310,13 +309,13 @@ cromwell::private::create_build_variables() { echo "githubSpecific" CROMWELL_BUILD_TYPE="${BUILD_TYPE}" - CROMWELL_BUILD_BRANCH="test-build-branch-string" - CROMWELL_BUILD_EVENT="test-build-event-string" - CROMWELL_BUILD_TAG="test-build-tag-string" - CROMWELL_BUILD_NUMBER="test-build-number-string" - CROMWELL_BUILD_URL="www.testBuildURL.com" - CROMWELL_BUILD_GIT_USER_EMAIL="testGitEmail" - CROMWELL_BUILD_GIT_USER_NAME="testGitUsername" + CROMWELL_BUILD_BRANCH="${GITHUB_REF_NAME}" + CROMWELL_BUILD_EVENT="${GITHUB_EVENT_NAME}" + CROMWELL_BUILD_TAG="" + CROMWELL_BUILD_NUMBER="${GITHUB_RUN_ID}" + CROMWELL_BUILD_URL="${$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID}" + CROMWELL_BUILD_GIT_USER_EMAIL="" + CROMWELL_BUILD_GIT_USER_NAME="${GITHUB_ACTOR}" CROMWELL_BUILD_HEARTBEAT_PATTERN="…" CROMWELL_BUILD_GENERATE_COVERAGE=true CROMWELL_BUILD_RUN_TESTS=true From d8ba52976fcffcf35059d603e7beca81f8f1ab03 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 14:18:00 -0500 Subject: [PATCH 192/213] testing --- src/ci/bin/test.inc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index aefe9a2019a..4ccaa97bf8b 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -313,7 +313,7 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_EVENT="${GITHUB_EVENT_NAME}" CROMWELL_BUILD_TAG="" CROMWELL_BUILD_NUMBER="${GITHUB_RUN_ID}" - CROMWELL_BUILD_URL="${$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID}" + CROMWELL_BUILD_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" CROMWELL_BUILD_GIT_USER_EMAIL="" CROMWELL_BUILD_GIT_USER_NAME="${GITHUB_ACTOR}" CROMWELL_BUILD_HEARTBEAT_PATTERN="…" From eb2552240d3fd4bf7a299330381e397f8c40d8dd Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 14:25:42 -0500 Subject: [PATCH 193/213] cleanup --- src/ci/bin/test.inc.sh | 15 ++------------- src/ci/bin/test.sh | 1 - src/ci/bin/testSingleWorkflowRunner.sh | 3 --- 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 4ccaa97bf8b..19e564159fb 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -306,8 +306,6 @@ cromwell::private::create_build_variables() { "${CROMWELL_BUILD_PROVIDER_GITHUB}") CROMWELL_BUILD_IS_CI=true CROMWELL_BUILD_IS_SECURE=true - echo "githubSpecific" - CROMWELL_BUILD_TYPE="${BUILD_TYPE}" CROMWELL_BUILD_BRANCH="${GITHUB_REF_NAME}" CROMWELL_BUILD_EVENT="${GITHUB_EVENT_NAME}" @@ -469,7 +467,6 @@ cromwell::private::create_build_variables() { export CROMWELL_BUILD_WAIT_FOR_IT_FILENAME export CROMWELL_BUILD_WAIT_FOR_IT_SCRIPT export CROMWELL_BUILD_WAIT_FOR_IT_URL - echo "end creating build varaibles" } cromwell::private::echo_build_variables() { @@ -834,12 +831,8 @@ cromwell::private::install_adoptopenjdk() { grep UBUNTU_CODENAME /etc/os-release | cut -d = -f 2 ) main" | sudo tee /etc/apt/sources.list.d/adoptopenjdk.list - lsb_release -a - echo "before update" sudo apt-get update - echo "before get install" sudo apt-get install -y adoptopenjdk-11-hotspot - echo "before alternatives" sudo update-java-alternatives --set adoptopenjdk-11-hotspot-amd64 } @@ -901,7 +894,6 @@ cromwell::private::install_vault() { --output "${CROMWELL_BUILD_VAULT_ZIP}" \ "https://releases.hashicorp.com/vault/1.6.3/vault_1.6.3_${CROMWELL_BUILD_OS}_amd64.zip" unzip "${CROMWELL_BUILD_VAULT_ZIP}" -d "$(dirname "${CROMWELL_BUILD_VAULT_EXECUTABLE}")" - echo "Installed Vault" } cromwell::private::install_git_secrets() { @@ -1087,10 +1079,8 @@ cromwell::private::login_vault() { write -field=token \ auth/approle/login role_id="${VAULT_ROLE_ID}" secret_id="${VAULT_SECRET_ID}" )" - echo "Trying to login to vault" else vault_token="${VAULT_TOKEN:-}" - echo "Using Env Vault Token" fi if [[ -n "${vault_token}" ]]; then @@ -1100,7 +1090,6 @@ cromwell::private::login_vault() { login "${vault_token}" < /dev/null > /dev/null \ && echo vault login success \ || true - echo "Vault login" fi fi } @@ -1473,7 +1462,8 @@ cromwell::build::setup_common_environment() { cromwell::private::login_vault cromwell::private::login_docker #Note: Unlike with other CI providers, we are using Github Actions to install Java and sbt for us. - #This is automatically handled in the set_up_cromwell Github Action, which can be found in [cromwell root]/.github/set_up_cromwell_aciton. + #This is automatically handled in the set_up_cromwell Github Action, which can be found in + #[cromwell root]/.github/set_up_cromwell_aciton. cromwell::private::install_docker_compose cromwell::private::delete_boto_config cromwell::private::delete_sbt_boot @@ -1487,7 +1477,6 @@ cromwell::build::setup_common_environment() { cromwell::private::setup_secure_resources cromwell::private::start_build_heartbeat - echo "end of setup common environment" } diff --git a/src/ci/bin/test.sh b/src/ci/bin/test.sh index 1e203e56840..1e576430e11 100755 --- a/src/ci/bin/test.sh +++ b/src/ci/bin/test.sh @@ -4,5 +4,4 @@ set -o errexit -o nounset -o pipefail # import in shellcheck / CI / IntelliJ compatible ways # shellcheck source=/dev/null source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh -echo "in the root script" cromwell::build::exec_test_script diff --git a/src/ci/bin/testSingleWorkflowRunner.sh b/src/ci/bin/testSingleWorkflowRunner.sh index 7a13cc90cd6..fcf0f626822 100755 --- a/src/ci/bin/testSingleWorkflowRunner.sh +++ b/src/ci/bin/testSingleWorkflowRunner.sh @@ -5,13 +5,10 @@ set -o errexit -o nounset -o pipefail +x # shellcheck source=/dev/null source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh -echo "about to set up common environment" cromwell::build::setup_common_environment -echo "about to assemble jars" cromwell::build::assemble_jars -echo "running java command" # Test 1: basic hello world java \ -jar "${CROMWELL_BUILD_CROMWELL_JAR}" \ From b4ea121998712e0e1cbf33a3728066710ac446ee Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 14:45:35 -0500 Subject: [PATCH 194/213] testing --- .gitignore | 2 +- src/ci/bin/test.inc.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 35e8e4f9301..a80b4a21713 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,7 @@ tags target /site -#from running tests +#from running integration tests locally actual.json console_output.txt expected.json diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 19e564159fb..058f63cdacf 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -484,6 +484,10 @@ cromwell::private::echo_build_variables() { echo "CROMWELL_BUILD_PROVIDER='${CROMWELL_BUILD_PROVIDER}'" echo "CROMWELL_BUILD_OS='${CROMWELL_BUILD_OS}'" echo "CROMWELL_BUILD_URL='${CROMWELL_BUILD_URL}'" + echo "CROMWELL_BUILD_URL='${BUILD_POSTGRESQL}'" + echo "CROMWELL_BUILD_URL='${BUILD_MATRIADB}'" + echo "CROMWELL_BUILD_URL='${BUILD_MYSQL}'" + } # Create environment variables used by the DatabaseTestKit and cromwell::private::create_centaur_variables() From fee57ffdbfdb857dee3670223ab1fe04fe83801c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 15:07:07 -0500 Subject: [PATCH 195/213] comments --- .github/workflows/centaur_integration_tests.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index fedfc33bbd3..7271a076ea4 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -8,7 +8,7 @@ run-name: ${{ github.actor }} running Centaur Integration Tests. #What will trigger the workflow to run. on: workflow_dispatch: #Manual trigger from GitHub UI - push: + push: #git push permissions: contents: read @@ -16,7 +16,8 @@ permissions: jobs: centaur-tests: strategy: - fail-fast: false + fail-fast: false #disabling fail-fast means that even if one tests fail, the others will still try to complete. + #Each entry below is a single integration test that lives in /src/ci/bin/. matrix: include: - build_type: centaurPapiV2beta @@ -55,9 +56,11 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@v3 # checkout the cromwell repo - - uses: ./.github/set_up_cromwell_action #Exectute this reusable github action. It will set up java/sbt/git-secrets/cromwell. + - uses: ./.github/set_up_cromwell_action #This github action will set up git-secrets, caching, java, and sbt. with: cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + #This script bascially just looks up another script to exececute, assuming that the other script's filename is + #src/ci/bin/test${BUILD_TYPE}.sh - name: Run Centaur Test run: | set -e From 8bbd7d2cb36706d62c827b43d65e03222cd4c402 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 15:09:59 -0500 Subject: [PATCH 196/213] testing --- src/ci/bin/test.inc.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 058f63cdacf..65092767c84 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -484,9 +484,9 @@ cromwell::private::echo_build_variables() { echo "CROMWELL_BUILD_PROVIDER='${CROMWELL_BUILD_PROVIDER}'" echo "CROMWELL_BUILD_OS='${CROMWELL_BUILD_OS}'" echo "CROMWELL_BUILD_URL='${CROMWELL_BUILD_URL}'" - echo "CROMWELL_BUILD_URL='${BUILD_POSTGRESQL}'" - echo "CROMWELL_BUILD_URL='${BUILD_MATRIADB}'" - echo "CROMWELL_BUILD_URL='${BUILD_MYSQL}'" + echo "BUILD_POSTGRESQL='${BUILD_POSTGRESQL}'" + echo "BUILD_MATRIADB='${BUILD_MATRIADB}'" + echo "BUILD_MYSQL='${BUILD_MYSQL}'" } From a4e3c587d3b4da6ab6338e5aa5f9874a934f8d66 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 15:23:50 -0500 Subject: [PATCH 197/213] testing --- .github/workflows/centaur_integration_tests.yml | 10 +++++----- src/ci/bin/test.inc.sh | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index 7271a076ea4..cbda155eaad 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -1,9 +1,9 @@ name: 'Centaur Integration Tests' -#This github action runs all of Cromwell's unit tests. +#This github action runs all of Cromwell's integration tests. #This is what shows up in the github workflows page as the title. -run-name: ${{ github.actor }} running Centaur Integration Tests. +run-name: ${{ github.actor }} Centaur Testing. #What will trigger the workflow to run. on: @@ -16,7 +16,7 @@ permissions: jobs: centaur-tests: strategy: - fail-fast: false #disabling fail-fast means that even if one tests fail, the others will still try to complete. + fail-fast: false #disabling fail-fast means that even if one test fails, the others will still try to complete. #Each entry below is a single integration test that lives in /src/ci/bin/. matrix: include: @@ -59,8 +59,8 @@ jobs: - uses: ./.github/set_up_cromwell_action #This github action will set up git-secrets, caching, java, and sbt. with: cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} - #This script bascially just looks up another script to exececute, assuming that the other script's filename is - #src/ci/bin/test${BUILD_TYPE}.sh + #This script bascially just looks up another script to run, assuming that the other script's filename is: + #src/ci/bin/test${BUILD_TYPE}.sh. The first letter of the BUILD_TYPE is automatically capitalized when looking for the filename. - name: Run Centaur Test run: | set -e diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 65092767c84..25aa8f33729 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1459,7 +1459,6 @@ cromwell::build::setup_common_environment() { cromwell::private::start_docker_databases ;; "${CROMWELL_BUILD_PROVIDER_GITHUB}") - cromwell::private::stop_travis_defaults # Try to login to vault, and if successful then use vault creds to login to docker. # For those committers with vault access this avoids pull rate limits reported in BT-143. cromwell::private::install_vault From 6b25da0d19b2f885be9e3ece7bd71d7e7f50d60f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 15:24:21 -0500 Subject: [PATCH 198/213] whitespace --- src/ci/bin/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/bin/test.sh b/src/ci/bin/test.sh index 1e576430e11..d148b1f808c 100755 --- a/src/ci/bin/test.sh +++ b/src/ci/bin/test.sh @@ -4,4 +4,5 @@ set -o errexit -o nounset -o pipefail # import in shellcheck / CI / IntelliJ compatible ways # shellcheck source=/dev/null source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh + cromwell::build::exec_test_script From f9e712bb4b4febc5f61013a02bcb9efd2e934794 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 15:26:24 -0500 Subject: [PATCH 199/213] commento --- .github/workflows/centaur_integration_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index cbda155eaad..2d7f59416e4 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -17,7 +17,8 @@ jobs: centaur-tests: strategy: fail-fast: false #disabling fail-fast means that even if one test fails, the others will still try to complete. - #Each entry below is a single integration test that lives in /src/ci/bin/. + #Each entry below is a single integration test that lives in /src/ci/bin/. + #Each will be launched on its own runner so they can occur in parallel. matrix: include: - build_type: centaurPapiV2beta From 90b0c7b1c4ef61b0cb91c855f690a9336b5503b2 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 15:28:54 -0500 Subject: [PATCH 200/213] commento --- .github/workflows/centaur_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index 2d7f59416e4..371aa3823da 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -47,7 +47,7 @@ jobs: build_mariadb: 10.3 env: BUILD_NAME: ${{ matrix.build_type }} - BUILD_TYPE: ${{ matrix.build_type }} + BUILD_TYPE: ${{ matrix.build_type }} #intentionally duplicated variable BUILD_MYSQL: ${{ matrix.build_mysql }} BUILD_POSTGRESQL: ${{ matrix.build_postgresql }} BUILD_MATRIADB: ${{ matrix.build_mariadb }} From a3504d0fb3cbec8756530667ef1c4dc33f5d46df Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 15:31:58 -0500 Subject: [PATCH 201/213] testing --- .github/workflows/centaur_integration_tests.yml | 2 +- src/ci/bin/test.inc.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index 371aa3823da..9ce7c4c0202 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -61,7 +61,7 @@ jobs: with: cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} #This script bascially just looks up another script to run, assuming that the other script's filename is: - #src/ci/bin/test${BUILD_TYPE}.sh. The first letter of the BUILD_TYPE is automatically capitalized when looking for the filename. + #src/ci/bin/test${BUILD_TYPE}.sh. The first letter of the BUILD_TYPE is automatically capitalized when looking. - name: Run Centaur Test run: | set -e diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 25aa8f33729..64965f7655a 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -487,7 +487,6 @@ cromwell::private::echo_build_variables() { echo "BUILD_POSTGRESQL='${BUILD_POSTGRESQL}'" echo "BUILD_MATRIADB='${BUILD_MATRIADB}'" echo "BUILD_MYSQL='${BUILD_MYSQL}'" - } # Create environment variables used by the DatabaseTestKit and cromwell::private::create_centaur_variables() From d51bdf8dca3bfcbb3616adf3cb6d8b78956a123b Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Feb 2023 15:44:29 -0500 Subject: [PATCH 202/213] testing --- src/ci/bin/test.inc.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 64965f7655a..fe6d8cbbdd8 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -484,9 +484,6 @@ cromwell::private::echo_build_variables() { echo "CROMWELL_BUILD_PROVIDER='${CROMWELL_BUILD_PROVIDER}'" echo "CROMWELL_BUILD_OS='${CROMWELL_BUILD_OS}'" echo "CROMWELL_BUILD_URL='${CROMWELL_BUILD_URL}'" - echo "BUILD_POSTGRESQL='${BUILD_POSTGRESQL}'" - echo "BUILD_MATRIADB='${BUILD_MATRIADB}'" - echo "BUILD_MYSQL='${BUILD_MYSQL}'" } # Create environment variables used by the DatabaseTestKit and cromwell::private::create_centaur_variables() @@ -497,7 +494,8 @@ cromwell::private::create_database_variables() { case "${CROMWELL_BUILD_PROVIDER}" in "${CROMWELL_BUILD_PROVIDER_TRAVIS}"|\ - "${CROMWELL_BUILD_PROVIDER_CIRCLE}") + "${CROMWELL_BUILD_PROVIDER_CIRCLE}"|\ + "${CROMWELL_BUILD_PROVIDER_GITHUB}") CROMWELL_BUILD_MARIADB_HOSTNAME="localhost" CROMWELL_BUILD_MARIADB_PORT="23306" CROMWELL_BUILD_MARIADB_DOCKER_TAG="${BUILD_MARIADB-}" From 032693a8f0e94bbf50df2d73764a2fd2ca466e98 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Feb 2023 10:03:19 -0500 Subject: [PATCH 203/213] these should pass --- .github/workflows/centaur_integration_tests.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/centaur_integration_tests.yml index 9ce7c4c0202..9d1efa11bf9 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/centaur_integration_tests.yml @@ -26,11 +26,8 @@ jobs: - build_type: dbms - build_type: centaurTes build_mysql: 5.7 - - build_type: centaurEngineUpgradeLocal - build_type: centaurLocal build_mysql: 5.7 - - build_type: centaurLocal - build_postgresql: 11.3 - build_type: checkPublish - build_type: centaurAws build_mysql: 5.7 @@ -38,13 +35,8 @@ jobs: build_mysql: 5.7 - build_type: centaurHoricromtalPapiV2beta build_mysql: 5.7 - - build_type: centaurSlurm - build_mysql: 5.7 - build_type: horicromtalDeadlock - build_type: singleWorkflowRunner - - build_type: referenceDiskManifestBuilderApp - - build_type: centaurLocal - build_mariadb: 10.3 env: BUILD_NAME: ${{ matrix.build_type }} BUILD_TYPE: ${{ matrix.build_type }} #intentionally duplicated variable From 1454c3ffd53d435e1e555fcf6971caa1542ca2ce Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Feb 2023 14:18:16 -0500 Subject: [PATCH 204/213] testing PR comments --- ...ntaur_integration_tests.yml => integration_tests.yml} | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) rename .github/workflows/{centaur_integration_tests.yml => integration_tests.yml} (92%) diff --git a/.github/workflows/centaur_integration_tests.yml b/.github/workflows/integration_tests.yml similarity index 92% rename from .github/workflows/centaur_integration_tests.yml rename to .github/workflows/integration_tests.yml index 9d1efa11bf9..ee32d1c0eb2 100644 --- a/.github/workflows/centaur_integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -1,9 +1,9 @@ -name: 'Centaur Integration Tests' +name: 'Integration Tests' #This github action runs all of Cromwell's integration tests. #This is what shows up in the github workflows page as the title. -run-name: ${{ github.actor }} Centaur Testing. +run-name: ${{ github.actor }} Integration Testing. #What will trigger the workflow to run. on: @@ -14,7 +14,7 @@ permissions: contents: read jobs: - centaur-tests: + integration-tests: strategy: fail-fast: false #disabling fail-fast means that even if one test fails, the others will still try to complete. #Each entry below is a single integration test that lives in /src/ci/bin/. @@ -23,6 +23,7 @@ jobs: include: - build_type: centaurPapiV2beta build_mysql: 5.7 + name: does this name work? - build_type: dbms - build_type: centaurTes build_mysql: 5.7 @@ -54,7 +55,7 @@ jobs: cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} #This script bascially just looks up another script to run, assuming that the other script's filename is: #src/ci/bin/test${BUILD_TYPE}.sh. The first letter of the BUILD_TYPE is automatically capitalized when looking. - - name: Run Centaur Test + - name: Run Integration Test run: | set -e echo Running test.sh From 3356d8afec1c6cab2d65b69ab0550b92b0c399fa Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Feb 2023 14:21:01 -0500 Subject: [PATCH 205/213] remove unused name --- .github/workflows/integration_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index ee32d1c0eb2..c9142508370 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -23,7 +23,6 @@ jobs: include: - build_type: centaurPapiV2beta build_mysql: 5.7 - name: does this name work? - build_type: dbms - build_type: centaurTes build_mysql: 5.7 From 2da0baeac1e1e80d48ea1effc7462de3a69782f2 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Feb 2023 14:33:28 -0500 Subject: [PATCH 206/213] testing name --- .github/workflows/integration_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index c9142508370..085be4843a7 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -37,6 +37,7 @@ jobs: build_mysql: 5.7 - build_type: horicromtalDeadlock - build_type: singleWorkflowRunner + name: ${{ matrix.build_type }} env: BUILD_NAME: ${{ matrix.build_type }} BUILD_TYPE: ${{ matrix.build_type }} #intentionally duplicated variable From dfcda981df8959641978bb48791b11955898cc79 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Feb 2023 14:34:33 -0500 Subject: [PATCH 207/213] naming --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 085be4843a7..7fa445ef82e 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -37,7 +37,7 @@ jobs: build_mysql: 5.7 - build_type: horicromtalDeadlock - build_type: singleWorkflowRunner - name: ${{ matrix.build_type }} + name: ${{ matrix.build_type }} env: BUILD_NAME: ${{ matrix.build_type }} BUILD_TYPE: ${{ matrix.build_type }} #intentionally duplicated variable From 8cd79e0ccd653deb073ec731d6721b4f2d0acbb0 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Feb 2023 14:38:14 -0500 Subject: [PATCH 208/213] Friendly Names --- .github/workflows/integration_tests.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 7fa445ef82e..0ea68b6050e 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -23,21 +23,30 @@ jobs: include: - build_type: centaurPapiV2beta build_mysql: 5.7 + friendly_name: Centaur Papi V2 Beta with MySQL 5.7 - build_type: dbms - build_type: centaurTes build_mysql: 5.7 + friendly_name: Centaur Tes with MySQL 5.7 - build_type: centaurLocal build_mysql: 5.7 + friendly_name: Centaur Local with MySQL 5.7 - build_type: checkPublish + friendly_name: Check Publish - build_type: centaurAws build_mysql: 5.7 + friendly_name: Centaur AWS with MySQL 5.7 - build_type: centaurDummy build_mysql: 5.7 + friendly_name: Centaur Dummy with MySQL 5.7 - build_type: centaurHoricromtalPapiV2beta build_mysql: 5.7 + friendly_name: Centaur Horicromtal PapiV2 Beta with MySQL 5.7 - build_type: horicromtalDeadlock + friendly_name: Horicromtal Deadlock - build_type: singleWorkflowRunner - name: ${{ matrix.build_type }} + friendly_name: Single Workflow Runner + name: ${{ matrix.friendly_name }} env: BUILD_NAME: ${{ matrix.build_type }} BUILD_TYPE: ${{ matrix.build_type }} #intentionally duplicated variable From 4648e5332fbe7351e7fd5c030e72033827c19f87 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Feb 2023 14:40:16 -0500 Subject: [PATCH 209/213] Friendly Names --- .github/workflows/integration_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 0ea68b6050e..80484516e5b 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -19,12 +19,14 @@ jobs: fail-fast: false #disabling fail-fast means that even if one test fails, the others will still try to complete. #Each entry below is a single integration test that lives in /src/ci/bin/. #Each will be launched on its own runner so they can occur in parallel. + #Friendly names are displayed on the Github UI and aren't used anywhere else. matrix: include: - build_type: centaurPapiV2beta build_mysql: 5.7 friendly_name: Centaur Papi V2 Beta with MySQL 5.7 - build_type: dbms + friendly_name: DBMS - build_type: centaurTes build_mysql: 5.7 friendly_name: Centaur Tes with MySQL 5.7 From 81246a418f8587e3de27f56fee0e4c3fc6bb51d1 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Feb 2023 15:46:43 -0500 Subject: [PATCH 210/213] Miguel saved a life today --- .github/workflows/integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 80484516e5b..7964a39cf6e 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -29,7 +29,7 @@ jobs: friendly_name: DBMS - build_type: centaurTes build_mysql: 5.7 - friendly_name: Centaur Tes with MySQL 5.7 + friendly_name: Centaur TES with MySQL 5.7 - build_type: centaurLocal build_mysql: 5.7 friendly_name: Centaur Local with MySQL 5.7 @@ -54,7 +54,7 @@ jobs: BUILD_TYPE: ${{ matrix.build_type }} #intentionally duplicated variable BUILD_MYSQL: ${{ matrix.build_mysql }} BUILD_POSTGRESQL: ${{ matrix.build_postgresql }} - BUILD_MATRIADB: ${{ matrix.build_mariadb }} + BUILD_MARIADB: ${{ matrix.build_mariadb }} VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} runs-on: ubuntu-latest From dbfb293bfa8fdb4f009a640501ba0bbc0c0db52f Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 16 Feb 2023 15:02:54 -0500 Subject: [PATCH 211/213] testing --- .github/workflows/integration_tests.yml | 14 ++++++++++++++ src/ci/bin/test.inc.sh | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 7964a39cf6e..5a8db86cc64 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -48,6 +48,20 @@ jobs: friendly_name: Horicromtal Deadlock - build_type: singleWorkflowRunner friendly_name: Single Workflow Runner + - build_type: centaurLocal + build_mariadb: 10.3 + friendly_name: Centaur Local with MariaDB 10.3 + - build_type: centaurLocal + build_postgresql: 11.3 + friendly_name: Centaur Local with PostgreSQL 11.3 + - build_type: centaurEngineUpgradeLocal + build_mysql: 5.7 + friendly_name: Centaur Engine Upgrade Local with MySQL 5.7 + - build_type: centaurSlurm + build_mysql: 5.7 + friendly_name: Centaur Slurm with MySQL 5.7 + - build_type: referenceDiskManifestBuilderApp + friendly_name: Reference Disk Manifest Builder App name: ${{ matrix.friendly_name }} env: BUILD_NAME: ${{ matrix.build_type }} diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index fe6d8cbbdd8..a5d9d4fe509 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -652,7 +652,8 @@ cromwell::private::create_centaur_variables() { # Pick **one** of the databases to run Centaur against case "${CROMWELL_BUILD_PROVIDER}" in "${CROMWELL_BUILD_PROVIDER_TRAVIS}"|\ - "${CROMWELL_BUILD_PROVIDER_CIRCLE}") + "${CROMWELL_BUILD_PROVIDER_CIRCLE}"|\ + "${CROMWELL_BUILD_PROVIDER_GITHUB}") if [[ -n "${CROMWELL_BUILD_MYSQL_DOCKER_TAG:+set}" ]]; then CROMWELL_BUILD_CENTAUR_SLICK_PROFILE="slick.jdbc.MySQLProfile$" From e0ff370008fa74d7ca24bcd6b2a6dc8cd05a577c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 16 Feb 2023 16:01:20 -0500 Subject: [PATCH 212/213] back out overzealous tests --- .github/workflows/integration_tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 5a8db86cc64..8c0397245a7 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -57,11 +57,6 @@ jobs: - build_type: centaurEngineUpgradeLocal build_mysql: 5.7 friendly_name: Centaur Engine Upgrade Local with MySQL 5.7 - - build_type: centaurSlurm - build_mysql: 5.7 - friendly_name: Centaur Slurm with MySQL 5.7 - - build_type: referenceDiskManifestBuilderApp - friendly_name: Reference Disk Manifest Builder App name: ${{ matrix.friendly_name }} env: BUILD_NAME: ${{ matrix.build_type }} From f227458cadfdf2aa1e70592df8bd3c8640de0be1 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 17 Feb 2023 16:02:37 -0500 Subject: [PATCH 213/213] make tests trigger on merge queue --- .github/workflows/cromwell_unit_tests.yml | 1 + .github/workflows/integration_tests.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml index d0927f8b954..797f38efd96 100644 --- a/.github/workflows/cromwell_unit_tests.yml +++ b/.github/workflows/cromwell_unit_tests.yml @@ -9,6 +9,7 @@ run-name: ${{ github.actor }} running Cromwell sbt unit tests. on: workflow_dispatch: #Manual trigger from GitHub UI push: + merge_group: permissions: contents: read diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 8c0397245a7..489e9e731a9 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -9,6 +9,7 @@ run-name: ${{ github.actor }} Integration Testing. on: workflow_dispatch: #Manual trigger from GitHub UI push: #git push + merge_group: permissions: contents: read