WX-905

.github/workflows/cromwell_unit_tests.yml

@@ -9,6 +9,7 @@ run-name: ${{ github.actor }} running Cromwell sbt unit tests.
 on:
   workflow_dispatch: #Manual trigger from GitHub UI
   push:
+  merge_group:
 
 permissions: 
   contents: read

.github/workflows/integration_tests.yml

@@ -0,0 +1,83 @@
+name: 'Integration Tests'
+
+#This github action runs all of Cromwell's integration tests.
+
+#This is what shows up in the github workflows page as the title. 
+run-name: ${{ github.actor }} Integration Testing.
+
+#What will trigger the workflow to run. 
+on:
+  workflow_dispatch: #Manual trigger from GitHub UI
+  push: #git push
+  merge_group:
+
+permissions: 
+  contents: read
+
+jobs:
+  integration-tests:
+    strategy:
+      fail-fast: false #disabling fail-fast means that even if one test fails, the others will still try to complete.
+      #Each entry below is a single integration test that lives in /src/ci/bin/. 
+      #Each will be launched on its own runner so they can occur in parallel.
+      #Friendly names are displayed on the Github UI and aren't used anywhere else.
+      matrix:
+        include:
+          - build_type: centaurPapiV2beta
+            build_mysql: 5.7
+            friendly_name: Centaur Papi V2 Beta with MySQL 5.7
+          - build_type: dbms
+            friendly_name: DBMS
+          - build_type: centaurTes
+            build_mysql: 5.7
+            friendly_name: Centaur TES with MySQL 5.7
+          - build_type: centaurLocal
+            build_mysql: 5.7
+            friendly_name: Centaur Local with MySQL 5.7
+          - build_type: checkPublish
+            friendly_name: Check Publish
+          - build_type: centaurAws
+            build_mysql: 5.7
+            friendly_name: Centaur AWS with MySQL 5.7
+          - build_type: centaurDummy
+            build_mysql: 5.7
+            friendly_name: Centaur Dummy with MySQL 5.7
+          - build_type: centaurHoricromtalPapiV2beta
+            build_mysql: 5.7
+            friendly_name: Centaur Horicromtal PapiV2 Beta with MySQL 5.7
+          - build_type: horicromtalDeadlock
+            friendly_name: Horicromtal Deadlock
+          - build_type: singleWorkflowRunner
+            friendly_name: Single Workflow Runner
+          - build_type: centaurLocal
+            build_mariadb: 10.3
+            friendly_name: Centaur Local with MariaDB 10.3
+          - build_type: centaurLocal
+            build_postgresql: 11.3
+            friendly_name: Centaur Local with PostgreSQL 11.3
+          - build_type: centaurEngineUpgradeLocal
+            build_mysql: 5.7
+            friendly_name: Centaur Engine Upgrade Local with MySQL 5.7
+    name: ${{ matrix.friendly_name }}
+    env:
+      BUILD_NAME: ${{ matrix.build_type }}
+      BUILD_TYPE: ${{ matrix.build_type }} #intentionally duplicated variable
+      BUILD_MYSQL: ${{ matrix.build_mysql }}
+      BUILD_POSTGRESQL: ${{ matrix.build_postgresql }}
+      BUILD_MARIADB: ${{ matrix.build_mariadb }}
+      VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }}
+      VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+    - uses: actions/checkout@v3 # checkout the cromwell repo
+    - uses: ./.github/set_up_cromwell_action #This github action will set up git-secrets, caching, java, and sbt.
+      with:
+        cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+    #This script bascially just looks up another script to run, assuming that the other script's filename is:
+    #src/ci/bin/test${BUILD_TYPE}.sh. The first letter of the BUILD_TYPE is automatically capitalized when looking.
+    - name: Run Integration Test
+      run: |
+        set -e
+        echo Running test.sh
+        ./src/ci/bin/test.sh

.gitignore

@@ -17,6 +17,12 @@ tags
 target
 /site
 
+#from running integration tests locally
+actual.json
+console_output.txt
+expected.json
+run_mode_metadata.json
+
 # custom config
 cromwell-executions
 cromwell-test-executions

src/ci/bin/test.inc.sh

@@ -83,6 +83,7 @@ cromwell::private::create_build_variables() {
     CROMWELL_BUILD_PROVIDER_TRAVIS="travis"
     CROMWELL_BUILD_PROVIDER_JENKINS="jenkins"
     CROMWELL_BUILD_PROVIDER_CIRCLE="circle"
+    CROMWELL_BUILD_PROVIDER_GITHUB="github"
     CROMWELL_BUILD_PROVIDER_UNKNOWN="unknown"
 
     if [[ "${TRAVIS-false}" == "true" ]]; then
@@ -91,6 +92,8 @@ cromwell::private::create_build_variables() {
         CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_JENKINS}"
     elif [[ "${CIRCLECI-false}" == "true" ]]; then
         CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_CIRCLE}"
+    elif [[ "${GITHUB_ACTIONS-false}" == "true" ]]; then
+        CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_GITHUB}"
     else
         CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_UNKNOWN}"
     fi
@@ -300,6 +303,21 @@ cromwell::private::create_build_variables() {
                 CROMWELL_BUILD_RUN_TESTS=true
             fi
             ;;
+        "${CROMWELL_BUILD_PROVIDER_GITHUB}")
+            CROMWELL_BUILD_IS_CI=true
+            CROMWELL_BUILD_IS_SECURE=true
+            CROMWELL_BUILD_TYPE="${BUILD_TYPE}"
+            CROMWELL_BUILD_BRANCH="${GITHUB_REF_NAME}"
+            CROMWELL_BUILD_EVENT="${GITHUB_EVENT_NAME}"
+            CROMWELL_BUILD_TAG=""
+            CROMWELL_BUILD_NUMBER="${GITHUB_RUN_ID}"
+            CROMWELL_BUILD_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+            CROMWELL_BUILD_GIT_USER_EMAIL=""
+            CROMWELL_BUILD_GIT_USER_NAME="${GITHUB_ACTOR}"
+            CROMWELL_BUILD_HEARTBEAT_PATTERN="…"
+            CROMWELL_BUILD_GENERATE_COVERAGE=true
+            CROMWELL_BUILD_RUN_TESTS=true
+            ;;
         *)
             CROMWELL_BUILD_IS_CI=false
             CROMWELL_BUILD_IS_SECURE=true
@@ -476,7 +494,8 @@ cromwell::private::create_database_variables() {
 
     case "${CROMWELL_BUILD_PROVIDER}" in
         "${CROMWELL_BUILD_PROVIDER_TRAVIS}"|\
-        "${CROMWELL_BUILD_PROVIDER_CIRCLE}")
+        "${CROMWELL_BUILD_PROVIDER_CIRCLE}"|\
+        "${CROMWELL_BUILD_PROVIDER_GITHUB}")
             CROMWELL_BUILD_MARIADB_HOSTNAME="localhost"
             CROMWELL_BUILD_MARIADB_PORT="23306"
             CROMWELL_BUILD_MARIADB_DOCKER_TAG="${BUILD_MARIADB-}"
@@ -633,7 +652,8 @@ cromwell::private::create_centaur_variables() {
     # Pick **one** of the databases to run Centaur against
     case "${CROMWELL_BUILD_PROVIDER}" in
         "${CROMWELL_BUILD_PROVIDER_TRAVIS}"|\
-        "${CROMWELL_BUILD_PROVIDER_CIRCLE}")
+        "${CROMWELL_BUILD_PROVIDER_CIRCLE}"|\
+        "${CROMWELL_BUILD_PROVIDER_GITHUB}")
 
             if [[ -n "${CROMWELL_BUILD_MYSQL_DOCKER_TAG:+set}" ]]; then
                 CROMWELL_BUILD_CENTAUR_SLICK_PROFILE="slick.jdbc.MySQLProfile$"
@@ -1436,6 +1456,21 @@ cromwell::build::setup_common_environment() {
             cromwell::private::setup_pyenv_python_latest
             cromwell::private::start_docker_databases
             ;;
+        "${CROMWELL_BUILD_PROVIDER_GITHUB}")
+            # Try to login to vault, and if successful then use vault creds to login to docker.
+            # For those committers with vault access this avoids pull rate limits reported in BT-143.
+            cromwell::private::install_vault
+            cromwell::private::login_vault
+            cromwell::private::login_docker
+            #Note: Unlike with other CI providers, we are using Github Actions to install Java and sbt for us.
+            #This is automatically handled in the set_up_cromwell Github Action, which can be found in
+            #[cromwell root]/.github/set_up_cromwell_aciton.
+            cromwell::private::install_docker_compose
+            cromwell::private::delete_boto_config
+            cromwell::private::delete_sbt_boot
+            cromwell::private::upgrade_pip
+            cromwell::private::start_docker_databases
+            ;;
         "${CROMWELL_BUILD_PROVIDER_JENKINS}"|\
         *)
             ;;