Extract Cohort optimizations [VS-493] [VS-1516] #4935
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test GATK 4 | |
| on: | |
| push: | |
| branches: | |
| - 'master' | |
| pull_request: | |
| workflow_dispatch: | |
| env: | |
| CROMWELL_VERSION: 84 | |
| TERM: dumb | |
| GRADLE_OPTS: "-Xmx2048m -Dorg.gradle.daemon=false" | |
| HELLBENDER_TEST_INPUTS: gs://hellbender/test/resources/ | |
| HELLBENDER_TEST_STAGING: gs://hellbender-test-logs/staging/ | |
| HELLBENDER_TEST_LOGS: /hellbender-test-logs/build_reports/ | |
| HELLBENDER_TEST_PROJECT: broad-dsde-dev | |
| TEST_VERBOSITY: minimal | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| jobs: | |
| ## This workaround is necessary since there is no equivalent to the old TRAVIS_SECURE_ENVIRONMENT variable that indicated | |
| ## if a run was privileged and had secrets. Since the GCP credentials are necessary for all tests in order to upload their, | |
| ## results that makes them a reasonable proxy for testing the credentials of this entire execution. https://github.com/actions/runner/issues/520 | |
| check-secrets: | |
| name: check if the environment has privileges | |
| outputs: | |
| google-credentials: ${{ steps.google-credentials.outputs.defined }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: google-credentials | |
| env: | |
| GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }} | |
| if: "${{ env.GCP_CREDENTIALS != '' }}" | |
| run: echo defined=true >> $GITHUB_OUTPUT | |
| buildDocker: | |
| name: build and push gatk docker to artifacts | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout gatk | |
| uses: actions/checkout@v3 | |
| - uses: ./.github/actions/purge-runner-disk | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| tags: | | |
| type=raw,value=${{github.sha}} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| outputs: type=docker,dest=/tmp/myimage.tar | |
| # By uploading the docker image as an artifact we save ourselves the requirement for the image to be built with ghcr push permission | |
| - name: Upload docker image as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: gatkDockerImage | |
| path: /tmp/myimage.tar | |
| #Run our non-docker tests | |
| test: | |
| runs-on: ubuntu-latest | |
| needs: check-secrets | |
| strategy: | |
| matrix: | |
| java: [ 17.0.6+10 ] | |
| experimental: [ false ] | |
| scalaVersion: [ 2.13 ] | |
| testType: [ cloud, integration, unit ] | |
| fail-fast: false | |
| continue-on-error: ${{ matrix.experimental }} | |
| env: | |
| TEST_TYPE: ${{ matrix.testType }} | |
| SCALA_VERSION: ${{ matrix.scalaVersion }} | |
| name: Java ${{ matrix.Java }} build and test ${{ matrix.testType }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/purge-runner-disk | |
| - name: 'Set up java ${{ matrix.Java }}' | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.Java }} | |
| distribution: 'temurin' | |
| cache: gradle | |
| - name: 'Compile with Gradle' | |
| run: | | |
| ./gradlew compileJava ; ./gradlew installDist | |
| #Google Cloud stuff | |
| - id: 'gcloud-auth' | |
| if: needs.check-secrets.outputs.google-credentials == 'true' | |
| uses: google-github-actions/auth@v0 | |
| with: | |
| credentials_json: ${{ secrets.GCP_CREDENTIALS }} | |
| project_id: ${{ env.HELLBENDER_TEST_PROJECT }} | |
| create_credentials_file: true | |
| - name: "export the credentials for GATK tests" | |
| if: needs.check-secrets.outputs.google-credentials == 'true' | |
| run: echo "HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV | |
| - name: 'Set up Cloud SDK' | |
| if: needs.check-secrets.outputs.google-credentials == 'true' | |
| uses: google-github-actions/setup-gcloud@v0 | |
| - name: pull lfs files | |
| run: git lfs pull | |
| - name: compile test code | |
| run: ./gradlew compileTestJava | |
| - name: run-tests | |
| if: ${{ needs.check-secrets.outputs.google-credentials == 'true' || matrix.testType != 'cloud'}} | |
| id: jacoco-tests | |
| run: | | |
| ./gradlew --daemon -Dscala.version=${{ env.SCALA_VERSION }} jacocoTestReport | |
| - uses: ./.github/actions/upload-gatk-test-results | |
| if: always() | |
| with: | |
| warnPR: ${{ github.event_name == 'pull_request' && steps.jacoco-tests.outcome != 'success' }} | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| job-matrix-id: ${{ github.run_id }}.1${{ strategy.job-index }} | |
| repo-path: ${{ github.ref_name }}_${{ github.run_id }}.1${{ strategy.job-index }} | |
| bot-comment-key: ${{ secrets.GATK_BOT_COMMENT_KEY }} | |
| identifier: Java ${{ matrix.Java }} build and test ${{ matrix.testType }} | |
| only-artifact: ${{ needs.check-secrets.outputs.google-credentials != 'true' }} | |
| #Run our docker tests | |
| testOnDocker: | |
| needs: [buildDocker, check-secrets] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| java: [ 17.0.6+10 ] | |
| experimental: [ false ] | |
| scalaVersion: [ 2.13 ] | |
| testType: [ integration, unit, variantcalling, conda ] | |
| fail-fast: false | |
| continue-on-error: ${{ matrix.experimental }} | |
| env: | |
| TEST_TYPE: ${{ matrix.testType }} | |
| SCALA_VERSION: ${{ matrix.scalaVersion }} | |
| name: Java ${{ matrix.Java }} test on docker ${{matrix.testType}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 #TODO make this shallow | |
| - uses: ./.github/actions/purge-runner-disk | |
| - name: 'Set up java ${{ matrix.Java }}' | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.Java }} | |
| distribution: 'temurin' | |
| cache: gradle | |
| #Google Cloud stuff | |
| - id: 'gcloud-auth' | |
| uses: google-github-actions/auth@v0 | |
| if: needs.check-secrets.outputs.google-credentials == 'true' | |
| with: | |
| credentials_json: ${{ secrets.GCP_CREDENTIALS }} | |
| project_id: ${{ env.HELLBENDER_TEST_PROJECT }} | |
| create_credentials_file: true | |
| - name: "export the credentials for GATK tests" | |
| if: needs.check-secrets.outputs.google-credentials == 'true' | |
| run: echo "HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV | |
| - name: 'Set up Cloud SDK' | |
| if: needs.check-secrets.outputs.google-credentials == 'true' | |
| uses: google-github-actions/setup-gcloud@v0 | |
| - name: build test jars | |
| run: ./gradlew clean shadowTestClassJar shadowTestJar | |
| - name: set up test folders and links | |
| run: | | |
| mkdir ./testJars; | |
| echo $( find ./build/libs/ -name "gatk*test.jar"); | |
| cp -r $( find ./build/libs/ -name "gatk*test.jar" ) ./testJars; | |
| cp -r $( find ./build/libs/ -name "gatk*testDependencies.jar" ) ./testJars; | |
| mkdir -p build/reports/; | |
| chmod -R a+w build/reports/; | |
| cp scripts/docker/dockertest.gradle . | |
| - name: Download docker image artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: gatkDockerImage | |
| path: /tmp | |
| - name: Load Docker image | |
| run: | | |
| docker load --input /tmp/myimage.tar | |
| docker image ls -a | |
| - name: pull lfs files | |
| run: git lfs pull | |
| - name: run-docker-tests | |
| id: docker-jacocco-tests | |
| run: | | |
| docker run \ | |
| -v $(pwd):/gatkCloneMountPoint:cached \ | |
| -v $(pwd)/testJars:/jars:cached \ | |
| --rm \ | |
| -e "scala.version=${SCALA_VERSION}" \ | |
| -e "TEST_VERBOSITY=minimal" \ | |
| -e "TEST_TYPE=${TEST_TYPE}" \ | |
| -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} \ | |
| bash --init-file /gatk/gatkenv.rc /root/run_unit_tests.sh; | |
| TEST_EXIT_VALUE=$?; | |
| $( exit ${TEST_EXIT_VALUE} ); | |
| sudo chmod -R a+w build/reports/; | |
| mkdir build/reports/tests/test \ | |
| && cp -rp build/reports/tests/testOnPackagedReleaseJar/* build/reports/tests/test \ | |
| && rm -r build/reports/tests/testOnPackagedReleaseJar; | |
| - uses: ./.github/actions/upload-gatk-test-results | |
| if: always() | |
| with: | |
| warnPR: ${{ github.event_name == 'pull_request' && steps.docker-jacocco-tests.outcome != 'success' }} | |
| is-docker: true | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| job-matrix-id: ${{ github.run_id }}.${{ strategy.job-index }} | |
| repo-path: ${{ github.ref_name }}_${{ github.run_id }}.${{ strategy.job-index }} | |
| bot-comment-key: ${{ secrets.GATK_BOT_COMMENT_KEY }} | |
| identifier: Java ${{ matrix.Java }} test on docker ${{matrix.testType}} | |
| only-artifact: ${{ needs.check-secrets.outputs.google-credentials != 'true' }} | |
| #run wdl validation on WDLs in the scripts directory | |
| scriptsWdlValidation: | |
| runs-on: ubuntu-latest | |
| name: Validate script WDLs using womtools | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/purge-runner-disk | |
| - name: Set up java 17 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| cache: gradle | |
| - uses: ./.github/actions/install-cromwell | |
| with: | |
| CROMWELL_VERSION: ${{ env.CROMWELL_VERSION }} | |
| - name: Run Scripts WDL Validation Test | |
| run: ./gradlew gatkValidateScriptsWdl | |
| #run wdl validation on generated WDLs | |
| generatedWdlValidation: | |
| runs-on: ubuntu-latest | |
| name: Validate generated WDLs using womtools | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/purge-runner-disk | |
| - name: Set up java 17 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| cache: gradle | |
| - uses: ./.github/actions/install-cromwell | |
| with: | |
| CROMWELL_VERSION: ${{ env.CROMWELL_VERSION }} | |
| - name: Run Generated WDL Validation Test | |
| run: ./gradlew gatkValidateGeneratedWdl | |
| #Run our various targeted medium scale wdl wiring tests | |
| wdlTests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| wdlTest: [ 'RUN_CNV_GERMLINE_COHORT_WDL', 'RUN_CNV_GERMLINE_CASE_WDL', 'RUN_CNV_SOMATIC_WDL', 'RUN_M2_WDL', 'RUN_CNN_WDL', 'RUN_VCF_SITE_LEVEL_FILTERING_WDL' ] | |
| continue-on-error: true | |
| name: WDL test ${{ matrix.wdlTest }} on cromwell | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/purge-runner-disk | |
| - name: pull lfs files | |
| run: git lfs pull | |
| - name: Set up java 17 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| cache: gradle | |
| - uses: ./.github/actions/install-cromwell | |
| with: | |
| CROMWELL_VERSION: ${{ env.CROMWELL_VERSION }} | |
| - name: "Exporting Environment Variables for Wdl Scripts" | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]] | |
| then | |
| echo "CI_PULL_REQUEST=${{ github.event.number }}" >> $GITHUB_ENV | |
| fi | |
| echo "CI_COMMIT=${{ github.sha }}" >> $GITHUB_ENV | |
| - name: "CNV_GERMLINE_COHORT_WDL_TEST" | |
| if: ${{ matrix.wdlTest == 'RUN_CNV_GERMLINE_COHORT_WDL' }} | |
| run: | | |
| echo "Running CNV germline cohort workflow"; | |
| bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh COHORT; | |
| - name: "CNV_GERMLINE_CASE_WDL_TEST" | |
| if: ${{ matrix.wdlTest == 'RUN_CNV_GERMLINE_CASE_WDL' }} | |
| run: | | |
| echo "Running CNV germline case workflow"; | |
| bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh CASE; | |
| - name: "CNV_SOMATIC_WDL_TEST" | |
| if: ${{ matrix.wdlTest == 'RUN_CNV_SOMATIC_WDL' }} | |
| run: | | |
| echo "Running CNV somatic workflows"; | |
| bash scripts/cnv_cromwell_tests/somatic/run_cnv_somatic_workflows.sh; | |
| - name: "M2_WDL_TEST" | |
| if: ${{ matrix.wdlTest == 'RUN_M2_WDL' }} | |
| run: | | |
| echo "Deleting some unused files before running M2 WDL..."; | |
| rm -Rf src/test/resources/large/VQSR; | |
| echo "Running M2 WDL"; | |
| bash scripts/m2_cromwell_tests/run_m2_wdl.sh; | |
| - name: "CNN_WDL_TEST" | |
| if: ${{ matrix.wdlTest == 'RUN_CNN_WDL' }} | |
| run: | | |
| echo "Running CNN WDL"; | |
| bash scripts/cnn_variant_cromwell_tests/run_cnn_variant_wdl.sh; | |
| - name: "VCF_SITE_LEVEL_FILTERING_WDL_TEST" | |
| if: ${{ matrix.wdlTest == 'RUN_VCF_SITE_LEVEL_FILTERING_WDL' }} | |
| run: | | |
| echo "Running VCF Site Level Filtering WDL"; | |
| bash scripts/vcf_site_level_filtering_cromwell_tests/run_vcf_site_level_filtering_wdl.sh; |