-
Notifications
You must be signed in to change notification settings - Fork 594
362 lines (320 loc) · 12.9 KB
/
gatk-tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
name: Build and Test GATK 4
on:
push:
branches:
- 'master'
pull_request:
workflow_dispatch:
env:
CROMWELL_VERSION: 84
TERM: dumb
GRADLE_OPTS: "-Xmx2048m -Dorg.gradle.daemon=false"
HELLBENDER_TEST_INPUTS: gs://hellbender/test/resources/
HELLBENDER_TEST_STAGING: gs://hellbender-test-logs/staging/
HELLBENDER_TEST_LOGS: /hellbender-test-logs/build_reports/
HELLBENDER_TEST_PROJECT: broad-dsde-dev
TEST_VERBOSITY: minimal
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
jobs:
## This workaround is necessary since there is no equivalent to the old TRAVIS_SECURE_ENVIRONMENT variable that indicated
## if a run was privileged and had secrets. Since the GCP credentials are necessary for all tests in order to upload their,
## results that makes them a reasonable proxy for testing the credentials of this entire execution. https://github.com/actions/runner/issues/520
check-secrets:
name: check if the environment has privileges
outputs:
google-credentials: ${{ steps.google-credentials.outputs.defined }}
runs-on: ubuntu-latest
steps:
- id: google-credentials
env:
GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }}
if: "${{ env.GCP_CREDENTIALS != '' }}"
run: echo defined=true >> $GITHUB_OUTPUT
buildDocker:
name: build and push gatk docker to artifacts
runs-on: ubuntu-latest
steps:
- name: checkout gatk
uses: actions/checkout@v3
- uses: ./.github/actions/purge-runner-disk
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v4
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=raw,value=${{github.sha}}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build and push Docker image
uses: docker/build-push-action@v4
with:
context: .
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
outputs: type=docker,dest=/tmp/myimage.tar
# By uploading the docker image as an artifact we save ourselves the requirement for the image to be built with ghcr push permission
- name: Upload docker image as artifact
uses: actions/upload-artifact@v3
with:
name: gatkDockerImage
path: /tmp/myimage.tar
#Run our non-docker tests
test:
runs-on: ubuntu-latest
needs: check-secrets
strategy:
matrix:
java: [ 17.0.6+10 ]
experimental: [ false ]
scalaVersion: [ 2.13 ]
testType: [ cloud, integration, unit ]
fail-fast: false
continue-on-error: ${{ matrix.experimental }}
env:
TEST_TYPE: ${{ matrix.testType }}
SCALA_VERSION: ${{ matrix.scalaVersion }}
name: Java ${{ matrix.Java }} build and test ${{ matrix.testType }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: ./.github/actions/purge-runner-disk
- name: 'Set up java ${{ matrix.Java }}'
uses: actions/setup-java@v3
with:
java-version: ${{ matrix.Java }}
distribution: 'temurin'
cache: gradle
- name: 'Compile with Gradle'
run: |
./gradlew compileJava ; ./gradlew installDist
#Google Cloud stuff
- id: 'gcloud-auth'
if: needs.check-secrets.outputs.google-credentials == 'true'
uses: google-github-actions/auth@v0
with:
credentials_json: ${{ secrets.GCP_CREDENTIALS }}
project_id: ${{ env.HELLBENDER_TEST_PROJECT }}
create_credentials_file: true
- name: "export the credentials for GATK tests"
if: needs.check-secrets.outputs.google-credentials == 'true'
run: echo "HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV
- name: 'Set up Cloud SDK'
if: needs.check-secrets.outputs.google-credentials == 'true'
uses: google-github-actions/setup-gcloud@v0
- name: pull lfs files
run: git lfs pull
- name: compile test code
run: ./gradlew compileTestJava
- name: run-tests
if: ${{ needs.check-secrets.outputs.google-credentials == 'true' || matrix.testType != 'cloud'}}
id: jacoco-tests
run: |
./gradlew --daemon -Dscala.version=${{ env.SCALA_VERSION }} jacocoTestReport
- uses: ./.github/actions/upload-gatk-test-results
if: always()
with:
warnPR: ${{ github.event_name == 'pull_request' && steps.jacoco-tests.outcome != 'success' }}
repo-token: ${{ secrets.GITHUB_TOKEN }}
job-matrix-id: ${{ github.run_id }}.1${{ strategy.job-index }}
repo-path: ${{ github.ref_name }}_${{ github.run_id }}.1${{ strategy.job-index }}
bot-comment-key: ${{ secrets.GATK_BOT_COMMENT_KEY }}
identifier: Java ${{ matrix.Java }} build and test ${{ matrix.testType }}
only-artifact: ${{ needs.check-secrets.outputs.google-credentials != 'true' }}
#Run our docker tests
testOnDocker:
needs: [buildDocker, check-secrets]
runs-on: ubuntu-latest
strategy:
matrix:
java: [ 17.0.6+10 ]
experimental: [ false ]
scalaVersion: [ 2.13 ]
testType: [ integration, unit, variantcalling, conda ]
fail-fast: false
continue-on-error: ${{ matrix.experimental }}
env:
TEST_TYPE: ${{ matrix.testType }}
SCALA_VERSION: ${{ matrix.scalaVersion }}
name: Java ${{ matrix.Java }} test on docker ${{matrix.testType}}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0 #TODO make this shallow
- uses: ./.github/actions/purge-runner-disk
- name: 'Set up java ${{ matrix.Java }}'
uses: actions/setup-java@v3
with:
java-version: ${{ matrix.Java }}
distribution: 'temurin'
cache: gradle
#Google Cloud stuff
- id: 'gcloud-auth'
uses: google-github-actions/auth@v0
if: needs.check-secrets.outputs.google-credentials == 'true'
with:
credentials_json: ${{ secrets.GCP_CREDENTIALS }}
project_id: ${{ env.HELLBENDER_TEST_PROJECT }}
create_credentials_file: true
- name: "export the credentials for GATK tests"
if: needs.check-secrets.outputs.google-credentials == 'true'
run: echo "HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV
- name: 'Set up Cloud SDK'
if: needs.check-secrets.outputs.google-credentials == 'true'
uses: google-github-actions/setup-gcloud@v0
- name: build test jars
run: ./gradlew clean shadowTestClassJar shadowTestJar
- name: set up test folders and links
run: |
mkdir ./testJars;
echo $( find ./build/libs/ -name "gatk*test.jar");
cp -r $( find ./build/libs/ -name "gatk*test.jar" ) ./testJars;
cp -r $( find ./build/libs/ -name "gatk*testDependencies.jar" ) ./testJars;
mkdir -p build/reports/;
chmod -R a+w build/reports/;
cp scripts/docker/dockertest.gradle .
- name: Download docker image artifact
uses: actions/download-artifact@v3
with:
name: gatkDockerImage
path: /tmp
- name: Load Docker image
run: |
docker load --input /tmp/myimage.tar
docker image ls -a
- name: pull lfs files
run: git lfs pull
- name: run-docker-tests
id: docker-jacocco-tests
run: |
docker run \
-v $(pwd):/gatkCloneMountPoint:cached \
-v $(pwd)/testJars:/jars:cached \
--rm \
-e "scala.version=${SCALA_VERSION}" \
-e "TEST_VERBOSITY=minimal" \
-e "TEST_TYPE=${TEST_TYPE}" \
-t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} \
bash --init-file /gatk/gatkenv.rc /root/run_unit_tests.sh;
TEST_EXIT_VALUE=$?;
$( exit ${TEST_EXIT_VALUE} );
sudo chmod -R a+w build/reports/;
mkdir build/reports/tests/test \
&& cp -rp build/reports/tests/testOnPackagedReleaseJar/* build/reports/tests/test \
&& rm -r build/reports/tests/testOnPackagedReleaseJar;
- uses: ./.github/actions/upload-gatk-test-results
if: always()
with:
warnPR: ${{ github.event_name == 'pull_request' && steps.docker-jacocco-tests.outcome != 'success' }}
is-docker: true
repo-token: ${{ secrets.GITHUB_TOKEN }}
job-matrix-id: ${{ github.run_id }}.${{ strategy.job-index }}
repo-path: ${{ github.ref_name }}_${{ github.run_id }}.${{ strategy.job-index }}
bot-comment-key: ${{ secrets.GATK_BOT_COMMENT_KEY }}
identifier: Java ${{ matrix.Java }} test on docker ${{matrix.testType}}
only-artifact: ${{ needs.check-secrets.outputs.google-credentials != 'true' }}
#run wdl validation on WDLs in the scripts directory
scriptsWdlValidation:
runs-on: ubuntu-latest
name: Validate script WDLs using womtools
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: ./.github/actions/purge-runner-disk
- name: Set up java 17
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
cache: gradle
- uses: ./.github/actions/install-cromwell
with:
CROMWELL_VERSION: ${{ env.CROMWELL_VERSION }}
- name: Run Scripts WDL Validation Test
run: ./gradlew gatkValidateScriptsWdl
#run wdl validation on generated WDLs
generatedWdlValidation:
runs-on: ubuntu-latest
name: Validate generated WDLs using womtools
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: ./.github/actions/purge-runner-disk
- name: Set up java 17
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
cache: gradle
- uses: ./.github/actions/install-cromwell
with:
CROMWELL_VERSION: ${{ env.CROMWELL_VERSION }}
- name: Run Generated WDL Validation Test
run: ./gradlew gatkValidateGeneratedWdl
#Run our various targeted medium scale wdl wiring tests
wdlTests:
runs-on: ubuntu-latest
strategy:
matrix:
wdlTest: [ 'RUN_CNV_GERMLINE_COHORT_WDL', 'RUN_CNV_GERMLINE_CASE_WDL', 'RUN_CNV_SOMATIC_WDL', 'RUN_M2_WDL', 'RUN_CNN_WDL', 'RUN_VCF_SITE_LEVEL_FILTERING_WDL' ]
continue-on-error: true
name: WDL test ${{ matrix.wdlTest }} on cromwell
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: ./.github/actions/purge-runner-disk
- name: pull lfs files
run: git lfs pull
- name: Set up java 17
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
cache: gradle
- uses: ./.github/actions/install-cromwell
with:
CROMWELL_VERSION: ${{ env.CROMWELL_VERSION }}
- name: "Exporting Environment Variables for Wdl Scripts"
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]
then
echo "CI_PULL_REQUEST=${{ github.event.number }}" >> $GITHUB_ENV
fi
echo "CI_COMMIT=${{ github.sha }}" >> $GITHUB_ENV
- name: "CNV_GERMLINE_COHORT_WDL_TEST"
if: ${{ matrix.wdlTest == 'RUN_CNV_GERMLINE_COHORT_WDL' }}
run: |
echo "Running CNV germline cohort workflow";
bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh COHORT;
- name: "CNV_GERMLINE_CASE_WDL_TEST"
if: ${{ matrix.wdlTest == 'RUN_CNV_GERMLINE_CASE_WDL' }}
run: |
echo "Running CNV germline case workflow";
bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh CASE;
- name: "CNV_SOMATIC_WDL_TEST"
if: ${{ matrix.wdlTest == 'RUN_CNV_SOMATIC_WDL' }}
run: |
echo "Running CNV somatic workflows";
bash scripts/cnv_cromwell_tests/somatic/run_cnv_somatic_workflows.sh;
- name: "M2_WDL_TEST"
if: ${{ matrix.wdlTest == 'RUN_M2_WDL' }}
run: |
echo "Deleting some unused files before running M2 WDL...";
rm -Rf src/test/resources/large/VQSR;
echo "Running M2 WDL";
bash scripts/m2_cromwell_tests/run_m2_wdl.sh;
- name: "CNN_WDL_TEST"
if: ${{ matrix.wdlTest == 'RUN_CNN_WDL' }}
run: |
echo "Running CNN WDL";
bash scripts/cnn_variant_cromwell_tests/run_cnn_variant_wdl.sh;
- name: "VCF_SITE_LEVEL_FILTERING_WDL_TEST"
if: ${{ matrix.wdlTest == 'RUN_VCF_SITE_LEVEL_FILTERING_WDL' }}
run: |
echo "Running VCF Site Level Filtering WDL";
bash scripts/vcf_site_level_filtering_cromwell_tests/run_vcf_site_level_filtering_wdl.sh;