Code Review for #8752

.github/actions/upload-gatk-test-results/action.yml

@@ -40,9 +40,10 @@ runs:
         name: test-results-${{ inputs.is-docker == 'true' && 'docker-' ||  '' }}${{ matrix.Java }}-${{ matrix.testType }}
         path: build/reports/tests
 
-    - name: Upload to codecov
-      run: bash <(curl -s https://raw.githubusercontent.com/broadinstitute/codecov-bash-uploader/main/codecov-verified.bash)
-      shell: bash
+          # Disabling codecov because it is timing out and failing builds that otherwise succeed.
+          ##    - name: Upload to codecov
+          ##      run: bash <(curl -s https://raw.githubusercontent.com/broadinstitute/codecov-bash-uploader/main/codecov-verified.bash)
+          ##      shell: bash
 
     - name: Upload Reports
       if: ${{ inputs.only-artifact != 'true' }}
@@ -91,4 +92,4 @@ runs:
       run: |
         pip install --user PyGithub;
         python scripts/github_actions/Reporter.py ${{ steps.uploadreports.outputs.view_url }};
-      shell: bash
+      shell: bash

.github/workflows/dependency_submission.yml

@@ -0,0 +1,22 @@
+name: Dependency Submission
+
+on:
+  push:
+    branches: [ 'master' ]
+
+permissions:
+  contents: write
+
+jobs:
+  dependency-submission:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout sources
+      uses: actions/checkout@v4
+    - name: Setup Java
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: 17
+    - name: Generate and submit dependency graph
+      uses: gradle/actions/dependency-submission@v4

.github/workflows/gatk-tests.yml

@@ -291,7 +291,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        wdlTest: [ 'RUN_CNV_GERMLINE_COHORT_WDL', 'RUN_CNV_GERMLINE_CASE_WDL', 'RUN_CNV_SOMATIC_WDL', 'RUN_M2_WDL', 'RUN_CNN_WDL', 'RUN_VCF_SITE_LEVEL_FILTERING_WDL' ]
+        wdlTest: [ 'RUN_CNV_GERMLINE_COHORT_WDL', 'RUN_CNV_GERMLINE_CASE_WDL', 'RUN_CNV_SOMATIC_WDL', 'RUN_M2_WDL', 'RUN_VCF_SITE_LEVEL_FILTERING_WDL' ]
     continue-on-error: true
     name: WDL test ${{ matrix.wdlTest }} on cromwell
     steps:
@@ -345,12 +345,6 @@ jobs:
           echo "Running M2 WDL";
           bash scripts/m2_cromwell_tests/run_m2_wdl.sh;
 
-      - name: "CNN_WDL_TEST"
-        if: ${{ matrix.wdlTest == 'RUN_CNN_WDL' }}
-        run: |
-          echo "Running CNN WDL";
-          bash scripts/cnn_variant_cromwell_tests/run_cnn_variant_wdl.sh;
-
       - name: "VCF_SITE_LEVEL_FILTERING_WDL_TEST"
         if: ${{ matrix.wdlTest == 'RUN_VCF_SITE_LEVEL_FILTERING_WDL' }}
         run: |

.gitignore

@@ -44,3 +44,4 @@ funcotator_tmp
 
 #Test generated dot files
 test*.dot
+.vscode/

Dockerfile

@@ -1,16 +1,18 @@
-ARG BASE_DOCKER=broadinstitute/gatk:gatkbase-3.2.0
+ARG BASE_DOCKER=broadinstitute/gatk:gatkbase-3.3.1
 
 # stage 1 for constructing the GATK zip
 FROM ${BASE_DOCKER} AS gradleBuild
 LABEL stage=gatkIntermediateBuildImage
 ARG RELEASE=false
 
-RUN ls .
+
 ADD . /gatk
 WORKDIR /gatk
 
 # Get an updated gcloud signing key, in case the one in the base image has expired
-RUN rm /etc/apt/sources.list.d/google-cloud-sdk.list && \
+#Download only resources required for the build, not for testing
+RUN ls . && \
+    rm /etc/apt/sources.list.d/google-cloud-sdk.list && \
     apt update &&\
     apt-key list && \
     curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \
@@ -19,16 +21,13 @@ RUN rm /etc/apt/sources.list.d/google-cloud-sdk.list && \
     apt-get -y clean  && \
     apt-get -y autoclean  && \
     apt-get -y autoremove && \
-    rm -rf /var/lib/apt/lists/*
-RUN git lfs install --force
-
-#Download only resources required for the build, not for testing
-RUN git lfs pull --include src/main/resources/large
-
-RUN export GRADLE_OPTS="-Xmx4048m -Dorg.gradle.daemon=false" && /gatk/gradlew clean collectBundleIntoDir shadowTestClassJar shadowTestJar -Drelease=$RELEASE
-RUN cp -r $( find /gatk/build -name "*bundle-files-collected" )/ /gatk/unzippedJar/
-RUN unzip -o -j $( find /gatk/unzippedJar -name "gatkPython*.zip" ) -d /gatk/unzippedJar/scripts
-RUN chmod -R a+rw /gatk/unzippedJar
+    rm -rf /var/lib/apt/lists/* && \
+    git lfs install --force && \
+    git lfs pull --include src/main/resources/large && \
+    export GRADLE_OPTS="-Xmx4048m -Dorg.gradle.daemon=false" && /gatk/gradlew clean collectBundleIntoDir shadowTestClassJar shadowTestJar -Drelease=$RELEASE && \
+    cp -r $( find /gatk/build -name "*bundle-files-collected" )/ /gatk/unzippedJar/ && \
+    unzip -o -j $( find /gatk/unzippedJar -name "gatkPython*.zip" ) -d /gatk/unzippedJar/scripts && \
+    chmod -R a+rw /gatk/unzippedJar
 
 FROM ${BASE_DOCKER}
 
@@ -47,17 +46,17 @@ RUN chmod -R a+rw /gatk
 COPY --from=gradleBuild /gatk/unzippedJar .
 
 #Setup linked jars that may be needed for running gatk
-RUN ln -s $( find /gatk -name "gatk*local.jar" ) gatk.jar
-RUN ln -s $( find /gatk -name "gatk*local.jar" ) /root/gatk.jar
-RUN ln -s $( find /gatk -name "gatk*spark.jar" ) gatk-spark.jar
+RUN ln -s $( find /gatk -name "gatk*local.jar" ) gatk.jar && \
+    ln -s $( find /gatk -name "gatk*local.jar" ) /root/gatk.jar && \
+    ln -s $( find /gatk -name "gatk*spark.jar" ) gatk-spark.jar
 
 WORKDIR /root
 
  # Make sure we can see a help message
-RUN java -jar gatk.jar -h
-RUN mkdir /gatkCloneMountPoint
-RUN mkdir /jars
-RUN mkdir .gradle
+RUN java -jar gatk.jar -h && \
+    mkdir /gatkCloneMountPoint && \
+    mkdir /jars && \
+    mkdir .gradle
 
 WORKDIR /gatk
 
@@ -80,16 +79,13 @@ RUN echo "source activate gatk" > /root/run_unit_tests.sh && \
     echo "ln -s /gatkCloneMountPoint/build/ /gatkCloneMountPoint/scripts/docker/build" >> /root/run_unit_tests.sh && \
     echo "cd /gatk/ && /gatkCloneMountPoint/gradlew -Dfile.encoding=UTF-8 -b /gatkCloneMountPoint/dockertest.gradle testOnPackagedReleaseJar jacocoTestReportOnPackagedReleaseJar -a -p /gatkCloneMountPoint" >> /root/run_unit_tests.sh
 
-WORKDIR /root
-RUN cp -r /root/run_unit_tests.sh /gatk
-RUN cp -r gatk.jar /gatk
-ENV CLASSPATH /gatk/gatk.jar:$CLASSPATH
+RUN cp -r /root/run_unit_tests.sh /gatk && \
+    cp -r /root/gatk.jar /gatk
+ENV CLASSPATH=/gatk/gatk.jar:$CLASSPATH PATH=$CONDA_PATH/envs/gatk/bin:$CONDA_PATH/bin:$PATH
 
 # Start GATK Python environment
 
-WORKDIR /gatk
-ENV PATH $CONDA_PATH/envs/gatk/bin:$CONDA_PATH/bin:$PATH
-RUN conda env create -n gatk -f /gatk/gatkcondaenv.yml && \
+RUN conda env create -vv -n gatk -f /gatk/gatkcondaenv.yml && \
     echo "source activate gatk" >> /gatk/gatkenv.rc && \
     echo "source /gatk/gatk-completion.sh" >> /gatk/gatkenv.rc && \
     conda clean -afy && \

README.md

@@ -48,6 +48,7 @@ releases of the toolkit.
 * [How to contribute to GATK](#contribute)
 * [Discussions](#discussions)
 * [Authors](#authors)
+* [Citing GATK](#citing)
 * [License](#license)
 
 ## <a name="requirements">Requirements</a>
@@ -78,15 +79,31 @@ releases of the toolkit.
    docker client, which can be found on the [docker website](https://www.docker.com/get-docker).
 * Python Dependencies:<a name="python"></a>
     * GATK4 uses the [Conda](https://conda.io/docs/index.html) package manager to establish and manage the
-      Python environment and dependencies required by GATK tools that have a Python dependency. This environment also 
-      includes the R dependencies used for plotting in some of the tools. The ```gatk``` environment 
-      requires hardware with AVX support for tools that depend on TensorFlow (e.g. CNNScoreVariant). The GATK Docker image 
-      comes with the ```gatk``` environment pre-configured.
-      	* At this time, the only supported platforms are 64-bit Linux distributions. The required Conda environment is not
-	  currently supported on OS X/macOS. 
+      Python environment and dependencies required by Python-based GATK tools. This environment also 
+      includes the R dependencies used for plotting in some of the tools. The GATK Docker image 
+      comes with the ```gatk``` conda environment pre-configured and activated.
     * To establish the environment when not using the Docker image, a conda environment must first be "created", and
       then "activated":
-        * First, make sure [Miniconda or Conda](https://conda.io/docs/index.html) is installed (Miniconda is sufficient).
+        * First, make sure [Miniconda or Conda](https://conda.io/docs/index.html) is installed. We recommend installing
+          ```Miniconda3-py310_23.10.0-1``` from [the miniconda download page](https://repo.anaconda.com/miniconda/), selecting the Linux or
+          MacOS version of the installer as appropriate.
+            * This is the same version of ```miniconda``` used by the official GATK docker image.
+            * If you use a different version, you may run into issues.
+            * If you have an ARM-based Mac, you must select the `MacOSX-x86_64` installer, not the `MacOSX-arm64` installer,
+              and rely on Mac OS's built-in x86 emulation.
+        * Set up miniconda:
+            * Install miniconda to a location on your PATH such as ```/opt/miniconda```, and then restart your shell: 
+              ```
+              bash Miniconda3-py310_23.10.0-1-[YOUR_OS].sh -p /opt/miniconda -b
+              ```
+            * Disable conda auto-updates, which can cause compatibility issues with GATK: 
+              ```
+              conda config --set auto_update_conda false
+              ```
+            * Enable the (much) faster ```libmamba``` solver to greatly speed up creation of the conda environment: 
+              ```
+              conda config --set solver libmamba
+              ```
         * To "create" the conda environment:
             * If running from a zip or tar distribution, run the command ```conda env create -f gatkcondaenv.yml``` to
               create the ```gatk``` environment.
@@ -156,7 +173,9 @@ For more details on system packages, see the GATK [Base Dockerfile](scripts/dock
 
     * This creates a zip archive in the `build/` directory with a name like `gatk-VERSION.zip` containing a complete standalone GATK distribution, including our launcher `gatk`, both the local and spark jars, and this README.    
     * You can also run GATK commands directly from the root of your git clone after running this command.
-    * Note that you *must* have a full git clone in order to build GATK, including the git-lfs files in src/main/resources. The zipped source code alone is not buildable.
+    * Note that you *must* have a full git clone in order to build GATK, including the git-lfs files in `src/main/resources/large`. The zipped source code alone is not buildable.
+    * The large files under `src/main/resources/large/` are required to build GATK, since they are packaged inside the GATK jar and used by tools at runtime. These include things like ML models and native C/C++ libraries used for acceleration of certain tools.
+    * The large files under `src/test/resources/large/`, on the other hand, are only required by the test suite when running tests, and are not required to build GATK.
 
 * **Other ways to build:**
     * `./gradlew installDist`  
@@ -671,5 +690,8 @@ Thank you for getting involved!
 The authors list is maintained in the [AUTHORS](https://github.com/broadinstitute/gatk/edit/master/AUTHORS) file. 
 See also the [Contributors](https://github.com/broadinstitute/gatk/graphs/contributors) list at github. 
 
+## <a name="citing">Citing GATK</a>
+If you use GATK in your research, please see [this article](https://gatk.broadinstitute.org/hc/en-us/articles/360035530852-How-should-I-cite-GATK-in-my-own-publications) for details on how to properly cite GATK.
+
 ## <a name="license">License</a>
 Licensed under the Apache 2.0 License. See the [LICENSE.txt](https://github.com/broadinstitute/gatk/blob/master/LICENSE.TXT) file.