WDL generation. (#6504)

.travis.yml

@@ -18,6 +18,7 @@ env:
   - SCALA_VERSION=2.11 RUN_CNV_SOMATIC_WDL=true TESTS_REQUIRE_GCLOUD=true
   - SCALA_VERSION=2.11 RUN_M2_WDL=true TESTS_REQUIRE_GCLOUD=true
   - SCALA_VERSION=2.11 RUN_CNN_WDL=true TESTS_REQUIRE_GCLOUD=true
+  - SCALA_VERSION=2.11 TEST_TYPE=wdlGen
   global:
   #gradle needs this
   - TERM=dumb
@@ -41,7 +42,8 @@ env:
   #artifactory password
   - secure: "E0LWXgX3aWSE/DWHXXDx4vrAq4uX6vKg402wToaZ5otbHQ/UP0H7/FA5jomavAXoC46oMVHZcEltZ5OVhuJ0NW8yYxUCecJ1D/YvVQmnfFABcV/qLM+k4e2rYQOKVw/pejB2gG8XdTA+XE2WyTeENbmIkputS8f1ndKWCmZxuuk="
   # For cromwell jar download
-  - CROMWELL_JAR=$HOME/cromwell-51.jar
+  - export CROMWELL_JAR=$HOME/cromwell-51.jar
+  - export WOMTOOL_JAR=$HOME/womtool-51.jar
 matrix:
   fast_finish: true
   include:
@@ -89,8 +91,9 @@ before_install:
     echo "Done testing github authentication";
   fi;
 # Download Cromwell jar -- if you change the version, please change the CROMWELL_JAR env variable above, too.
-- if [[ $RUN_CNV_GERMLINE_COHORT_WDL == true || $RUN_CNV_GERMLINE_CASE_WDL == true || $RUN_CNV_SOMATIC_WDL == true || $RUN_M2_WDL == true || $RUN_CNN_WDL == true ]]; then
+- if [[ $TEST_TYPE == wdlGen || $RUN_CNV_GERMLINE_COHORT_WDL == true || $RUN_CNV_GERMLINE_CASE_WDL == true || $RUN_CNV_SOMATIC_WDL == true || $RUN_M2_WDL == true || $RUN_CNN_WDL == true ]]; then
     wget -O $CROMWELL_JAR https://github.com/broadinstitute/cromwell/releases/download/51/cromwell-51.jar;
+    wget -O $WOMTOOL_JAR https://github.com/broadinstitute/cromwell/releases/download/51/womtool-51.jar;
   fi;
 # Download git lfs files
 - git lfs
@@ -122,6 +125,8 @@ script:
 # run tests
 - if [[ $TRAVIS_EVENT_TYPE == cron ]]; then
     echo "Not running any tests for nightly builds";
+  elif [[ $TEST_TYPE == wdlGen ]]; then
+    ./gradlew gatkWDLGenValidation;
   elif [[ $TRAVIS_SECURE_ENV_VARS == false && $TEST_TYPE == cloud ]]; then
     echo "Can't run cloud tests without keys so don't run tests";
   elif [[ $RUN_CNV_GERMLINE_COHORT_WDL == true ]]; then

build.gradle

@@ -746,6 +746,95 @@ task gatkTabComplete(type: Javadoc, dependsOn: classes) {
     options.addStringOption("caller-post-arg-max-occurs", "1 1 1 1 1 1 1 1 1 1")
 }
 
+def getWLDInputJSONTestFileNameFromWDLName(File wdlName) {
+    String fileWithoutExt = wdlName.name.take(wdlName.name.lastIndexOf('.'))
+    return new File (wdlName.getParentFile(), fileWithoutExt + "Inputs.json").getAbsolutePath()
+}
+
+// Generate GATK Tool WDL
+task gatkWDLGen(type: Javadoc, dependsOn: classes) {
+    final File gatkWDLDir = new File("$docBuildDir/wdlGen")
+    outputs.dir(gatkWDLDir)
+    doFirst {
+        // make sure the output folder exists or we can create it
+        if (!gatkWDLDir.exists() && !gatkWDLDir.mkdirs()) {
+            throw new GradleException(String.format("Failure creating folder (%s) for GATK DL output in task (%s)",
+                    gatkWDLDir.getAbsolutePath(),
+                    it.name));
+        }
+        copy {
+            from('src/main/resources/org/broadinstitute/hellbender/utils/wdlTemplates/common.html')
+            into gatkWDLDir
+        }
+    }
+    source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
+    include '**/*.java'
+
+    // The gatkWDLGen process instantiates any documented feature classes, so to run it we need the entire
+    // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives.
+    classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles
+    options.docletpath = classpath.asType(List)
+    options.doclet = "org.broadinstitute.hellbender.utils.help.GATKWDLDoclet"
+
+    outputs.dir(gatkWDLDir)
+    options.destinationDirectory(gatkWDLDir)
+
+    options.addStringOption("settings-dir", "src/main/resources/org/broadinstitute/hellbender/utils/wdlTemplates");
+    options.addStringOption("output-file-extension", "wdl")
+    options.addStringOption("index-file-extension", "html")
+
+    options.addStringOption("absolute-version", getVersion())
+    options.addStringOption("build-timestamp", ZonedDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME))
+
+    // the wdl doclet will populate the test JSON input files with the name of a dummy
+    // file in this location, in order to satisfy cromwell's attempts to localize inputs and outputs
+    options.addStringOption("build-dir", System.getenv("TRAVIS_BUILD_DIR") ?: ".")
+}
+
+task gatkWDLGenValidation(dependsOn: [gatkWDLGen, shadowJar]) {
+    def execWDLValidation = { validateWDL ->
+        println "Executing: $validateWDL"
+        try {
+            def retCode = validateWDL.execute().waitFor()
+            if (retCode.intValue() != 0) {
+                throw new GradleException("Execution of \"$validateWDL\" failed with exit code: $retCode.")
+            }
+            return retCode
+        } catch (IOException e) {
+            throw new GradleException("An IOException occurred while attempting to execute the command $validateWDL.")
+        }
+    }
+
+    doLast {
+        // first, run the womtool validator
+        final File wdlGenFolder = new File("$docBuildDir/wdlGen")
+        def wdlFiles = fileTree(dir: wdlGenFolder).filter {
+            f -> !f.getAbsolutePath().endsWith(".html") && !f.getAbsolutePath().endsWith(".json")
+        }
+        wdlFiles.any() { wdlFile ->
+            final womtoolLocation = System.getenv('WOMTOOL_JAR')
+            final validateWDLCommand = "java -jar $womtoolLocation validate $wdlFile"
+            execWDLValidation(validateWDLCommand)
+        }
+
+        // now execute the *AllArgs test wdls using cromwell
+        wdlFiles = fileTree(dir: wdlGenFolder).filter {
+            f -> f.getAbsolutePath().endsWith("AllArgsTest.wdl")
+        }
+        wdlFiles.any() { wdlFile ->
+            final cromwellLocation = System.getenv('CROMWELL_JAR')
+            final testInputJSON = getWLDInputJSONTestFileNameFromWDLName(wdlFile)
+            final runWDLCommand = "java -jar $cromwellLocation run --inputs $testInputJSON $wdlFile"
+            // the test JSON input file is populated by the WDL gen process with this dummy file name
+            // to satisfy cromwell's attempt to de/localize input files
+            def buildDir = System.getenv("TRAVIS_BUILD_DIR") ?: "."
+            final createDummyTestInputFile = "touch $buildDir/dummyWDLTestFile"
+            execWDLValidation(createDummyTestInputFile)
+            execWDLValidation(runWDLCommand)
+        }
+    }
+
+}
 
 
 /**

src/main/java/org/broadinstitute/hellbender/CommandLineArgumentValidatorMain.java

@@ -0,0 +1,28 @@
+package org.broadinstitute.hellbender;
+
+import org.broadinstitute.hellbender.cmdline.CommandLineProgram;
+import org.broadinstitute.hellbender.cmdline.CommandLineArgumentValidator;
+
+import java.util.Arrays;
+
+/**
+ * Main class to be used as an alternative entry point to {@code org.broadinstitute.hellbender.Main} for performing
+ * command line validation only rather than executing the tool. Used only for testing. Failures manifest as exceptions.
+ */
+public class CommandLineArgumentValidatorMain extends Main {
+
+    // The main entry point to run GATK tools in command line validation mode only.
+    public static void main(final String[] argv) {
+        new CommandLineArgumentValidatorMain().validateCommandLine(argv);
+    }
+
+    /**
+     * Call the command line program (specified in the input arguments) in command line validation mode only.
+     * @param argv the raw arguments, including the name of the target tool, to run in command line validation mode
+     */
+    public void validateCommandLine(final String[] argv) {
+        final CommandLineProgram program = setupConfigAndExtractProgram(argv, getPackageList(), getClassList(), getCommandLineName());
+        final String[] mainArgs = Arrays.copyOfRange(argv, 1, argv.length);
+        new CommandLineArgumentValidator(program).instanceMain(mainArgs);
+    }
+}

src/main/java/org/broadinstitute/hellbender/Main.java

@@ -3,10 +3,7 @@
 import com.google.cloud.storage.StorageException;
 import htsjdk.samtools.util.StringUtil;
 import org.broadinstitute.barclay.argparser.*;
-import org.broadinstitute.hellbender.cmdline.CommandLineProgram;
-import org.broadinstitute.hellbender.cmdline.DeprecatedToolsRegistry;
-import org.broadinstitute.hellbender.cmdline.PicardCommandLineProgramExecutor;
-import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
+import org.broadinstitute.hellbender.cmdline.*;
 import org.broadinstitute.hellbender.exceptions.PicardNonZeroExitException;
 import org.broadinstitute.hellbender.exceptions.UserException;
 import org.broadinstitute.hellbender.utils.ClassUtils;
@@ -314,7 +311,8 @@ private CommandLineProgram extractCommandLineProgram( final String[] args,
         toCheck.addAll(classList);
         final Map<String, Class<?>> simpleNameToClass = new LinkedHashMap<>();
         for (final Class<?> clazz : toCheck) {
-            if (clazz.equals(PicardCommandLineProgramExecutor.class)) {
+            if (clazz.equals(PicardCommandLineProgramExecutor.class) ||
+                    clazz.equals(CommandLineArgumentValidator.class)) {
                 continue;
             }
             // No interfaces, synthetic, primitive, local, or abstract classes.

src/main/java/org/broadinstitute/hellbender/cmdline/CommandLineArgumentValidator.java

@@ -0,0 +1,37 @@
+package org.broadinstitute.hellbender.cmdline;
+
+import org.broadinstitute.hellbender.exceptions.GATKException;
+
+/**
+ * Adapter shim/alternate GATK entry point for use by GATK tests to run tools in command line argument
+ * validation mode. This class does not actually tools, it only validates that the command line arguments
+ * are legal for a given invocation.
+ *
+ * Note that this class does not have it's own CommandLineProgramProperties annotation.
+ */
+public class CommandLineArgumentValidator extends CommandLineProgram {
+
+    // Our target command line program, to which we delegate arg parsing calls.
+    final private CommandLineProgram targetCommandLineProgram;
+
+    public CommandLineArgumentValidator(final CommandLineProgram targetCommandLineProgram) {
+        this.targetCommandLineProgram = targetCommandLineProgram;
+    }
+
+    /**
+     * Entry point to run command line argument validation only.
+     */
+    @Override
+    public Object instanceMain(final String[] argv) {
+        // just call parseArgs and then return
+        return targetCommandLineProgram.parseArgs(argv);
+    }
+
+    @Override
+    protected Object doWork() {
+        // This method should never be called directly. Call instanceMain instead.
+        throw new GATKException.ShouldNeverReachHereException(
+                String.format("Attempt to call the doWork method on the validator test tool \"%s\" directly.",
+                        targetCommandLineProgram.getClass().getName()));
+    }
+}

src/main/java/org/broadinstitute/hellbender/cmdline/argumentcollections/OptionalReferenceInputArgumentCollection.java

@@ -1,6 +1,7 @@
 package org.broadinstitute.hellbender.cmdline.argumentcollections;
 
 import org.broadinstitute.barclay.argparser.Argument;
+import org.broadinstitute.barclay.argparser.WorkflowResource;
 import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
 import org.broadinstitute.hellbender.engine.GATKPath;
 
@@ -10,6 +11,7 @@
 public final class OptionalReferenceInputArgumentCollection extends ReferenceInputArgumentCollection {
     private static final long serialVersionUID = 1L;
 
+    @WorkflowResource(input=true, output=false, companionResources = {"referenceDictionary", "referenceIndex"})
     @Argument(fullName = StandardArgumentDefinitions.REFERENCE_LONG_NAME, shortName = StandardArgumentDefinitions.REFERENCE_SHORT_NAME, doc = "Reference sequence", optional = true)
     private GATKPath referenceInputPathSpecifier;
 

src/main/java/org/broadinstitute/hellbender/cmdline/argumentcollections/RequiredReferenceInputArgumentCollection.java

@@ -1,6 +1,7 @@
 package org.broadinstitute.hellbender.cmdline.argumentcollections;
 
 import org.broadinstitute.barclay.argparser.Argument;
+import org.broadinstitute.barclay.argparser.WorkflowResource;
 import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
 import org.broadinstitute.hellbender.engine.GATKPath;
 
@@ -10,6 +11,7 @@
 public final class RequiredReferenceInputArgumentCollection extends ReferenceInputArgumentCollection {
     private static final long serialVersionUID = 1L;
 
+    @WorkflowResource(input=true, output=false, companionResources = {"referenceDictionary", "referenceIndex"})
     @Argument(fullName = StandardArgumentDefinitions.REFERENCE_LONG_NAME, shortName = StandardArgumentDefinitions.REFERENCE_SHORT_NAME, doc = "Reference sequence file", optional = false)
     private GATKPath referenceInputPathSpecifier;
 

src/main/java/org/broadinstitute/hellbender/tools/PrintReads.java

@@ -2,7 +2,9 @@
 
 import org.broadinstitute.barclay.argparser.Argument;
 import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
+import org.broadinstitute.barclay.argparser.WorkflowResource;
 import org.broadinstitute.barclay.help.DocumentedFeature;
+import org.broadinstitute.barclay.argparser.RuntimeProperties;
 import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
 import org.broadinstitute.hellbender.engine.GATKPath;
 import picard.cmdline.programgroups.ReadDataManipulationProgramGroup;
@@ -83,8 +85,10 @@
         programGroup = ReadDataManipulationProgramGroup.class
 )
 @DocumentedFeature
+@RuntimeProperties(memory = "1GB")
 public final class PrintReads extends ReadWalker {
 
+    @WorkflowResource(input=false, output=true, companionResources={StandardArgumentDefinitions.OUTPUT_LONG_NAME + "Index"})
     @Argument(fullName = StandardArgumentDefinitions.OUTPUT_LONG_NAME,
             shortName = StandardArgumentDefinitions.OUTPUT_SHORT_NAME,
             doc="Write output to this file")

src/main/java/org/broadinstitute/hellbender/tools/walkers/varianteval/VariantEval.java

@@ -186,7 +186,7 @@ public class VariantEval extends MultiVariantWalker {
      * List of feature tracks to be used for specifying "known" variants other than dbSNP.
      */
     @Argument(shortName="known-name", doc="Name of feature bindings containing variant sites that should be treated as known when splitting eval features into known and novel subsets", optional=true)
-    protected HashSet<String> KNOWN_NAMES = new HashSet<String>();
+    protected Set<String> KNOWN_NAMES = new HashSet<String>();
     List<FeatureInput<VariantContext>> knowns = new ArrayList<>();
 
     // Stratification arguments