Updating htsjdk to 2.19.0

build.gradle

@@ -64,7 +64,7 @@ def ensureBuildPrerequisites(requiredJavaVersion, buildPrerequisitesMessage) {
 }
 ensureBuildPrerequisites(requiredJavaVersion, buildPrerequisitesMessage)
 
-final htsjdkVersion = System.getProperty('htsjdk.version', '2.18.2')
+final htsjdkVersion = System.getProperty('htsjdk.version', '2.19.0')
 
 // We use a custom shaded build of the NIO library to avoid a regression in the authentication layer.
 // GATK does the same, see https://github.com/broadinstitute/gatk/issues/3591

src/main/java/picard/sam/SamFormatConverter.java

@@ -33,14 +33,13 @@
 import htsjdk.samtools.util.IOUtil;
 import htsjdk.samtools.util.Log;
 import htsjdk.samtools.util.ProgressLogger;
+import org.broadinstitute.barclay.argparser.Argument;
+import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
 import org.broadinstitute.barclay.help.DocumentedFeature;
 import picard.PicardException;
 import picard.cmdline.CommandLineProgram;
-import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
-import org.broadinstitute.barclay.argparser.Argument;
 import picard.cmdline.StandardOptionDefinitions;
 import picard.cmdline.programgroups.ReadDataManipulationProgramGroup;
-import picard.illumina.parser.ReadData;
 
 import java.io.File;
 
@@ -55,15 +54,12 @@
         oneLineSummary = "Convert a BAM file to a SAM file, or a SAM to a BAM",
         programGroup = ReadDataManipulationProgramGroup.class)
 @DocumentedFeature
-public class
-        SamFormatConverter extends CommandLineProgram {
-
-    private static final String PROGRAM_VERSION = "1.0";
+public class SamFormatConverter extends CommandLineProgram {
 
     // The following attributes define the command-line arguments
-
     @Argument(doc = "The BAM or SAM file to parse.", shortName = StandardOptionDefinitions.INPUT_SHORT_NAME)
     public File INPUT;
+
     @Argument(doc = "The BAM or SAM output file. ", shortName = StandardOptionDefinitions.OUTPUT_SHORT_NAME)
     public File OUTPUT;
 
@@ -72,12 +68,24 @@ public static void main(final String[] argv) {
     }
 
     protected int doWork() {
-        IOUtil.assertFileIsReadable(INPUT);
-        IOUtil.assertFileIsWritable(OUTPUT);
-        final SamReader reader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(INPUT);
-        final SAMFileWriter writer = new SAMFileWriterFactory().makeWriter(reader.getFileHeader(), true, OUTPUT, REFERENCE_SEQUENCE);
+        convert(INPUT, OUTPUT, REFERENCE_SEQUENCE, CREATE_INDEX);
+        return 0;
+    }
 
-        if (CREATE_INDEX && writer.getFileHeader().getSortOrder() != SAMFileHeader.SortOrder.coordinate) {
+    /**
+     * Convert a file from one of sam/bam/cram format to another based on the extension of output.
+     *
+     * @param input             input file in one of sam/bam/cram format
+     * @param output            output to write converted file to, the conversion is based on the extension of this filename
+     * @param referenceSequence the reference sequence to use, necessary when reading/writing cram
+     * @param createIndex       whether or not an index should be written alongside the output file
+     */
+    public static void convert(final File input, final File output, final File referenceSequence, final Boolean createIndex) {
+        IOUtil.assertFileIsReadable(input);
+        IOUtil.assertFileIsWritable(output);
+        final SamReader reader = SamReaderFactory.makeDefault().referenceSequence(referenceSequence).open(input);
+        final SAMFileWriter writer = new SAMFileWriterFactory().makeWriter(reader.getFileHeader(), true, output, referenceSequence);
+        if (createIndex && writer.getFileHeader().getSortOrder() != SAMFileHeader.SortOrder.coordinate) {
             throw new PicardException("Can't CREATE_INDEX unless sort order is coordinate");
         }
 
@@ -88,6 +96,5 @@ protected int doWork() {
         }
         CloserUtil.close(reader);
         writer.close();
-        return 0;
     }
 }

src/main/java/picard/vcf/RenameSampleInVcf.java

@@ -115,7 +115,7 @@ protected int doWork() {
         IOUtil.assertFileIsReadable(INPUT);
         IOUtil.assertFileIsWritable(OUTPUT);
 
-        final VCFFileReader in = new VCFFileReader(INPUT);
+        final VCFFileReader in = new VCFFileReader(INPUT, false);
         final VCFHeader header = in.getFileHeader();
 
         if (header.getGenotypeSamples().size() > 1) {

src/test/java/picard/analysis/replicates/CollectIndependentReplicatesMetricTest.java

@@ -60,7 +60,7 @@ public void prepareBams() throws IOException {
 
     @AfterTest
     public void tearDown() {
-        TestUtil.recursiveDelete(bamOutDir);
+        IOUtil.recursiveDelete(bamOutDir.toPath());
     }
 
     @DataProvider(name = "simpleTests")

src/test/java/picard/fastq/BamToBfqTest.java

@@ -42,7 +42,7 @@ public void testBamToBfq(final File input, final boolean isPairedRun,
 
             Assert.assertEquals(Files.readAllBytes(output.toPath()), Files.readAllBytes(expectedBFQ.toPath()));
         } finally {
-            TestUtil.recursiveDelete(analysisDir);
+            IOUtil.recursiveDelete(analysisDir.toPath());
         }
     }
 }

src/test/java/picard/illumina/IlluminaBasecallsToFastqTest.java

@@ -116,7 +116,7 @@ public void testMultiplexWithIlluminaReadNameHeaders() throws Exception {
             }
 
         } finally {
-            TestUtil.recursiveDelete(outputDir);
+            IOUtil.recursiveDelete(outputDir.toPath());
         }
     }
 
@@ -222,7 +222,7 @@ private void runStandardTest(final int lane, final String jobName, final String
                 }
             }
         } finally {
-            TestUtil.recursiveDelete(outputDir);
+            IOUtil.recursiveDelete(outputDir.toPath());
         }
     }
 

src/test/java/picard/illumina/IlluminaBasecallsToSamTest.java

@@ -260,8 +260,7 @@ private void runStandardTest(final int lane, final String jobName, final String
                 IOUtil.assertFilesEqual(outputSam, new File(testDataDir, outputSam.getName()));
             }
         } finally {
-            TestUtil.recursiveDelete(outputDir.toFile());
-            Files.delete(outputDir);
+            IOUtil.recursiveDelete(outputDir);
         }
     }
 }

src/test/java/picard/sam/SamFileConverterTest.java → src/test/java/picard/sam/SamFormatConverterTest.java

@@ -24,6 +24,8 @@
 
 package picard.sam;
 
+import htsjdk.samtools.Defaults;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 import java.io.File;
@@ -34,63 +36,47 @@
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 
-public class SamFileConverterTest {
+public class SamFormatConverterTest {
 
-    private static final File TEST_DATA_DIR = new File("testdata/picard/sam/SamFileConverterTest");
+    private static final File TEST_DATA_DIR = new File("testdata/picard/sam/SamFormatConverterTest");
     private static final File unmappedSam = new File(TEST_DATA_DIR, "unmapped.sam");
     private static final File unmappedBam = new File(TEST_DATA_DIR, "unmapped.bam");
     private static final File unmappedCram = new File(TEST_DATA_DIR, "unmapped.cram");
 
-    @Test
-    public void testSAMToBAM() {
-        convertFile(unmappedSam, unmappedBam, ".bam");
-    }
-
-    @Test
-    public void testSAMToCRAM() {
-        convertFile(unmappedSam, unmappedCram, ".cram");
-    }
+    private static final File ESSENTIALLY_EMPTY_REFERENCE_TO_USE_WITH_UNMAPPED_CRAM = new File(TEST_DATA_DIR, "basicallyEmpty.fasta");
 
-    @Test
-    public void testBAMToCRAM() {
-        convertFile(unmappedBam, unmappedCram, ".cram");
-    }
-
-    @Test
-    public void testBAMToSAM() {
-        convertFile(unmappedBam, unmappedSam, ".sam");
-    }
+    @DataProvider
+    public Object[][] conversionCases() {
+        return new Object[][]{
+                {unmappedSam, unmappedBam, ".bam"},
+                {unmappedSam, unmappedCram, ".cram"},
+                {unmappedBam, unmappedCram, ".cram"},
+                {unmappedBam, unmappedSam, ".sam"},
+                {unmappedCram, unmappedBam, ".bam"},
+                {unmappedCram, unmappedSam, ".sam"},
 
-    @Test
-    public void testCRAMToBAM() {
-        convertFile(unmappedCram, unmappedBam, ".bam");
+        };
     }
 
-    @Test
-    public void testCRAMToSAM() {
-        convertFile(unmappedCram, unmappedSam, ".sam");
+    @Test(dataProvider = "conversionCases")
+    public void testConvert(File input, File expected, String extension) throws IOException {
+        convertFile(input, expected, extension);
     }
 
 
-    private void convertFile(final File inputFile, final File fileToCompare, final String extension) {
-        final SamFormatConverter samFormatConverter = new SamFormatConverter();
-        final List<File> samFiles = new ArrayList<File>();
+    private void convertFile(final File inputFile, final File fileToCompare, final String extension) throws IOException {
+        final List<File> samFiles = new ArrayList<>();
         final ValidateSamFile validateSamFile = new ValidateSamFile();
         final CompareSAMs compareSAMs = new CompareSAMs();
 
-        samFormatConverter.INPUT = inputFile;
-        try {
-            samFormatConverter.OUTPUT = File.createTempFile("SamFileConverterTest." + inputFile.getName(), extension);
-            samFormatConverter.OUTPUT.deleteOnExit();
-        } catch (final IOException e) {
-            e.printStackTrace();
-        }
-        samFormatConverter.doWork();
+        final File output = File.createTempFile("SamFormatConverterTest." + inputFile.getName(), extension);
+        output.deleteOnExit();
+        SamFormatConverter.convert(inputFile, output, ESSENTIALLY_EMPTY_REFERENCE_TO_USE_WITH_UNMAPPED_CRAM, Defaults.CREATE_INDEX);
 
-        validateSamFile.INPUT = samFormatConverter.OUTPUT;
+        validateSamFile.INPUT = output;
         assertEquals(validateSamFile.doWork(), 0);
 
-        samFiles.add(samFormatConverter.OUTPUT);
+        samFiles.add(output);
         samFiles.add(fileToCompare);
 
         compareSAMs.samFiles = samFiles;

src/test/java/picard/sam/markduplicates/AbstractMarkDuplicatesCommandLineProgramTester.java

@@ -33,6 +33,7 @@
 import htsjdk.samtools.metrics.MetricsFile;
 import htsjdk.samtools.util.CloseableIterator;
 import htsjdk.samtools.util.FormatUtil;
+import htsjdk.samtools.util.IOUtil;
 import htsjdk.samtools.util.TestUtil;
 import org.testng.Assert;
 import picard.cmdline.CommandLineProgram;
@@ -202,7 +203,7 @@ public MetricsFile<DuplicationMetrics, Double> testMetrics() throws IOException
             }
             return metricsOutput;
         } finally {
-            TestUtil.recursiveDelete(getOutputDir());
+            IOUtil.recursiveDelete(getOutputDir().toPath());
         }
     }
 

src/test/java/picard/sam/markduplicates/MarkDuplicatesSetSizeHistogramTester.java

@@ -26,14 +26,17 @@
 
 import htsjdk.samtools.metrics.MetricsFile;
 import htsjdk.samtools.util.Histogram;
-import htsjdk.samtools.util.TestUtil;
+import htsjdk.samtools.util.IOUtil;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import picard.cmdline.CommandLineProgram;
 import picard.sam.DuplicationMetrics;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * This class is an extension of AbstractMarkDuplicatesCommandLineProgramTester used to test MarkDuplicatesWithMateCigar with SAM files generated on the fly.
@@ -77,7 +80,7 @@ public void test() throws IOException {
 
     @AfterClass
     public void afterTest() {
-        TestUtil.recursiveDelete(getOutputDir());
+        IOUtil.recursiveDelete(getOutputDir().toPath());
     }
 
 }

src/test/java/picard/sam/markduplicates/MarkDuplicatesTagRepresentativeReadIndexTester.java

@@ -29,6 +29,7 @@
 import htsjdk.samtools.SamReaderFactory;
 import htsjdk.samtools.metrics.MetricsFile;
 import htsjdk.samtools.util.CloserUtil;
+import htsjdk.samtools.util.IOUtil;
 import htsjdk.samtools.util.TestUtil;
 import org.testng.Assert;
 import picard.PicardException;
@@ -118,7 +119,7 @@ public void test() {
             Assert.assertEquals(observedMetrics.ESTIMATED_LIBRARY_SIZE, expectedMetrics.ESTIMATED_LIBRARY_SIZE, "ESTIMATED_LIBRARY_SIZE does not match expected");
             Assert.assertEquals(observedMetrics.SECONDARY_OR_SUPPLEMENTARY_RDS, expectedMetrics.SECONDARY_OR_SUPPLEMENTARY_RDS, "SECONDARY_OR_SUPPLEMENTARY_RDS does not match expected");
         } finally {
-            TestUtil.recursiveDelete(getOutputDir());
+            IOUtil.recursiveDelete(getOutputDir().toPath());
         }
     }
 

src/test/java/picard/sam/markduplicates/MarkDuplicatesTest.java

@@ -139,7 +139,7 @@ public void pgRecordChainingTest(final boolean suppressPg,
             }
 
         } finally {
-            TestUtil.recursiveDelete(outputDir);
+            IOUtil.recursiveDelete(outputDir.toPath());
         }
     }
 
@@ -195,7 +195,7 @@ public void testOpticalDuplicateDetection(final File sam, final long expectedNum
         markDuplicates.PROGRAM_RECORD_ID = null;
         Assert.assertEquals(markDuplicates.doWork(), 0);
         Assert.assertEquals(markDuplicates.numOpticalDuplicates(), expectedNumOpticalDuplicates);
-        TestUtil.recursiveDelete(outputDir);
+        IOUtil.recursiveDelete(outputDir.toPath());
 
     }
 

src/test/java/picard/sam/testers/CleanSamTester.java

@@ -9,6 +9,7 @@
 import htsjdk.samtools.SamReaderFactory;
 import htsjdk.samtools.ValidationStringency;
 import htsjdk.samtools.util.CloserUtil;
+import htsjdk.samtools.util.IOUtil;
 import htsjdk.samtools.util.TestUtil;
 import org.testng.Assert;
 import picard.sam.CleanSam;
@@ -58,7 +59,7 @@ protected void test() {
 
             Assert.assertTrue(validated, "ValidateSamFile failed");
         } finally {
-            TestUtil.recursiveDelete(getOutputDir());
+            IOUtil.recursiveDelete(getOutputDir().toPath());
         }
     }
 }

src/test/java/picard/util/IntervalListToolsTest.java

@@ -172,7 +172,7 @@ public static Iterator<Object[]> testScatterTestcases() {
 
     @AfterTest
     void deleteTempDirs() {
-        dirsToDelete.forEach(TestUtil::recursiveDelete);
+        dirsToDelete.forEach(file -> IOUtil.recursiveDelete(file.toPath()));
     }
 
     @Test(dataProvider = "testScatterTestcases")

src/test/java/picard/vcf/MendelianViolations/FindMendelianViolationsTest.java

@@ -128,7 +128,7 @@ public void testFindMedelianViolations() throws IOException {
         Assert.assertEquals(grepMv(violationsFAKE, Haploid_Other.name()), mv.NUM_HAPLOID_OTHER);
         Assert.assertEquals(grepMv(violationsFAKE, Other.name()), mv.NUM_OTHER);
 
-        TestUtil.recursiveDelete(directoryForViolations);
+        IOUtil.recursiveDelete(directoryForViolations.toPath());
     }
 
     /** returns the number of lines in the file that contain a regular expression (decorated with "MV=" and

testdata/picard/sam/SamFormatConverterTest/basicallyEmpty.fasta

@@ -0,0 +1,2 @@
+>chrNull
+NNNNNNN