From ed21e64cdb7b7ae796f9747af8b367936b868d11 Mon Sep 17 00:00:00 2001 From: Chris Norman Date: Mon, 18 Jun 2018 10:54:46 -0400 Subject: [PATCH] Integrate VariantWalkerBase with CountingVariantFilter. --- .../engine/TwoPassVariantWalker.java | 8 ++-- .../hellbender/engine/VariantWalkerBase.java | 10 +++-- .../engine/filters/VariantFilterLibrary.java | 30 ++++++++++++++- .../tools/funcotator/Funcotator.java | 7 +++- .../tools/funcotator/FuncotatorEngine.java | 12 +++--- .../walkers/variantutils/SelectVariants.java | 15 +++----- .../tools/walkers/vqsr/CNNScoreVariants.java | 14 +++---- .../engine/filters/VariantFilterUnitTest.java | 37 +++++++++++++++++++ 8 files changed, 100 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/broadinstitute/hellbender/engine/TwoPassVariantWalker.java b/src/main/java/org/broadinstitute/hellbender/engine/TwoPassVariantWalker.java index b0ff611956e..460b9a40186 100644 --- a/src/main/java/org/broadinstitute/hellbender/engine/TwoPassVariantWalker.java +++ b/src/main/java/org/broadinstitute/hellbender/engine/TwoPassVariantWalker.java @@ -2,6 +2,7 @@ import htsjdk.variant.variantcontext.VariantContext; import org.broadinstitute.hellbender.engine.filters.CountingReadFilter; +import org.broadinstitute.hellbender.engine.filters.CountingVariantFilter; import org.broadinstitute.hellbender.engine.filters.VariantFilter; import org.broadinstitute.hellbender.utils.SimpleInterval; @@ -18,12 +19,12 @@ public abstract class TwoPassVariantWalker extends VariantWalker { */ @Override public void traverse(){ - final VariantFilter variantContextFilter = makeVariantFilter(); + final CountingVariantFilter countingVariantFilter = makeVariantFilter(); final CountingReadFilter readFilter = makeReadFilter(); // First pass through the variants logger.info("Starting first pass through the variants"); - traverseVariants(variantContextFilter, readFilter, this::firstPassApply); + traverseVariants(countingVariantFilter, readFilter, this::firstPassApply); logger.info("Finished first pass through the variants"); // Process the data accumulated during the first pass @@ -31,8 +32,9 @@ public void traverse(){ // Second pass logger.info("Starting second pass through the variants"); - traverseVariants(variantContextFilter, readFilter, this::secondPassApply); + traverseVariants(countingVariantFilter, readFilter, this::secondPassApply); + logger.info(countingVariantFilter.getSummaryLine()); logger.info(readFilter.getSummaryLine()); } diff --git a/src/main/java/org/broadinstitute/hellbender/engine/VariantWalkerBase.java b/src/main/java/org/broadinstitute/hellbender/engine/VariantWalkerBase.java index b42e670a62e..4e3460f743b 100644 --- a/src/main/java/org/broadinstitute/hellbender/engine/VariantWalkerBase.java +++ b/src/main/java/org/broadinstitute/hellbender/engine/VariantWalkerBase.java @@ -4,6 +4,7 @@ import htsjdk.variant.variantcontext.VariantContext; import htsjdk.variant.vcf.VCFHeader; import org.broadinstitute.hellbender.engine.filters.CountingReadFilter; +import org.broadinstitute.hellbender.engine.filters.CountingVariantFilter; import org.broadinstitute.hellbender.engine.filters.VariantFilter; import org.broadinstitute.hellbender.engine.filters.VariantFilterLibrary; import org.broadinstitute.hellbender.transformers.VariantTransformer; @@ -143,9 +144,10 @@ protected Stream getTransformedVariantStream(final VariantFilter */ @Override public void traverse() { + final CountingVariantFilter countingVariantfilter = makeVariantFilter(); final CountingReadFilter readFilter = makeReadFilter(); // Process each variant in the input stream. - getTransformedVariantStream( makeVariantFilter() ) + getTransformedVariantStream(countingVariantfilter) .forEach(variant -> { final SimpleInterval variantInterval = new SimpleInterval(variant); apply(variant, @@ -155,6 +157,8 @@ public void traverse() { progressMeter.update(variantInterval); }); + + logger.info(countingVariantfilter.getSummaryLine()); } /** @@ -166,8 +170,8 @@ public void traverse() { * Subclasses can extend to provide own filters (ie override and call super). * Multiple filters can be composed by using {@link VariantFilter} composition methods. */ - protected VariantFilter makeVariantFilter() { - return VariantFilterLibrary.ALLOW_ALL_VARIANTS; + protected CountingVariantFilter makeVariantFilter() { + return new CountingVariantFilter(VariantFilterLibrary.ALLOW_ALL_VARIANTS); } /** diff --git a/src/main/java/org/broadinstitute/hellbender/engine/filters/VariantFilterLibrary.java b/src/main/java/org/broadinstitute/hellbender/engine/filters/VariantFilterLibrary.java index 1aedcdfe599..da250a28708 100644 --- a/src/main/java/org/broadinstitute/hellbender/engine/filters/VariantFilterLibrary.java +++ b/src/main/java/org/broadinstitute/hellbender/engine/filters/VariantFilterLibrary.java @@ -1,9 +1,35 @@ package org.broadinstitute.hellbender.engine.filters; +import htsjdk.variant.variantcontext.VariantContext; + /** * Collects common variant filters. */ public final class VariantFilterLibrary { - public static VariantFilter ALLOW_ALL_VARIANTS = variant -> true; - public static VariantFilter NOT_SV_OR_SYMBOLIC = variant -> !variant.isSymbolicOrSV(); + public static VariantFilter ALLOW_ALL_VARIANTS = new AllowAllVariantsVariantFilter(); + public static VariantFilter NOT_SV_OR_SYMBOLIC = new NotSymbolicOrSVVariantFilter(); + public static VariantFilter PASSES_FILTERS = new PassesFiltersVariantFilter(); + + /** Do not filter out any variants. */ + public static class AllowAllVariantsVariantFilter implements VariantFilter { + private static final long serialVersionUID = 1L; + @Override public boolean test(final VariantContext variant) { return true; } + } + + /** Filter out any variants that are symbolic or SV. */ + public static class NotSymbolicOrSVVariantFilter implements VariantFilter { + private static final long serialVersionUID = 1L; + @Override public boolean test(final VariantContext variant) { + return !variant.isSymbolicOrSV(); + } + } + + /** Filter out any variants that fail (variant-level) filters. */ + public static class PassesFiltersVariantFilter implements VariantFilter { + private static final long serialVersionUID = 1L; + @Override public boolean test(final VariantContext variant) { + return !variant.isFiltered(); + } + } + } diff --git a/src/main/java/org/broadinstitute/hellbender/tools/funcotator/Funcotator.java b/src/main/java/org/broadinstitute/hellbender/tools/funcotator/Funcotator.java index d591da1f2e9..839d1d074b3 100644 --- a/src/main/java/org/broadinstitute/hellbender/tools/funcotator/Funcotator.java +++ b/src/main/java/org/broadinstitute/hellbender/tools/funcotator/Funcotator.java @@ -12,6 +12,9 @@ import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions; import org.broadinstitute.hellbender.engine.*; import org.broadinstitute.hellbender.engine.filters.VariantFilter; +import org.broadinstitute.hellbender.engine.filters.CountingVariantFilter; +import org.broadinstitute.hellbender.engine.filters.VariantFilterLibrary; +import org.broadinstitute.hellbender.exceptions.GATKException; import org.broadinstitute.hellbender.exceptions.UserException; import org.broadinstitute.hellbender.tools.funcotator.dataSources.DataSourceUtils; import org.broadinstitute.hellbender.tools.funcotator.metadata.VcfFuncotationMetadata; @@ -317,8 +320,8 @@ private void checkReferenceDictionaryIsSupersetOfVariantDictionary() { } @Override - protected VariantFilter makeVariantFilter() { - return funcotatorEngine.makeVariantFilter(); + protected CountingVariantFilter makeVariantFilter() { + return new CountingVariantFilter(funcotatorEngine.makeVariantFilter()); } @Override diff --git a/src/main/java/org/broadinstitute/hellbender/tools/funcotator/FuncotatorEngine.java b/src/main/java/org/broadinstitute/hellbender/tools/funcotator/FuncotatorEngine.java index 67d2b4f30fe..e0d08bb7482 100644 --- a/src/main/java/org/broadinstitute/hellbender/tools/funcotator/FuncotatorEngine.java +++ b/src/main/java/org/broadinstitute/hellbender/tools/funcotator/FuncotatorEngine.java @@ -12,6 +12,7 @@ import org.broadinstitute.hellbender.engine.GATKTool; import org.broadinstitute.hellbender.engine.ReferenceContext; import org.broadinstitute.hellbender.engine.filters.VariantFilter; +import org.broadinstitute.hellbender.engine.filters.VariantFilterLibrary; import org.broadinstitute.hellbender.exceptions.GATKException; import org.broadinstitute.hellbender.exceptions.UserException; import org.broadinstitute.hellbender.tools.funcotator.dataSources.DataSourceUtils; @@ -216,13 +217,10 @@ public OutputRenderer createOutputRenderer(final LinkedHashMap a * @return A {@link VariantFilter} that will ignore any variants that have been filtered (if the user requested that the filter is turned on). Otherwise returns a no-op filter. */ public VariantFilter makeVariantFilter() { - return variant -> { - // Ignore variants that have been filtered if the user requests it: - if ( funcotatorArgs.removeFilteredVariants && variant.isFiltered() ) { - return false; - } - return true; - }; + // Ignore variants that have been filtered if the user requests it: + return funcotatorArgs.removeFilteredVariants ? + VariantFilterLibrary.PASSES_FILTERS : + VariantFilterLibrary.ALLOW_ALL_VARIANTS; } /** diff --git a/src/main/java/org/broadinstitute/hellbender/tools/walkers/variantutils/SelectVariants.java b/src/main/java/org/broadinstitute/hellbender/tools/walkers/variantutils/SelectVariants.java index cf262d6d16d..0eba6e3e598 100644 --- a/src/main/java/org/broadinstitute/hellbender/tools/walkers/variantutils/SelectVariants.java +++ b/src/main/java/org/broadinstitute/hellbender/tools/walkers/variantutils/SelectVariants.java @@ -22,15 +22,12 @@ import org.broadinstitute.barclay.argparser.Hidden; import org.broadinstitute.barclay.help.DocumentedFeature; import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions; +import org.broadinstitute.hellbender.engine.filters.*; import picard.cmdline.programgroups.VariantManipulationProgramGroup; import org.broadinstitute.hellbender.engine.FeatureInput; import org.broadinstitute.hellbender.engine.FeatureContext; import org.broadinstitute.hellbender.engine.ReadsContext; import org.broadinstitute.hellbender.engine.ReferenceContext; -import org.broadinstitute.hellbender.engine.filters.VariantFilter; -import org.broadinstitute.hellbender.engine.filters.VariantIDsVariantFilter; -import org.broadinstitute.hellbender.engine.filters.VariantFilterLibrary; -import org.broadinstitute.hellbender.engine.filters.VariantTypesVariantFilter; import org.broadinstitute.hellbender.engine.VariantWalker; import org.broadinstitute.hellbender.exceptions.UserException; import org.broadinstitute.hellbender.tools.walkers.annotator.ChromosomeCounts; @@ -658,19 +655,19 @@ public void closeTool() { * Create filters for variant types, ids, and genomic intervals. */ @Override - protected VariantFilter makeVariantFilter() { - VariantFilter compositeFilter = VariantFilterLibrary.ALLOW_ALL_VARIANTS; + protected CountingVariantFilter makeVariantFilter() { + CountingVariantFilter compositeFilter = new CountingVariantFilter(VariantFilterLibrary.ALLOW_ALL_VARIANTS); if (!selectedTypes.isEmpty()) { - compositeFilter = compositeFilter.and(new VariantTypesVariantFilter(selectedTypes)); + compositeFilter = compositeFilter.and(new CountingVariantFilter(new VariantTypesVariantFilter(selectedTypes))); } if (rsIDsToKeep != null && !rsIDsToKeep.isEmpty()) { - compositeFilter = compositeFilter.and(new VariantIDsVariantFilter(rsIDsToKeep)); + compositeFilter = compositeFilter.and(new CountingVariantFilter(new VariantIDsVariantFilter(rsIDsToKeep))); } if (rsIDsToRemove != null && !rsIDsToRemove.isEmpty()) { - compositeFilter = compositeFilter.and(new VariantIDsVariantFilter(rsIDsToRemove).negate()); + compositeFilter = compositeFilter.and(new CountingVariantFilter(new VariantIDsVariantFilter(rsIDsToRemove).negate())); } return compositeFilter; diff --git a/src/main/java/org/broadinstitute/hellbender/tools/walkers/vqsr/CNNScoreVariants.java b/src/main/java/org/broadinstitute/hellbender/tools/walkers/vqsr/CNNScoreVariants.java index b4641c6fb5c..be4e0c34ebb 100644 --- a/src/main/java/org/broadinstitute/hellbender/tools/walkers/vqsr/CNNScoreVariants.java +++ b/src/main/java/org/broadinstitute/hellbender/tools/walkers/vqsr/CNNScoreVariants.java @@ -14,7 +14,7 @@ import org.broadinstitute.hellbender.exceptions.GATKException; import org.broadinstitute.hellbender.utils.haplotype.HaplotypeBAMWriter; import org.broadinstitute.hellbender.utils.io.IOUtils; -import org.broadinstitute.hellbender.utils.io.Resource; +import org.broadinstitute.hellbender.utils.io.Resource; import org.broadinstitute.hellbender.utils.python.StreamingPythonScriptExecutor; import org.broadinstitute.hellbender.utils.read.GATKRead; import org.broadinstitute.hellbender.utils.runtime.AsynchronousStreamWriter; @@ -207,12 +207,12 @@ public boolean requiresReference() { } @Override - protected VariantFilter makeVariantFilter(){ - if (filterSymbolicAndSV) { - return VariantFilterLibrary.NOT_SV_OR_SYMBOLIC; - } else { - return VariantFilterLibrary.ALLOW_ALL_VARIANTS; - } + protected CountingVariantFilter makeVariantFilter() { + return new CountingVariantFilter( + filterSymbolicAndSV ? + VariantFilterLibrary.NOT_SV_OR_SYMBOLIC: + VariantFilterLibrary.ALLOW_ALL_VARIANTS + ); } @Override diff --git a/src/test/java/org/broadinstitute/hellbender/engine/filters/VariantFilterUnitTest.java b/src/test/java/org/broadinstitute/hellbender/engine/filters/VariantFilterUnitTest.java index e7cd1ce6d29..48833438125 100644 --- a/src/test/java/org/broadinstitute/hellbender/engine/filters/VariantFilterUnitTest.java +++ b/src/test/java/org/broadinstitute/hellbender/engine/filters/VariantFilterUnitTest.java @@ -9,6 +9,7 @@ import org.broadinstitute.hellbender.utils.SimpleInterval; import org.broadinstitute.hellbender.GATKBaseTest; +import org.broadinstitute.hellbender.utils.Utils; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -102,4 +103,40 @@ public void testVariantTypeVariantFilter(VariantContext vc, Type[] types, boolea VariantTypesVariantFilter vtvf = new VariantTypesVariantFilter(typesSet); Assert.assertTrue(vtvf.test(vc) == expected); } + + @DataProvider + public Object[][] getNotSymbolicOrSVFilterTestVCs() { + return new Object[][]{ + {snpVC, true}, + {new VariantContextBuilder(snpVC).alleles( // SYMBOLIC + Arrays.asList( + Allele.create("A", true), + Allele.create("", false))).make(), false}, + {new VariantContextBuilder(snpVC).alleles( // SV = > 150 indel + Arrays.asList( + Allele.create("A", true), + Allele.create("A" + Utils.dupChar('G', 151), false))).make(), false}, + }; + } + + @Test(dataProvider = "getNotSymbolicOrSVFilterTestVCs") + public void testNotSVOrSymbolicVariantFilter(VariantContext vc, boolean expected) { + Assert.assertEquals(VariantFilterLibrary.NOT_SV_OR_SYMBOLIC.test(vc), expected); + } + + @DataProvider + public Object[][] getPassesFiltersFilterTestVCs() { + return new Object[][]{ + {snpVC, true}, + {new VariantContextBuilder(snpVC).unfiltered().make(), true}, + {new VariantContextBuilder(snpVC).passFilters().make(), true}, + {new VariantContextBuilder(snpVC).filter("unbelievable").make(), false}, + }; + } + + @Test(dataProvider = "getPassesFiltersFilterTestVCs") + public void testPassesFiltersVariantFilter(VariantContext vc, boolean expected) { + Assert.assertEquals(VariantFilterLibrary.PASSES_FILTERS.test(vc), expected); + } + }