Add new GeneBlacklistFilter (issue #457)

Clean-up uses of ImmutableList, Collectors.toImmutableList() and instanceOf casts to use Java 16+ features

exomiser-core/CHANGELOG.md

@@ -2,11 +2,31 @@
 
 ## 14.0.0 2023-MM-dd
 
+This release **requires data version >= 2210** and **Java version >= 17** (the most recent LTS release).
+
+- Added new GeneBlacklistFilter [#457](https://github.com/exomiser/Exomiser/issues/457)
 - Enabled independent update of ClinVar data [#501](https://github.com/exomiser/Exomiser/issues/501)
 - Added new `ClinVarDao` and `ClinVarWhiteListReader` to take advantage of the independently upgradeable ClinVar data files.
 - The `VariantWhiteList` is now dynamically loaded from the ClinVar data provided in the clinvar.mv.db file
 - `VariantDataServiceImpl` now requires a `ClinVarDao`
+- Fix for issue [#531](https://github.com/exomiser/Exomiser/issues/531) where the `priorityScoreFilter` and `regulatoryFeatureFilter` pass/fail counts were not displayed in the HTML.
+- Fix for issue [#534](https://github.com/exomiser/Exomiser/issues/534) where variant frequency and/or pathogenicity annotations are missing in certain run configurations.
 
+New APIs:
+- New `AnalysisDurationFormatter`
+- New `FilterResultsCounter`
+- New `FilterResultCount` data class
+- New `AnalysisResults.filterResultCounts` field
+- New `FilterRunner.filterCounts()` and `FilterRunner.logFilterResult()` methods
+- New `Filterable.failedFilter()` method
+- New `AlleleData` class to encapsulate building AlleleProto.Frequency and AlleleProto.PathogenicityScore instances
+- Added new `ClinVarDao` and `ClinVarWhiteListReader` to take advantage of the independently upgradeable ClinVar data files.
+- 
+API breaking changes:
+
+
+Other changes:
+- Updated Spring Boot to version 3.1.0
 
 ## 13.3.0 2023-10-17
 

exomiser-core/src/main/java/org/monarchinitiative/exomiser/core/analysis/Analysis.java

@@ -30,7 +30,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Sets;
 import de.charite.compbio.jannovar.mendel.SubModeOfInheritance;
 import org.monarchinitiative.exomiser.core.analysis.util.InheritanceModeOptions;
@@ -75,7 +74,7 @@ private Analysis(Builder builder) {
         this.inheritanceModeOptions = builder.inheritanceModeOptions;
         this.frequencySources = Sets.immutableEnumSet(builder.frequencySources);
         this.pathogenicitySources = Sets.immutableEnumSet(builder.pathogenicitySources);
-        this.analysisSteps = ImmutableList.copyOf(builder.analysisSteps);
+        this.analysisSteps = List.copyOf(builder.analysisSteps);
     }
 
     public AnalysisMode getAnalysisMode() {
@@ -101,8 +100,7 @@ public List<AnalysisStep> getAnalysisSteps() {
     @JsonIgnore
     public PriorityType getMainPrioritiserType() {
         for (AnalysisStep analysisStep : analysisSteps) {
-            if (analysisStep instanceof Prioritiser) {
-                Prioritiser<? extends PriorityResult> prioritiser = (Prioritiser<? extends PriorityResult>) analysisStep;
+            if (analysisStep instanceof Prioritiser<? extends PriorityResult> prioritiser) {
                 //OMIM, if combined with other prioritisers isn't the main one.
                 if (prioritiser.getPriorityType() != PriorityType.OMIM_PRIORITY) {
                     return prioritiser.getPriorityType();

exomiser-core/src/main/java/org/monarchinitiative/exomiser/core/analysis/AnalysisBuilder.java

@@ -161,6 +161,11 @@ public AnalysisBuilder addFrequencyFilter(float cutOff) {
         analysisSteps.add(new FrequencyFilter(cutOff));
         return this;
     }
+    @Override
+    public AnalysisBuilder addGeneBlacklistFilter() {
+        analysisSteps.add(new GeneBlacklistFilter());
+        return this;
+    }
 
     /**
      * Add a frequency filter using the maximum frequency for any defined mode of inheritance as the cut-off. Calling this

exomiser-core/src/main/java/org/monarchinitiative/exomiser/core/analysis/AnalysisFactory.java

@@ -63,14 +63,11 @@ public AnalysisRunner getAnalysisRunner(GenomeAssembly genomeAssembly, AnalysisM
         // below are package-private.
         GenomeAnalysisService genomeAnalysisService = genomeAnalysisServiceProvider.get(genomeAssembly);
 
-        switch (analysisMode) {
-            case FULL:
-                return new SimpleAnalysisRunner(genomeAnalysisService);
-            case PASS_ONLY:
-            default:
-                //this guy takes up the least RAM
-                return new PassOnlyAnalysisRunner(genomeAnalysisService);
-        }
+        return switch (analysisMode) {
+            case FULL -> new SimpleAnalysisRunner(genomeAnalysisService);
+            // PASS_ONLY takes up the least RAM
+            case PASS_ONLY -> new PassOnlyAnalysisRunner(genomeAnalysisService);
+        };
     }
 
     public AnalysisBuilder getAnalysisBuilder() {

exomiser-core/src/main/java/org/monarchinitiative/exomiser/core/analysis/AnalysisParser.java

@@ -20,7 +20,6 @@
 
 package org.monarchinitiative.exomiser.core.analysis;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Sets;
 import de.charite.compbio.jannovar.annotation.VariantEffect;
 import de.charite.compbio.jannovar.mendel.ModeOfInheritance;
@@ -62,7 +61,6 @@
 import java.util.Map.Entry;
 
 import static java.nio.file.Files.newInputStream;
-import static java.util.stream.Collectors.toList;
 
 /**
  * @since 7.0.0
@@ -363,8 +361,7 @@ private SubModeOfInheritance parseValueOfSubInheritanceMode(String value) {
             try {
                 return SubModeOfInheritance.valueOf(value);
             } catch (IllegalArgumentException e) {
-                List<SubModeOfInheritance> permitted = Arrays.stream(SubModeOfInheritance.values())
-                        .collect(toList());
+                List<SubModeOfInheritance> permitted = List.of(SubModeOfInheritance.values());
                 throw new AnalysisParserException(String.format("'%s' is not a valid mode of inheritance. Use one of: %s", value, permitted));
             }
         }
@@ -373,8 +370,7 @@ private ModeOfInheritance parseValueOfInheritanceMode(String value) {
             try {
                 return ModeOfInheritance.valueOf(value);
             } catch (IllegalArgumentException e) {
-                List<ModeOfInheritance> permitted = Arrays.stream(ModeOfInheritance.values())
-                        .collect(toList());
+                List<ModeOfInheritance> permitted = List.of(ModeOfInheritance.values());
                 throw new AnalysisParserException(String.format("'%s' is not a valid mode of inheritance. Use one of: %s", value, permitted));
             }
         }
@@ -434,6 +430,8 @@ private AnalysisBuilder addAnalysisStep(Entry<String, Map> entry, Map analysisMa
                     return makePriorityScoreFilter(analysisStepOptions, analysisBuilder);
                 case "regulatoryFeatureFilter":
                     return analysisBuilder.addRegulatoryFeatureFilter();
+                case "geneBlacklistFilter":
+                    return makeGeneBlacklistFilter(analysisStepOptions, analysisBuilder);
                 case "omimPrioritiser":
                     return analysisBuilder.addOmimPrioritiser();
                 case "hiPhivePrioritiser":
@@ -460,7 +458,7 @@ private AnalysisBuilder makeIntervalFilter(Map<String, Object> options, Analysis
         private List<ChromosomalRegion> parseIntervalFilterOptions(Map<String, Object> options){
             if (options.containsKey("interval")) {
                 String interval = (String) options.get("interval");
-                return ImmutableList.of(GeneticInterval.parseString(interval));
+                return List.of(GeneticInterval.parseString(interval));
             }
             if (options.containsKey("intervals")) {
                 List<String> intervalStrings = (List<String>) options.get("intervals");
@@ -470,7 +468,7 @@ private List<ChromosomalRegion> parseIntervalFilterOptions(Map<String, Object> o
             }
             if (options.containsKey("bed")) {
                 String bedPath = (String) options.get("bed");
-                return BedFiles.readChromosomalRegions(Paths.get(bedPath)).collect(toList());
+                return BedFiles.readChromosomalRegions(Paths.get(bedPath)).toList();
             }
             throw new AnalysisParserException("Interval filter requires a valid genetic interval e.g. {interval: 'chr10:122892600-122892700'} or bed file path {bed: /data/intervals.bed}", options);
         }
@@ -568,6 +566,14 @@ private Set<FrequencySource> parseFrequencySources(Map<String, List<String>> opt
             List<FrequencySource> sources = new ArrayList<>();
             for (String source : frequencySources) {
                 try {
+                    // legacy long-form ESP names
+                    if (source.startsWith("ESP")) {
+                        source = switch (source) {
+                            case "ESP_AFRICAN_AMERICAN" -> "ESP_AA";
+                            case "ESP_EUROPEAN_AMERICAN" -> "ESP_EA";
+                            default -> source;
+                        };
+                    }
                     FrequencySource frequencySource = FrequencySource.valueOf(source);
                     sources.add(frequencySource);
                 } catch (IllegalArgumentException ex) {
@@ -633,6 +639,10 @@ private AnalysisBuilder makeInheritanceFilter(InheritanceModeOptions inheritance
             return analysisBuilder.addInheritanceFilter();
         }
 
+        private AnalysisBuilder makeGeneBlacklistFilter(Map<String, String> options, AnalysisBuilder analysisBuilder) {
+            return analysisBuilder.addGeneBlacklistFilter();
+        }
+
         private AnalysisBuilder makeHiPhivePrioritiser(Map<String, String> options, AnalysisBuilder analysisBuilder) {
             HiPhiveOptions hiPhiveOptions = makeHiPhiveOptions(options);
             logger.debug("Made {}", hiPhiveOptions);

exomiser-core/src/main/java/org/monarchinitiative/exomiser/core/analysis/AnalysisPresetBuilder.java

@@ -115,7 +115,7 @@ protected Analysis buildGenomePreset() {
                 .analysisMode(AnalysisMode.PASS_ONLY)
                 .inheritanceModes(DEFAULT_INHERITANCE_MODE_OPTIONS)
                 .frequencySources(DEFAULT_FREQUENCY_SOURCES)
-                .pathogenicitySources(EnumSet.of(REVEL, MVP, REMM))
+                .pathogenicitySources(EnumSet.of(REVEL, MVP, REMM, SPLICE_AI))
                 .addHiPhivePrioritiser(HI_PHIVE_OPTIONS)
                 .addPriorityScoreFilter(PriorityType.HIPHIVE_PRIORITY, 0.5f)// will remove a lot of the weak PPI hits
                 .addFailedVariantFilter()
@@ -132,7 +132,7 @@ protected Analysis buildExomePreset() {
                 .analysisMode(AnalysisMode.PASS_ONLY)
                 .inheritanceModes(DEFAULT_INHERITANCE_MODE_OPTIONS)
                 .frequencySources(DEFAULT_FREQUENCY_SOURCES)
-                .pathogenicitySources(EnumSet.of(REVEL, MVP))
+                .pathogenicitySources(EnumSet.of(REVEL, MVP, SPLICE_AI))
                 .addVariantEffectFilter(EnumSet.of(
                         VariantEffect.FIVE_PRIME_UTR_EXON_VARIANT,
                         VariantEffect.FIVE_PRIME_UTR_INTRON_VARIANT,

exomiser-core/src/main/java/org/monarchinitiative/exomiser/core/analysis/AnalysisProtoBuilder.java

@@ -176,6 +176,12 @@ public AnalysisProtoBuilder addFrequencyFilter(float cutOff) {
         return this;
     }
 
+    @Override
+    public AnalysisProtoBuilder addGeneBlacklistFilter() {
+        builder.addSteps(stepBuilder().setGeneBlacklistFilter(FiltersProto.GeneBlacklistFilter.newBuilder()));
+        return this;
+    }
+
     /**
      * Add a frequency filter using the maximum frequency for any defined mode of inheritance as the cut-off. Calling this
      * method requires that the {@code inheritanceModes} method has already been called and supplied with a non-empty

exomiser-core/src/main/java/org/monarchinitiative/exomiser/core/analysis/AnalysisProtoConverter.java

@@ -39,9 +39,9 @@ public AnalysisProto.Analysis toProto(Analysis analysis) {
         return AnalysisProto.Analysis.newBuilder()
                 .setAnalysisMode(analysis.getAnalysisMode() == AnalysisMode.PASS_ONLY ? AnalysisProto.AnalysisMode.PASS_ONLY : AnalysisProto.AnalysisMode.FULL)
                 .putAllInheritanceModes(analysis.getInheritanceModeOptions().getMaxFreqs().entrySet().stream().collect(Collectors.toMap(subModeOfInheritanceFloatEntry -> subModeOfInheritanceFloatEntry.getKey().toString(), Map.Entry::getValue)))
-                .addAllFrequencySources(analysis.getFrequencySources().stream().map(Objects::toString).collect(Collectors.toList()))
-                .addAllPathogenicitySources(analysis.getPathogenicitySources().stream().map(Objects::toString).collect(Collectors.toList()))
-                .addAllSteps(analysis.getAnalysisSteps().stream().map(analysisStepToProto()).filter(Objects::nonNull).collect(Collectors.toList()))
+                .addAllFrequencySources(analysis.getFrequencySources().stream().map(Objects::toString).toList())
+                .addAllPathogenicitySources(analysis.getPathogenicitySources().stream().map(Objects::toString).toList())
+                .addAllSteps(analysis.getAnalysisSteps().stream().map(analysisStepToProto()).filter(Objects::nonNull).toList())
                 .build();
     }
 
@@ -59,21 +59,18 @@ private Function<AnalysisStep, AnalysisProto.AnalysisStep> analysisStepToProto()
 
     private AnalysisProto.AnalysisStep buildFilterProto(AnalysisStep analysisStep) {
         AnalysisProto.AnalysisStep.Builder stepBuilder = AnalysisProto.AnalysisStep.newBuilder();
-        if (analysisStep instanceof VariantEffectFilter) {
-            VariantEffectFilter variantEffectFilter = (VariantEffectFilter) analysisStep;
+        if (analysisStep instanceof VariantEffectFilter variantEffectFilter) {
             return stepBuilder
                     .setVariantEffectFilter(FiltersProto.VariantEffectFilter.newBuilder()
-                            .addAllRemove(variantEffectFilter.getOffTargetVariantTypes().stream().map(Objects::toString).collect(Collectors.toList())))
+                            .addAllRemove(variantEffectFilter.getOffTargetVariantTypes().stream().map(Objects::toString).toList()))
                     .build();
         }
-        if (analysisStep instanceof FrequencyFilter) {
-            FrequencyFilter frequencyFilter = (FrequencyFilter) analysisStep;
+        if (analysisStep instanceof FrequencyFilter frequencyFilter) {
             return stepBuilder
                     .setFrequencyFilter(FiltersProto.FrequencyFilter.newBuilder().setMaxFrequency(frequencyFilter.getMaxFreq()))
                     .build();
         }
-        if (analysisStep instanceof PathogenicityFilter) {
-            PathogenicityFilter pathogenicityFilter = (PathogenicityFilter) analysisStep;
+        if (analysisStep instanceof PathogenicityFilter pathogenicityFilter) {
             return stepBuilder
                     .setPathogenicityFilter(FiltersProto.PathogenicityFilter.newBuilder().setKeepNonPathogenic(pathogenicityFilter.keepNonPathogenic()))
                     .build();
@@ -88,25 +85,29 @@ private AnalysisProto.AnalysisStep buildFilterProto(AnalysisStep analysisStep) {
                     .setKnownVariantFilter(FiltersProto.KnownVariantFilter.newBuilder())
                     .build();
         }
-        if (analysisStep instanceof QualityFilter) {
-            QualityFilter qualityFilter = (QualityFilter) analysisStep;
+        if (analysisStep instanceof QualityFilter qualityFilter) {
             return stepBuilder
                     .setQualityFilter(FiltersProto.QualityFilter.newBuilder().setMinQuality((float) qualityFilter.getMimimumQualityThreshold()))
                     .build();
         }
-        if (analysisStep instanceof IntervalFilter) {
-            IntervalFilter intervalFilter = (IntervalFilter) analysisStep;
+        if (analysisStep instanceof IntervalFilter intervalFilter) {
             return stepBuilder
                     .setIntervalFilter(FiltersProto.IntervalFilter.newBuilder()
-                            .addAllIntervals(intervalFilter.getChromosomalRegions().stream().map(region -> region.contigId() + ":" + region.start() + "-" + region.end()).collect(Collectors.toList())))
+                            .addAllIntervals(intervalFilter.getChromosomalRegions().stream().map(region -> region.contigId() + ":" + region.start() + "-" + region.end()).toList()))
                     .build();
         }
-        if (analysisStep instanceof GeneSymbolFilter) {
-            GeneSymbolFilter geneSymbolFilter = (GeneSymbolFilter) analysisStep;
+        if (analysisStep instanceof GeneSymbolFilter geneSymbolFilter) {
             return stepBuilder
                     .setGenePanelFilter(FiltersProto.GenePanelFilter.newBuilder().addAllGeneSymbols(geneSymbolFilter.getGeneSymbols()))
                     .build();
         }
+
+        if (analysisStep instanceof GeneBlacklistFilter geneBlacklistFilter) {
+           return stepBuilder
+                   .setGeneBlacklistFilter(FiltersProto.GeneBlacklistFilter.newBuilder())
+                   .build();
+        }
+
         if (analysisStep instanceof InheritanceFilter) {
             return stepBuilder
                     .setInheritanceFilter(FiltersProto.InheritanceFilter.newBuilder().build())
@@ -115,8 +116,7 @@ private AnalysisProto.AnalysisStep buildFilterProto(AnalysisStep analysisStep) {
         if (analysisStep instanceof RegulatoryFeatureFilter) {
             return stepBuilder.setRegulatoryFeatureFilter(FiltersProto.RegulatoryFeatureFilter.newBuilder()).build();
         }
-        if (analysisStep instanceof PriorityScoreFilter) {
-            PriorityScoreFilter priorityScoreFilter = (PriorityScoreFilter) analysisStep;
+        if (analysisStep instanceof PriorityScoreFilter priorityScoreFilter) {
             return stepBuilder
                     .setPriorityScoreFilter(FiltersProto.PriorityScoreFilter.newBuilder()
                             .setMinPriorityScore((float) priorityScoreFilter.getMinPriorityScore())
@@ -133,8 +133,7 @@ private AnalysisProto.AnalysisStep buildPrioritiserProto(AnalysisStep analysisSt
                     .setOmimPrioritiser(PrioritisersProto.OmimPrioritiser.newBuilder())
                     .build();
         }
-        if (analysisStep instanceof HiPhivePriority) {
-            HiPhivePriority hiPhivePriority = (HiPhivePriority) analysisStep;
+        if (analysisStep instanceof HiPhivePriority hiPhivePriority) {
             HiPhiveOptions hiPhiveOptions = hiPhivePriority.getOptions();
             return stepBuilder
                     .setHiPhivePrioritiser(PrioritisersProto.HiPhivePrioritiser.newBuilder()
@@ -153,8 +152,7 @@ private AnalysisProto.AnalysisStep buildPrioritiserProto(AnalysisStep analysisSt
                     .setPhivePrioritiser(PrioritisersProto.PhivePrioritiser.newBuilder())
                     .build();
         }
-        if (analysisStep instanceof ExomeWalkerPriority) {
-            ExomeWalkerPriority exomeWalkerPriority = (ExomeWalkerPriority) analysisStep;
+        if (analysisStep instanceof ExomeWalkerPriority exomeWalkerPriority) {
             return stepBuilder
                     .setExomeWalkerPrioritiser(PrioritisersProto.ExomeWalkerPrioritiser.newBuilder().addAllSeedGeneIds(exomeWalkerPriority.getSeedGenes()))
                     .build();

exomiser-core/src/main/java/org/monarchinitiative/exomiser/core/analysis/AnalysisSampleValidator.java

@@ -92,15 +92,12 @@ private static boolean isPathogenicityDependent(AnalysisStep analysisStep) {
     }
 
     private static boolean isHpoDependent(AnalysisStep analysisStep) {
-        if (analysisStep instanceof Prioritiser) {
-            Prioritiser<? extends PriorityResult> prioritiser = (Prioritiser<? extends PriorityResult>) analysisStep;
+        if (analysisStep instanceof Prioritiser<? extends PriorityResult> prioritiser) {
             PriorityType priorityType = prioritiser.getPriorityType();
-            switch (priorityType) {
-                case HIPHIVE_PRIORITY:
-                case PHENIX_PRIORITY:
-                case PHIVE_PRIORITY:
-                    return true;
-            }
+            return switch (priorityType) {
+                case HIPHIVE_PRIORITY, PHENIX_PRIORITY, PHIVE_PRIORITY -> true;
+                case EXOMEWALKER_PRIORITY, OMIM_PRIORITY, NONE -> false;
+            };
         }
         return false;
     }