feat: remove plot-bamstats (#288) (#290)

snappy_pipeline/apps/snappy_snake.py

@@ -302,12 +302,13 @@ def main(argv=None):
             "not yet exist."
         ),
     )
+    rerun_triggers_default = ["mtime", "params", "input"]
     group.add_argument(
         "--rerun-triggers",
         nargs="+",
         choices=RERUN_TRIGGERS,
-        default=RERUN_TRIGGERS,
-        help=f"Expose --rerun-triggers from snakemake and set to {RERUN_TRIGGERS} by default",
+        default=rerun_triggers_default,
+        help=f"Expose --rerun-triggers from snakemake and set to {rerun_triggers_default} by default",
     )
 
     group = parser.add_argument_group(

snappy_pipeline/workflows/ngs_mapping/__init__.py

@@ -161,11 +161,9 @@
   called by default, and are linked out into the ``output/{mapper}.{library_name}/report/bam_qc``
   directory. The file names for these reports (and their MD5s) use the following naming convention:
 
-  - ``{mapper}.{library_name}.bamstats.html``
   - ``{mapper}.{library_name}.bamstats.txt``
   - ``{mapper}.{library_name}.flagstats.txt``
   - ``{mapper}.{library_name}.idxstats.txt``
-  - ``{mapper}.{library_name}.bamstats.html.md5``
   - ``{mapper}.{library_name}.bamstats.txt.md5``
   - ``{mapper}.{library_name}.flagstats.txt.md5``
   - ``{mapper}.{library_name}.idxstats.txt.md5``
@@ -183,8 +181,6 @@
     |   |   `-- bwa.P001-N1-DNA1-WES1.bam.md5
     |   `-- report
     |       `-- bam_qc
-    |           |-- bwa.P001-N1-DNA1-WES1.bam.bamstats.html
-    |           |-- bwa.P001-N1-DNA1-WES1.bam.bamstats.html.md5
     |           |-- bwa.P001-N1-DNA1-WES1.bam.bamstats.txt
     |           |-- bwa.P001-N1-DNA1-WES1.bam.bamstats.txt.md5
     |           |-- bwa.P001-N1-DNA1-WES1.bam.flagstats.txt
@@ -462,13 +458,12 @@ def get_output_files(self, action):
         assert action in self.actions
         for ext in self.extensions:
             yield ext[1:].replace(".", "_"), self.base_path_out.format(mapper=self.name, ext=ext)
-        for ext in (".bamstats.html", ".bamstats.txt", ".flagstats.txt", ".idxstats.txt"):
+        for ext in (".bamstats.txt", ".flagstats.txt", ".idxstats.txt"):
             path = (
                 "work/{mapper}.{{library_name}}/report/bam_qc/" "{mapper}.{{library_name}}.bam{ext}"
             ).format(mapper=self.name, ext=ext)
             yield "report_" + ".".join(ext.split(".")[1:3]).replace(".", "_"), path
         for ext in (
-            ".bamstats.html.md5",
             ".bamstats.txt.md5",
             ".flagstats.txt.md5",
             ".idxstats.txt.md5",
@@ -1317,16 +1312,6 @@ def get_result_files(self):
             ),
             report=("bamstats", "flagstats", "idxstats"),
         )
-        yield from self._yield_result_files(
-            os.path.join(
-                "output", name_pattern, "report", "bam_qc", name_pattern + ".bam.bamstats.html"
-            )
-        )
-        yield from self._yield_result_files(
-            os.path.join(
-                "output", name_pattern, "report", "bam_qc", name_pattern + ".bam.bamstats.html.md5"
-            )
-        )
 
         for sheet in self.shortcut_sheets:
             for ngs_library in sheet.all_ngs_libraries:

snappy_wrappers/wrappers/bwa/environment.yaml

@@ -7,6 +7,3 @@ dependencies:
   - samblaster ==0.1.26
   - seqtk ==1.3
   - trimadap ==r11
-  - gnuplot ==5.4.5
-  - libpng ==1.6.39
-  - inline-html ==1.0.0

snappy_wrappers/wrappers/bwa/wrapper.py

@@ -260,43 +260,7 @@
 samtools flagstat {snakemake.output.bam} > {snakemake.output.report_flagstats_txt}
 samtools idxstats {snakemake.output.bam} > {snakemake.output.report_idxstats_txt}
 
-# call plot-bamstats
-mkdir $TMPDIR/bamstats.d
-plot-bamstats \
-    -p $TMPDIR/bamstats.d/ \
-    {snakemake.output.report_bamstats_txt} \
-|| true  # ignore failure
-
-# Patch inline-html if necessary.
-cat >$TMPDIR/inline-html.diff <<EOF
-diff --git a/inline_html/inline_html.py b/inline_html/inline_html.py
-index 893086c..cbef6dd 100644
---- a/inline_html/inline_html.py
-+++ b/inline_html/inline_html.py
-@@ -20,7 +20,10 @@ def resource_to_data(path, in_file):
-     mime, _ = mimetypes.guess_type(path)
-     with open(path, 'rb') as fp:
-         data = fp.read()
--        data64 = b''.join(base64.encodestring(data).splitlines())
-+        try:
-+            data64 = b''.join(base64.encodestring(data).splitlines())
-+        except AttributeError:
-+            data64 = b''.join(base64.encodebytes(data).splitlines())
-         return 'data:%s;base64,%s' % (mime, data64.decode('ascii'))
-EOF
-pushd $(python3 -c 'import inline_html; print(inline_html.__path__[0])')
-if ! grep encodebytes inline_html.py; then
-    patch -p2 <$TMPDIR/inline-html.diff
-fi
-popd
-
-# Convert HTML report into one file.
-inline-html \
-    --in-file $TMPDIR/bamstats.d/index.html \
-    --out-file {snakemake.output.report_bamstats_html}
-
 # Build MD5 files for the reports
-md5sum {snakemake.output.report_bamstats_html} > {snakemake.output.report_bamstats_html_md5}
 md5sum {snakemake.output.report_bamstats_txt} > {snakemake.output.report_bamstats_txt_md5}
 md5sum {snakemake.output.report_flagstats_txt} >{snakemake.output.report_flagstats_txt_md5}
 md5sum {snakemake.output.report_idxstats_txt} > {snakemake.output.report_idxstats_txt_md5}

snappy_wrappers/wrappers/link_in_bam/wrapper.py

@@ -62,21 +62,7 @@
 samtools flagstat {snakemake.output.bam} > {snakemake.output.report_flagstats_txt}
 samtools idxstats {snakemake.output.bam} > {snakemake.output.report_idxstats_txt}
 
-# call plot-bamstats
-mkdir $TMPDIR/bamstats.d
-plot-bamstats \
-    -p $TMPDIR/bamstats.d/ \
-    {snakemake.output.report_bamstats_txt} \
-|| true  # ignore failure
-
-# Convert HTML report into one file.
-inline-html \
-    --in-file $TMPDIR/bamstats.d/index.html \
-    --out-file {snakemake.output.report_bamstats_html} \
-|| touch {snakemake.output.report_bamstats_html}
-
 # Build MD5 files for the reports
-md5sum {snakemake.output.report_bamstats_html} > {snakemake.output.report_bamstats_html_md5}
 md5sum {snakemake.output.report_bamstats_txt} > {snakemake.output.report_bamstats_txt_md5}
 md5sum {snakemake.output.report_flagstats_txt} >{snakemake.output.report_flagstats_txt_md5}
 md5sum {snakemake.output.report_idxstats_txt} > {snakemake.output.report_idxstats_txt_md5}

snappy_wrappers/wrappers/minialign/run/wrapper.py

@@ -63,21 +63,7 @@
 samtools flagstat {snakemake.output.bam} > {snakemake.output.report_flagstats_txt}
 samtools idxstats {snakemake.output.bam} > {snakemake.output.report_idxstats_txt}
 
-# call plot-bamstats
-mkdir $TMPDIR/bamstats.d
-plot-bamstats \
-    -p $TMPDIR/bamstats.d/ \
-    {snakemake.output.report_bamstats_txt} \
-|| true  # ignore failure
-
-# Convert HTML report into one file.
-inline-html \
-    --in-file $TMPDIR/bamstats.d/index.html \
-    --out-file {snakemake.output.report_bamstats_html} \
-|| touch {snakemake.output.report_bamstats_html}
-
 # Build MD5 files for the reports
-md5sum {snakemake.output.report_bamstats_html} > {snakemake.output.report_bamstats_html_md5}
 md5sum {snakemake.output.report_bamstats_txt} > {snakemake.output.report_bamstats_txt_md5}
 md5sum {snakemake.output.report_flagstats_txt} >{snakemake.output.report_flagstats_txt_md5}
 md5sum {snakemake.output.report_idxstats_txt} > {snakemake.output.report_idxstats_txt_md5}

snappy_wrappers/wrappers/minimap2/environment.yaml

@@ -6,8 +6,3 @@ dependencies:
   - minimap2 ==2.24
   - samtools ==1.9
   - htslib ==1.9
-  # NB: gnuplot 5.2.7 is incompatible with samtools ==1.9
-  # see https://github.com/samtools/samtools/issues/1065
-  - gnuplot ==5.2.6
-  - libpng ==1.6.37
-  - inline-html ==1.0.0

snappy_wrappers/wrappers/minimap2/wrapper.py

@@ -107,44 +107,7 @@
 samtools flagstat {snakemake.output.bam} > {snakemake.output.report_flagstats_txt}
 samtools idxstats {snakemake.output.bam} > {snakemake.output.report_idxstats_txt}
 
-# call plot-bamstats
-mkdir $TMPDIR/bamstats.d
-plot-bamstats \
-    -p $TMPDIR/bamstats.d/ \
-    {snakemake.output.report_bamstats_txt} \
-|| true  # ignore failure
-
-# Patch inline-html if necessary.
-cat >$TMPDIR/inline-html.diff <<EOF
-diff --git a/inline_html/inline_html.py b/inline_html/inline_html.py
-index 893086c..cbef6dd 100644
---- a/inline_html/inline_html.py
-+++ b/inline_html/inline_html.py
-@@ -20,7 +20,10 @@ def resource_to_data(path, in_file):
-     mime, _ = mimetypes.guess_type(path)
-     with open(path, 'rb') as fp:
-         data = fp.read()
--        data64 = b''.join(base64.encodestring(data).splitlines())
-+        try:
-+            data64 = b''.join(base64.encodestring(data).splitlines())
-+        except AttributeError:
-+            data64 = b''.join(base64.encodebytes(data).splitlines())
-         return 'data:%s;base64,%s' % (mime, data64.decode('ascii'))
-EOF
-pushd $(python3 -c 'import inline_html; print(inline_html.__path__[0])')
-if ! grep encodebytes inline_html.py; then
-    patch -p2 <$TMPDIR/inline-html.diff
-fi
-popd
-
-# Convert HTML report into one file.
-inline-html \
-    --in-file $TMPDIR/bamstats.d/index.html \
-    --out-file {snakemake.output.report_bamstats_html} \
-|| touch {snakemake.output.report_bamstats_html}
-
 # Build MD5 files for the reports
-md5sum {snakemake.output.report_bamstats_html} > {snakemake.output.report_bamstats_html_md5}
 md5sum {snakemake.output.report_bamstats_txt} > {snakemake.output.report_bamstats_txt_md5}
 md5sum {snakemake.output.report_flagstats_txt} >{snakemake.output.report_flagstats_txt_md5}
 md5sum {snakemake.output.report_idxstats_txt} > {snakemake.output.report_idxstats_txt_md5}

snappy_wrappers/wrappers/star/environment.yaml

@@ -7,8 +7,3 @@ dependencies:
   - samblaster ==0.1.24
   - seqtk ==1.3
   - trimadap ==r11
-  # NB: gnuplot 5.2.7 is incompatible with samtools ==1.9
-  # see https://github.com/samtools/samtools/issues/1065
-  - gnuplot ==5.2.6
-  - libpng ==1.6.37
-  - inline-html ==1.0.0

snappy_wrappers/wrappers/star/wrapper.py

@@ -202,43 +202,7 @@
 samtools flagstat {snakemake.output.bam} > {snakemake.output.report_flagstats_txt}
 samtools idxstats {snakemake.output.bam} > {snakemake.output.report_idxstats_txt}
 
-# call plot-bamstats
-mkdir $TMPDIR/bamstats.d
-plot-bamstats \
-    -p $TMPDIR/bamstats.d/ \
-    {snakemake.output.report_bamstats_txt} \
-|| true  # ignore failure
-
-# Patch inline-html if necessary.
-cat >$TMPDIR/inline-html.diff <<EOF
-diff --git a/inline_html/inline_html.py b/inline_html/inline_html.py
-index 893086c..cbef6dd 100644
---- a/inline_html/inline_html.py
-+++ b/inline_html/inline_html.py
-@@ -20,7 +20,10 @@ def resource_to_data(path, in_file):
-     mime, _ = mimetypes.guess_type(path)
-     with open(path, 'rb') as fp:
-         data = fp.read()
--        data64 = b''.join(base64.encodestring(data).splitlines())
-+        try:
-+            data64 = b''.join(base64.encodestring(data).splitlines())
-+        except AttributeError:
-+            data64 = b''.join(base64.encodebytes(data).splitlines())
-         return 'data:%s;base64,%s' % (mime, data64.decode('ascii'))
-EOF
-pushd $(python3 -c 'import inline_html; print(inline_html.__path__[0])')
-if ! grep encodebytes inline_html.py; then
-    patch -p2 <$TMPDIR/inline-html.diff
-fi
-popd
-
-# Convert HTML report into one file.
-inline-html \
-    --in-file $TMPDIR/bamstats.d/index.html \
-    --out-file {snakemake.output.report_bamstats_html}
-
 # Build MD5 files for the reports
-md5sum {snakemake.output.report_bamstats_html} > {snakemake.output.report_bamstats_html_md5}
 md5sum {snakemake.output.report_bamstats_txt} > {snakemake.output.report_bamstats_txt_md5}
 md5sum {snakemake.output.report_flagstats_txt} >{snakemake.output.report_flagstats_txt_md5}
 md5sum {snakemake.output.report_idxstats_txt} > {snakemake.output.report_idxstats_txt_md5}

tests/snappy_pipeline/apps/test_snappy_snake.py

@@ -50,8 +50,6 @@ def test_snappy_snake_list_output(germline_sheet_fake_project_ngs_mapping_fs, mo
             "mtime",
             "params",
             "input",
-            "software-env",
-            "code",
             "-S",
             "--verbose",
             "--cores",

tests/snappy_pipeline/workflows/test_workflows_ngs_mapping.py

@@ -94,7 +94,7 @@ def get_expected_output_files_dict(bam_base_out, report_base_out):
     :type bam_base_out: str
 
     :param report_base_out: Expected name pattern of report associated files without extension.
-    For example if the full path would be '/path/to/step_report.bam.bamstats.html', argument should
+    For example if the full path would be '/path/to/step_report.bam.bamstats.txt', argument should
     be '/path/to/step_report'.
 
     :return: Returns dictionary with expected path for BAM and report associated files based on the
@@ -106,8 +106,6 @@ def get_expected_output_files_dict(bam_base_out, report_base_out):
         "bam_bai": bam_base_out + ".bam.bai",
         "bam_bai_md5": bam_base_out + ".bam.bai.md5",
         "bam_md5": bam_base_out + ".bam.md5",
-        "report_bamstats_html": report_base_out + ".bam.bamstats.html",
-        "report_bamstats_html_md5": report_base_out + ".bam.bamstats.html.md5",
         "report_bamstats_txt": report_base_out + ".bam.bamstats.txt",
         "report_bamstats_txt_md5": report_base_out + ".bam.bamstats.txt.md5",
         "report_flagstats_txt": report_base_out + ".bam.flagstats.txt",
@@ -837,12 +835,6 @@ def test_ngs_mapping_workflow_files(ngs_mapping_workflow):
         for i in range(1, 7)
         for stats in ("bamstats", "flagstats", "idxstats")
     ]
-    bam_stats_html_out = (
-        "output/bwa.P00{i}-N1-DNA1-WGS1/report/bam_qc/bwa.P00{i}-N1-DNA1-WGS1.bam.bamstats.{ext}"
-    )
-    expected += [
-        bam_stats_html_out.format(i=i, ext=ext) for ext in ("html", "html.md5") for i in range(1, 7)
-    ]
     expected += [
         "output/bwa.P00{i}-N1-DNA1-WGS1/report/cov/bwa.P00{i}-N1-DNA1-WGS1.cov.{ext}".format(
             i=i, ext=ext

tests/snappy_pipeline/workflows/test_workflows_ngs_mapping_processed_fastq.py

@@ -95,7 +95,7 @@ def get_expected_output_files_dict(bam_base_out, report_base_out):
     :type bam_base_out: str
 
     :param report_base_out: Expected name pattern of report associated files without extension.
-    For example if the full path would be '/path/to/step_report.bam.bamstats.html', argument should
+    For example if the full path would be '/path/to/step_report.bam.bamstats.txt', argument should
     be '/path/to/step_report'.
 
     :return: Returns dictionary with expected path for BAM and report associated files based on the
@@ -107,8 +107,6 @@ def get_expected_output_files_dict(bam_base_out, report_base_out):
         "bam_bai": bam_base_out + ".bam.bai",
         "bam_bai_md5": bam_base_out + ".bam.bai.md5",
         "bam_md5": bam_base_out + ".bam.md5",
-        "report_bamstats_html": report_base_out + ".bam.bamstats.html",
-        "report_bamstats_html_md5": report_base_out + ".bam.bamstats.html.md5",
         "report_bamstats_txt": report_base_out + ".bam.bamstats.txt",
         "report_bamstats_txt_md5": report_base_out + ".bam.bamstats.txt.md5",
         "report_flagstats_txt": report_base_out + ".bam.flagstats.txt",
@@ -768,12 +766,6 @@ def test_ngs_mapping_workflow_files(ngs_mapping_workflow):
         for i in range(1, 7)
         for stats in ("bamstats", "flagstats", "idxstats")
     ]
-    bam_stats_html_out = (
-        "output/bwa.P00{i}-N1-DNA1-WGS1/report/bam_qc/bwa.P00{i}-N1-DNA1-WGS1.bam.bamstats.{ext}"
-    )
-    expected += [
-        bam_stats_html_out.format(i=i, ext=ext) for ext in ("html", "html.md5") for i in range(1, 7)
-    ]
     expected += [
         "output/bwa.P00{i}-N1-DNA1-WGS1/report/cov/bwa.P00{i}-N1-DNA1-WGS1.cov.{ext}".format(
             i=i, ext=ext

tests/snappy_wrappers/wrappers/bwa_mem_pe/Snakefile

@@ -21,8 +21,6 @@ rule run_bwa_mem_paired_end:
     output:
         bam="output/out.bam",
         bam_bai="output/out.bam.bai",
-        report_bamstats_html="output/out.bamstats.html",
-        report_bamstats_html_md5="output/out.bamstats.html.md5",
         report_bamstats_txt="output/out.bamstats.txt",
         report_bamstats_txt_md5="output/out.bamstats.txt.md5",
         report_flagstats_txt="output/out.flagstats.txt",