Merge pull request #37 from metagenlab/slurm

Add SLURM profile

.github/workflows/unit-tests.yml

@@ -64,7 +64,7 @@ jobs:
       - name: Cache apptainer containers
         uses: actions/cache@v3
         with:
-          path: .snakemake/apptainer # Cache the Apptainer containers directory
+          path: .snakemake/singularity # Cache the Apptainer containers directory
           key: ${{ runner.os }}-apptainer-${{ hashFiles('mess/workflow/envs/containers.yml') }}
           restore-keys: |
             ${{ runner.os }}-apptainer-

mess/__main__.py

@@ -55,6 +55,7 @@
         "options": [
             "--tech",
             "--bases",
+            "--rotate",
             "--tool",
             "--error",
             "--mean-len",
@@ -114,7 +115,6 @@
         "options": [
             "--asm-summary",
             "--fasta",
-            "--rotate",
         ],
     },
 )
@@ -137,8 +137,6 @@
         "options": [
             "--configfile",
             "--profile",
-            "--sdm",
-            "--prefix",
             "--snake-default",
         ],
     },

mess/config/config.yaml

@@ -69,5 +69,5 @@ resources:
     time: "00:30:00"
   sml:
     cpu: 1
-    mem: 2000
+    mem: 1000
     time: "00:00:10"

mess/profiles/slurm/config.yaml

@@ -0,0 +1,6 @@
+executor: slurm
+singularity-prefix: /mnt/slow_storage/containers
+default-resources:  
+  slurm_partition: short
+  runtime: 30
+jobs: 100

mess/util.py

@@ -12,8 +12,6 @@
     msg_box,
 )
 
-workflow_basedir = os.path.join(os.path.dirname(os.path.realpath(__file__)))
-
 
 def snake_base(rel_path):
     """Get the filepath to a Snaketool system file (relative to __main__.py)"""
@@ -41,21 +39,13 @@ def default_to_output(ctx, param, value):
     return value
 
 
-def get_deployment_prefix(ctx, param, value):
-    """Callback for click options; joins software deployment name and prefix dir"""
-    if not value:
-        return os.path.join(os.getcwd(), ".snakemake", ctx.params["sdm"])
-    return value
-
-
 def run_snakemake(
     configfile=None,
     system_config=None,
     snakefile_path=None,
     merge_config=None,
     threads=1,
-    sdm=None,
-    prefix=None,
+    sdm="apptainer",
     snake_default=None,
     snake_args=[],
     profile=None,
@@ -110,17 +100,20 @@ def run_snakemake(
         snake_command += ["--cores", threads]
 
     # add software deployment args
-    if sdm:
-        if sdm == "conda":
-            snake_command += ["--sdm conda"]
-            if prefix:
-                snake_command += ["--conda-prefix", prefix]
-        if sdm == "apptainer":
-            snake_command += [
-                f"--sdm apptainer --apptainer-args '-B {workflow_basedir}:{workflow_basedir}'"
-            ]
-            if prefix:
-                snake_command += ["--apptainer-prefix", prefix]
+
+    if sdm == "conda":
+        snake_command += ["--sdm conda"]
+
+    if sdm == "apptainer":
+        snake_command += ["--sdm apptainer --apptainer-args"]
+        paths = [
+            os.path.join(os.path.dirname(os.path.realpath(__file__))),
+            os.path.abspath(snake_config["args"]["output"]),
+        ]
+        args = " ".join([f"-B {path}:{path}" for path in paths])
+
+        snake_command += [f"'{args}'"]
+
     # add snakemake default args
     if snake_default:
         snake_command += snake_default
@@ -191,21 +184,6 @@ def common_options(func):
             help="Snakemake profile to use",
             show_default=False,
         ),
-        click.option(
-            "--sdm",
-            type=click.Choice(["apptainer", "conda"]),
-            default="apptainer",
-            help="Software deplolyment method",
-            show_default=True,
-        ),
-        click.option(
-            "--prefix",
-            default=None,
-            help="Custom softawre deployment directory",
-            callback=get_deployment_prefix,
-            type=click.Path(),
-            show_default=False,
-        ),
         click.option(
             "--snake-default",
             multiple=True,

mess/workflow/Snakefile

@@ -43,8 +43,8 @@ OUTPUT = config.args.output
 LOG = os.path.join(OUTPUT, "mess.log")
 THREADS = config.args.threads
 
+
 # download options
-CONDA_PREFIX = config.args.conda_prefix
 TAXONKIT = config.args.taxonkit
 API_KEY = config.args.api_key
 LIMIT = config.args.limit

mess/workflow/download.smk

@@ -36,7 +36,6 @@ INPUT = os.path.abspath(str(config.args.input))
 OUTPUT = config.args.output
 LOG = os.path.join(OUTPUT, "mess.log")
 THREADS = config.args.threads
-CONDA_PREFIX = config.args.conda_prefix
 TAXONKIT = config.args.taxonkit
 API_KEY = config.args.api_key
 LIMIT = config.args.limit

mess/workflow/rules/preflight/functions.smk

@@ -6,6 +6,9 @@ from itertools import product
 from Bio import SeqIO
 import random
 
+# To get rid of the setlocale warning
+os.environ["LC_ALL"] = "C.UTF-8"
+
 
 wildcard_constraints:
     sample="[^/]+",
@@ -99,6 +102,10 @@ def list_fastas(wildcards):
 table_cache = {}
 
 
+def get_cov_df(wildcards):
+    return checkpoints.split_contigs.get(**wildcards).output[0]
+
+
 def get_cov_table(wildcards, key, idx_col):
     cov_table = checkpoints.split_contigs.get(**wildcards).output[0]
     if cov_table not in table_cache:

mess/workflow/rules/simulate/long_reads.smk

@@ -18,7 +18,8 @@ else:
 
 rule pbsim3:
     input:
-        fasta,
+        df=get_cov_df,
+        fa=fasta,
     output:
         pbsim3_out,
         temp(prefix + ".maf"),
@@ -66,7 +67,7 @@ rule pbsim3:
         --pass-num {params.passes} \\
         --accuracy-mean {params.accuracy} \\
         --depth {params.cov} \\
-        --genome {input} &> {log}
+        --genome {input.fa} &> {log}
         
         mv {params.prefix}_0001.maf {params.prefix}.maf
         mv {params.prefix}_0001.ref {params.prefix}.ref

mess/workflow/rules/simulate/short_reads.smk

@@ -51,7 +51,8 @@ if CIRCULAR:
 
 rule art_illumina:
     input:
-        fasta,
+        df=get_cov_df,
+        fa=fasta,
     output:
         fastqs=fastq_out,
         sam=sam_out,
@@ -78,7 +79,7 @@ rule art_illumina:
         containers.art
     shell:
         """
-        art_illumina -i {input} \\
+        art_illumina -i {input.fa} \\
         -rs {params.seed} -l {params.read_len} \\
         -f {params.cov} -na {params.args} \\
         -o {params.prefix} &> {log}