Profile cleanup and singularity implementation

.github/workflows/run_WtP.yml

@@ -8,4 +8,4 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: run_What_the_Phage
-      run: curl -s https://get.nextflow.io | bash && ./nextflow run phage.nf -profile git_action
+      run: curl -s https://get.nextflow.io | bash && ./nextflow run phage.nf -profile git_action,docker --cores 2

.gitignore

@@ -7,3 +7,4 @@ work/
 .nextflow/
 results/
 .Rhistory
+singularity_images/

README.md

@@ -52,7 +52,9 @@
 
 * WtP runs with the workflow manager `nextflow` using `docker`
 * this means all the other programs are automatically pulled via docker
-* Only `docker` and `nextflow` needs to be installed
+* Only `docker` or `singularity`, and `nextflow` needs to be installed
+* the first execution will take time as tons of databases and container get pulled/downloaded
+   * we recommend in executing the first time with a small data set!
 
 ### Easy Installation
 * if you dont have experience with bioinformatic tools use this
@@ -89,6 +91,7 @@ nextflow run replikation/What_the_Phage -r v0.5 --fasta ~/.nextflow/assets/repli
 
 * Docker installation [here](https://docs.docker.com/v17.09/engine/installation/linux/docker-ce/ubuntu/#install-docker-ce)
 * Nextflow installation [here](https://www.nextflow.io/)
+* Singularity installation [here](https://github.com/sylabs/singularity/blob/master/INSTALL.md)
 * move or add the nextflow executable to a bin path
 * add docker to your User group via `sudo usermod -a -G docker $USER`
 
@@ -98,7 +101,7 @@ nextflow run replikation/What_the_Phage -r v0.5 --fasta ~/.nextflow/assets/repli
 * for local use of the `master` branch you could either clone the git and do:
 
 ```bash
-./phage.nf --fasta 'test-data/*.fasta'
+./phage.nf --fasta 'test-data/*.fasta' -profile local,docker
 ```
 
 * or execute it directly via a release candidate ( `-r releasenumber` , e.g. `-r v0.5`)
@@ -111,6 +114,13 @@ nextflow run replikation/What_the_Phage -r v0.5 --fasta your-file.fasta
 
 * long-read input is supported via `--fastq`
 
+### Profiles
+
+* we are currently setting up a few profiles to allow for the execution in different working environments
+* `-profile local,docker` -> local use with docker container
+* `-profile local,singularity` -> local use with singularity container
+* if you need executioner profiles (slurm lsf etc.) please contact us and so we can implement that
+
 ### Tool control
 
 * all the tools can be separatly turned off via adding the related option flag
@@ -125,7 +135,7 @@ nextflow run replikation/What_the_Phage -r v0.5 --fasta your-file.fasta --dv --m
 ### Temporary files
 
 * all temporary files are stored in `/tmp/nextflow-phage-$USER`
-* you can change the location via adding '-work-dir new/path/here' to the command
+* you can change the location via adding '--workdir new/path/here' to the command
 
 ## Results
 

configs/container.config

@@ -0,0 +1,21 @@
+process {
+    withLabel: chromomap        { container = 'nanozoo/r_fungi:0.1--097b1bb' }
+    withLabel: deepvirfinder    { container = 'multifractal/deepvirfinder:0.1' }
+    withLabel: emboss           { container = 'quay.io/biocontainers/emboss:6.5.7--4' }
+    withLabel: ggplot2          { container = 'michelsteuwer/ggplot2:latest' }
+    withLabel: hmmscan          { container = 'mhoelzer/hmmscan:0.1' }
+    withLabel: marvel           { container = 'multifractal/marvel:0.1' }
+    withLabel: metaphinder      { container = 'multifractal/metaphinder:0.1' }
+    withLabel: pprmeta          { container = 'multifractal/ppr-meta:0.1' }
+    withLabel: prodigal         { container = 'nanozoo/prodigal:2.6.3--2769024' }
+    withLabel: python           { container = 'nanozoo/template:3.8--ccd0653' }
+    withLabel: r_plot           { container = 'replikation/r-phage-plot:latest' }
+    withLabel: samtools         { container = 'nanozoo/samtools:1.9--76b9270' }
+    withLabel: sourmash         { container = 'nanozoo/sourmash:2.3.0--4257650'  }
+    withLabel: ubuntu           { container = 'nanozoo/basics:1.0--962b907' }
+    withLabel: upsetr           { container = 'nanozoo/upsetr:1.4.0--0ea25b3' }
+    withLabel: vibrant          { container = 'multifractal/vibrant:0.4'  }
+    withLabel: virfinder        { container = 'multifractal/virfinder:0.1' }
+    withLabel: virnet           { container = 'multifractal/virnet:0.5.1' }
+    withLabel: virsorter        { container = 'multifractal/virsorter:0.1.2' }
+}

configs/local.config

@@ -1,26 +1,23 @@
 process.executor = 'local'
-docker { enabled = true }
-params.cpus = params.cores
-workDir = params.workdir
 
 process {
-            withLabel: deepvirfinder { cpus = 1; container = 'multifractal/deepvirfinder:0.1' }
-            withLabel: emboss { cpus = params.cores ; container = 'quay.io/biocontainers/emboss:6.5.7--4' }
-            withLabel: ggplot2 { cpus = 1 ; container = 'michelsteuwer/ggplot2:latest' }
-            withLabel: marvel { cpus = 1; container = 'multifractal/marvel:0.1' }
-            withLabel: metaphinder { cpus = 1; container = ' multifractal/metaphinder:0.1' }
-            withLabel: pprmeta { cpus = 1; container = 'multifractal/ppr-meta:0.1' }
-            withLabel: python { cpus = 1; container =  'nanozoo/template:3.8--ccd0653' }
-            withLabel: r_plot { cpus = 1; container = 'replikation/r-phage-plot:latest' }
-            withLabel: samtools { cpus = 1 ; container = 'nanozoo/samtools:latest' }
-            withLabel: sourmash { cpus = 1 ; container = 'nanozoo/sourmash:2.3.0--4257650'  }
-            withLabel: ubuntu { cpus = 1; container = 'nanozoo/basics:1.0--962b907' }
-            withLabel: upsetr { cpus = 1; container = 'nanozoo/upsetr:1.4.0--0ea25b3' }
-            withLabel: vibrant { cpus = 1 ; container = 'multifractal/vibrant:0.4'  }
-            withLabel: virfinder { cpus = 1; container = 'multifractal/virfinder:0.1' }
-            withLabel: virnet { cpus = 1; container = 'multifractal/virnet:0.5.1' }
-            withLabel: virsorter { cpus = 1; container = 'quay.io/biocontainers/virsorter:1.0.6--pl526h516909a_1' }
-            withLabel: prodigal { cpus = 1; container = 'nanozoo/prodigal:2.6.3--2769024' }
-            withLabel: hmmscan { cpus = params.cores; container = 'mhoelzer/hmmscan:0.1' }
-            withLabel: chromomap { cpus = params.cores; container = 'nanozoo/r_fungi:0.1--097b1bb' }
+    withLabel: chromomap        { cpus = params.cores }
+    withLabel: deepvirfinder    { cpus = 4 }
+    withLabel: emboss           { cpus = params.cores }
+    withLabel: ggplot2          { cpus = 1 }
+    withLabel: hmmscan          { cpus = params.cores }
+    withLabel: marvel           { cpus = 4 }
+    withLabel: metaphinder      { cpus = 4 }
+    withLabel: pprmeta          { cpus = 4 }
+    withLabel: prodigal         { cpus = 1 }
+    withLabel: python           { cpus = 1 }
+    withLabel: r_plot           { cpus = 1 }
+    withLabel: samtools         { cpus = 1 }
+    withLabel: sourmash         { cpus = 4 }
+    withLabel: ubuntu           { cpus = 1 }
+    withLabel: upsetr           { cpus = 1 }
+    withLabel: vibrant          { cpus = 4 }
+    withLabel: virfinder        { cpus = params.cores }
+    withLabel: virnet           { cpus = 4 }
+    withLabel: virsorter        { cpus = params.cores }
 }

configs/node.config

@@ -0,0 +1,25 @@
+
+process {
+
+            //errorStrategy = "retry"
+            //maxRetries = 1
+            withLabel: chromomap        { cpus = 1 }
+            withLabel: deepvirfinder    { cpus = 24; memory = '32 GB' }
+            withLabel: emboss           { cpus = 4; memory = '8 GB' }
+            withLabel: ggplot2          { cpus = 1; memory = '8 GB' }
+            withLabel: hmmscan          { cpus = 4 }
+            withLabel: marvel           { cpus = 24; memory = '32 GB' }
+            withLabel: metaphinder      { cpus = 24; memory = '32 GB' }
+            withLabel: pprmeta          { cpus = 24; memory = '32 GB' }
+            withLabel: prodigal         { cpus = 4 }
+            withLabel: python           { cpus = 1 }
+            withLabel: r_plot           { cpus = 4; memory = '4 GB' }
+            withLabel: samtools         { cpus = 1 ; memory = '32 GB' }
+            withLabel: sourmash         { cpus = 8 ; memory = '32 GB' }
+            withLabel: ubuntu           { cpus = 4; memory = '4 GB' }
+            withLabel: upsetr           { cpus = 1; memory = '4 GB' }
+            withLabel: vibrant          { cpus = 4 }
+            withLabel: virfinder        { cpus = 24; memory = '32 GB' }
+            withLabel: virnet           { cpus = 4 }
+            withLabel: virsorter        { cpus = 24; memory = '32 GB' }
+}

modules/databases/download_pvog_DB.nf

@@ -2,10 +2,10 @@
 process pvog_DB {
   label 'noDocker'    
   if (params.cloudProcess) { 
-    publishDir "${params.cloudDatabase}/", mode: 'copy' 
+    publishDir "${params.databases}/", mode: 'copy' 
   }
   else { 
-    storeDir "nextflow-autodownload-databases/pvog" 
+    storeDir "${params.databases}/pvog" 
   }  
 
   output:

modules/databases/download_references.nf

@@ -1,9 +1,9 @@
 process download_references {
         if (params.cloudProcess) {
-           publishDir "${params.cloudDatabase}/references/", mode: 'copy', pattern: "phage_references.fa"
+           publishDir "${params.databases}/references/", mode: 'copy', pattern: "phage_references.fa"
         }
         else {
-           storeDir "nextflow-autodownload-databases/references/"
+           storeDir "${params.databases}/references/"
         }
        label 'noDocker'    
       output:

modules/databases/download_rvdb_DB.nf

@@ -1,10 +1,10 @@
 process rvdb_DB {
   label 'noDocker'    
   if (params.cloudProcess) { 
-    publishDir "${params.cloudDatabase}/", mode: 'copy', pattern: "rvdb" 
+    publishDir "${params.databases}/", mode: 'copy', pattern: "rvdb" 
   }
   else { 
-    storeDir "nextflow-autodownload-databases/rvdb" 
+    storeDir "${params.databases}/rvdb" 
   }  
 
   output:

modules/databases/download_vog_DB.nf

@@ -1,10 +1,10 @@
 process vog_DB {
   label 'noDocker'    
   if (params.cloudProcess) { 
-    publishDir "${params.cloudDatabase}/", mode: 'copy', pattern: "vogdb" 
+    publishDir "${params.databases}/", mode: 'copy', pattern: "vogdb" 
   }
   else { 
-    storeDir "nextflow-autodownload-databases/vog" 
+    storeDir "${params.databases}/vog" 
   }  
 
   output:

modules/databases/phage_references_blastDB.nf

@@ -1,9 +1,9 @@
 process phage_references_blastDB {
         if (params.cloudProcess) {
-           publishDir "${params.cloudDatabase}/", mode: 'copy', pattern: "phage_db.*"
+           publishDir "${params.databases}/", mode: 'copy', pattern: "phage_db.*"
         }
         else {
-           storeDir "nextflow-autodownload-databases/blast_phage_DB"
+           storeDir "${params.databases}/blast_phage_DB"
         }
       label 'metaphinder'
     input:

modules/databases/ppr_download_dependencies.nf

@@ -1,9 +1,9 @@
 process ppr_download_dependencies {
       if (params.cloudProcess) {
-         publishDir "${params.cloudDatabase}/pprmeta/", mode: 'copy', pattern: "PPR-Meta"
+         publishDir "${params.databases}/pprmeta/", mode: 'copy', pattern: "PPR-Meta"
       }
       else {
-         storeDir "nextflow-autodownload-databases/pprmeta/"
+         storeDir "${params.databases}/pprmeta/"
       }
       label 'noDocker'    
       output:

modules/databases/sourmash_download_DB.nf

@@ -1,9 +1,9 @@
 process sourmash_download_DB {
         if (params.cloudProcess) {
-           publishDir "${params.cloudDatabase}/sourmash/", mode: 'copy', pattern: "phages.sbt.json.tar.gz"
+           publishDir "${params.databases}/sourmash/", mode: 'copy', pattern: "phages.sbt.json.tar.gz"
         }
         else {
-           storeDir "nextflow-autodownload-databases/sourmash/"
+           storeDir "${params.databases}/sourmash/"
         }
        label 'sourmash' 
       input:

modules/databases/vibrant_download_DB.nf

@@ -1,9 +1,9 @@
 process vibrant_download_DB {
         if (params.cloudProcess) {
-           publishDir "${params.cloudDatabase}/Vibrant/", mode: 'copy', pattern: "database.tar.gz"
+           publishDir "${params.databases}/Vibrant/", mode: 'copy', pattern: "database.tar.gz"
         }
         else {
-           storeDir "nextflow-autodownload-databases/Vibrant"
+           storeDir "${params.databases}/Vibrant"
         }
        label 'vibrant'    
       output:

modules/databases/virnet_download_dependencies.nf

@@ -1,9 +1,9 @@
 process virnet_download_dependencies {
       if (params.cloudProcess) {
-         publishDir "${params.cloudDatabase}/virnet/", mode: 'copy', pattern: "virnet"
+         publishDir "${params.databases}/virnet/", mode: 'copy', pattern: "virnet"
       }
       else {
-         storeDir "nextflow-autodownload-databases/virnet/"
+         storeDir "${params.databases}/virnet/"
       }
       label 'noDocker'    
       output:

modules/databases/virsorter_download_DB.nf

@@ -1,9 +1,9 @@
 process virsorter_download_DB {
         if (params.cloudProcess) {
-           publishDir "${params.cloudDatabase}/virsorter/", mode: 'copy', pattern: "virsorter-data"
+           publishDir "${params.databases}/virsorter/", mode: 'copy', pattern: "virsorter-data"
         }
         else {
-           storeDir "nextflow-autodownload-databases/virsorter/"
+           storeDir "${params.databases}/virsorter/"
         }
        label 'noDocker'    
       output:

modules/hmmscan.nf

@@ -1,22 +1,14 @@
 process hmmscan {
-      publishDir "${params.output}/${name}/hmm/", mode: 'copy'
-      label 'hmmscan'
-
+        publishDir "${params.output}/${name}/raw_data/hmm/", mode: 'copy'
+        label 'hmmscan'
     input:
-      tuple val(name), file(faa) 
-      file(pvog_db)
-      // file(vog_db)
-      // file(rvdb_db)
-
+        tuple val(name), file(faa) 
+        file(pvog_db)
     output:
-      tuple val(name), file("${name}_${pvog_db}_hmmscan.tbl"), file(faa)
-      // tuple val(name), file("${name}_${vog_db}_hmmscan.tbl"), file(faa)
-      // tuple val(name), file("${name}_${rvdb_db}_hmmscan.tbl"), file(faa)
-
+        tuple val(name), file("${name}_${pvog_db}_hmmscan.tbl"), file(faa)
     script:
     """
-      hmmscan --cpu ${task.cpus} --noali --domtblout ${name}_${pvog_db}_hmmscan.tbl ${pvog_db}/${pvog_db}.hmm ${faa}
-
+    hmmscan --cpu ${task.cpus} --noali --domtblout ${name}_${pvog_db}_hmmscan.tbl ${pvog_db}/${pvog_db}.hmm ${faa}
     """
 }
 

modules/parser/filter_tool_names.nf

@@ -1,5 +1,5 @@
 process filter_tool_names {
-      label 'upsetr'
+      label 'ubuntu'
     input:
       tuple val(name), file(files)
     output:

modules/prodigal.nf

@@ -1,13 +1,10 @@
 process prodigal {
-      publishDir "${params.output}/${name}/prodigal_out", mode: 'copy'
-      label 'prodigal'
-
+        publishDir "${params.output}/${name}/raw_data/prodigal_out", mode: 'copy'
+        label 'prodigal'
     input:
-    tuple val(name), path(positive_contigs) 
-
+        tuple val(name), path(positive_contigs) 
     output:
-    tuple val(name), path("${name}_prodigal.faa")
-
+        tuple val(name), path("${name}_prodigal.faa")
     script:
     """
     prodigal -p "meta" -a ${name}_prodigal.faa -i ${positive_contigs}

modules/tools/deepvirfinder.nf

@@ -7,7 +7,7 @@ process deepvirfinder {
       tuple val(name), file("${name}_*.list")
     script:
       """  
-      dvf.py -c ${params.cpus} -i ${fasta} -o ${name}
+      dvf.py -c ${task.cpus} -i ${fasta} -o ${name}
       cp ${name}/*.txt ${name}_\${PWD##*/}.list
       """
 }

modules/tools/marvel.nf

@@ -8,7 +8,7 @@ process marvel {
     script:
       """
       # Marvel
-      marvel_bins.py -i ${contig_bins} -t ${params.cpus} > results_\${PWD##*/}.txt
+      marvel_bins.py -i ${contig_bins} -t ${task.cpus} > results_\${PWD##*/}.txt
       """
 }