new parameter --setup to preload all DBs and ref files, closes #102

README.md

@@ -349,7 +349,7 @@ nextflow run hoelzer-lab/rnaflow -profile test,local,docker
 nextflow run hoelzer-lab/rnaflow -profile test,slurm,singularity
 ```
 
-As a __best practice__ for a local execution, we recommend to run the pipeline with `--cores 1 --max_cores 1` the first time you use `Singularity`, because we experienced issues when generating the `Singularity` images in parallel the first time the pipeline is executed with this engine option.
+As a __best practice__ for a local execution, we recommend to run the pipeline with `--cores 1 --max_cores 1` the first time you use `Singularity`, because we experienced issues when generating the `Singularity` images in parallel the first time the pipeline is executed with this engine option. It is also possible to run the pipeline once with `--setup` set. In setup mode all the necessary files (DBs, reference files and images) are being downloaded and set up.
 
 You can customize where `conda` environments are stored using
 
@@ -562,6 +562,7 @@ Caching:
 --singularityCacheDir    Location for storing the singularity images [default: singularity]
 --workdir                Working directory for all intermediate results [default: null] (DEPRECATED: use `-w your/workdir` instead)
 --softlink_results       Softlink result files instead of copying.
+--setup                  Download all necessary DB, reference and image files without running the pipeline. [default: false]
 
 Nextflow options:
 -with-tower              Activate monitoring via Nextflow Tower (needs TOWER_ACCESS_TOKEN set).

configs/container.config

@@ -1,19 +1,20 @@
 process {
-    withLabel: hisat2       { container = 'nanozoo/hisat2:2.1.0--66dae66' } 
-    withLabel: minimap2     { container = 'nanozoo/minimap2:2.22--b178b67' } 
-    withLabel: python3      { container = 'nanozoo/python_rnaseq:3--23823db' } 
-    withLabel: deseq2       { container = 'nanozoo/deseq2:1.28.0--0df1612' } 
-    withLabel: fastp        { container = 'nanozoo/fastp:0.23.1--9f2e255' } 
-    withLabel: fastqc       { container = 'nanozoo/fastqc:0.11.8--fbfa1d7' } 
-    withLabel: subread      { container = 'nanozoo/subread:2.0.1--713a8e7' } 
-    withLabel: multiqc      { container = 'nanozoo/multiqc:1.11--9dfdee6' }
-    withLabel: nanoplot     { container = 'nanozoo/nanoplot:1.38.1--e303519' }
-    withLabel: sortmerna    { container = 'nanozoo/sortmerna:2.1b--ceea1a1' } 
-    withLabel: trinity      { container = 'trinityrnaseq/trinityrnaseq:2.13.2' } 
-    withLabel: stringtie    { container = 'nanozoo/stringtie:2.1.7--420b0db' } 
-    withLabel: busco        { container = 'ezlabgva/busco:v5.2.1_cv1' } 
-    withLabel: dammit       { container = 'nanozoo/dammit:1.2--b47259e' } 
-    withLabel: dammitDB     { container = 'nanozoo/dammit:1.2--b47259e' }
-    withLabel: basic_tools  { container = 'nanozoo/python_rnaseq:3.8--7a7808c' }
-    withLabel: rattle       { container = 'huxleys/rattle:1.0.0--24021329c8b365f21959c56ee1cfb693c768c14e' }
+    withLabel:	hisat2	{ container = "nanozoo/hisat2:2.1.0--66dae66" } 
+    withLabel:	minimap2	{ container = "nanozoo/minimap2:2.22--b178b67" } 
+    withLabel:	python3	{ container = "nanozoo/python_rnaseq:3--23823db" } 
+    withLabel:	deseq2	{ container = "nanozoo/deseq2:1.28.0--0df1612" } 
+    withLabel:	fastp	{ container = "nanozoo/fastp:0.23.1--9f2e255" } 
+    withLabel:	fastqc	{ container = "nanozoo/fastqc:0.11.8--fbfa1d7" } 
+    withLabel:	subread	{ container = "nanozoo/subread:2.0.1--713a8e7" } 
+    withLabel:	multiqc	{ container = "nanozoo/multiqc:1.11--9dfdee6" } 
+    withLabel:	nanoplot	{ container = "nanozoo/nanoplot:1.38.1--e303519" }
+    withLabel:	sortmerna	{ container = "nanozoo/sortmerna:2.1b--ceea1a1" } 
+    withLabel:	trinity	{ container = "trinityrnaseq/trinityrnaseq:2.13.2" } 
+    withLabel:	stringtie	{ container = "nanozoo/stringtie:2.1.7--420b0db" } 
+    withLabel:	busco	{ container = "ezlabgva/busco:v5.2.1_cv1" } 
+    withLabel:	dammit	{ container = "nanozoo/dammit:1.2--b47259e" } 
+    withLabel:	dammitDB	{ container = "nanozoo/dammit:1.2--b47259e" }
+    withLabel:	basic_tools	{ container = "nanozoo/python_rnaseq:3.8--7a7808c" }
+    withLabel:	rattle	{ container = "huxleys/rattle:1.0.0--24021329c8b365f21959c56ee1cfb693c768c14e" }
 }
+

modules/busco.nf

@@ -48,4 +48,4 @@ process busco {
 Buscos plotting feature does not work in Docker by default.
 The "hack" modifies via sed the Rscript that gets generated after generate_plot.py
 After changing to pdf (circumventing resolutions) we rerun the script via Rscript. Tadaa... we have a plot
-*/
+*/

modules/buscoGetDB.nf

@@ -6,6 +6,7 @@ process buscoGetDB {
     maxRetries 2
 
     if (params.cloudProcess) { publishDir "${params.permanentCacheDir}/databases/busco/${params.busco_db}", mode: 'copy', pattern: "${params.busco_db}_odb10.tar.gz" }
+
     else { storeDir "${params.permanentCacheDir}/databases/busco/${params.busco_db}" }
 
   output:

modules/containerGet.nf

@@ -0,0 +1,49 @@
+process containerGet {
+    label 'smallTask'
+    tag "$tool"
+
+    errorStrategy 'retry'
+    maxRetries 10
+
+    if ( workflow.profile.contains('singularity') ) { storeDir "${params.singularityCacheDir}" }
+    else if ( workflow.profile.contains('conda') ) { storeDir "${params.condaCacheDir}" }
+
+  input:
+    tuple val(tool), val(path)
+
+  output:
+    path("${img_file_name}.img")
+
+  script:
+  img_file_name = path.replace("/", "-").replace(":","-")
+  if ( workflow.profile.contains('singularity') )
+    """
+    if [ -e ${params.singularityCacheDir}/${img_file_name}.img ] 
+        then
+            echo "${tool} singularity image file already exists, skipping."
+    else
+        singularity pull --name ${img_file_name}.img "docker://${path}"
+    fi
+    """
+  else if ( workflow.profile.contains('docker') )
+    """
+    if [[ "\$(docker images -q ${path} 2> /dev/null)" == "" ]]; 
+    then
+      docker pull ${path}
+      touch ${img_file_name}.img
+	  else
+	    echo "${tool} docker image file already exists, skipping."
+      touch ${img_file_name}.img
+    fi
+    """
+  else if ( workflow.profile.contains('conda') )
+    """
+    echo "The setup mode currently doesn't support conda environments, skipping."
+    touch ${img_file_name}.img
+    """
+  else 
+    """
+    echo 'unknown container engine, exiting'
+    exit 1
+    """
+}

modules/dammitGetDB.nf

@@ -1,5 +1,5 @@
 process dammitGetDB {
-    label 'dammit'
+    label 'basic_tools'
     label 'smallTask'
 
     errorStrategy 'retry'

nextflow.config

@@ -39,6 +39,7 @@ params {
     featurecounts_additional_params = '-t exon -g gene_id' // default values
     feature_id_type = 'ensembl_gene_id'
     rna = false
+    setup = false
 
     busco_db = 'euarchontoglires'
     dammit_uniref90 = false
@@ -106,7 +107,7 @@ profiles {
     slurm {
         executor {
             name = "slurm"
-            queueSize = 100
+            queueSize = params.setup ? 1 : 100
             // exitReadTimeout = "360 sec"
             // submitRateLimit = '1 / 2 s'
             // pollInterval = '30 sec'
@@ -119,7 +120,7 @@ profiles {
     lsf {
         executor {
             name = "lsf"
-            queueSize = 100
+            queueSize = params.setup ? 1 : 100
             // exitReadTimeout = "360 sec"
             // submitRateLimit = '1 / 2 s'
             // pollInterval = '30 sec'