diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2c2fed6e2..d6853ffb7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,12 +76,12 @@ jobs: steps: - uses: actions/checkout@v2 - # - name: Cache conda dependencies - # uses: actions/cache@v2 - # with: - # path: | - # .tests/.snakemake/conda - # key: technology-${{ runner.os }}-${{ matrix.rule }}-${{ matrix.technology }}-${{ matrix.seq_method }}-${{ hashFiles('*.tests/.snakemake/conda/*.yaml') }} + # android - will release about 10 GB if you don't need Android + # dotnet - will release about 20 GB if you don't need .NET + - name: Free up some disk sapce + run: | + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet - name: Prepare test data for all technologies if: steps.test-data.outputs.cache-hit != true && (startsWith(matrix.rule, 'all') && matrix.technology == 'all' || matrix.rule == 'compare_assemblers') diff --git a/workflow/rules/assembly.smk b/workflow/rules/assembly.smk index f0b87bb63..a39a6a5b9 100644 --- a/workflow/rules/assembly.smk +++ b/workflow/rules/assembly.smk @@ -93,7 +93,7 @@ rule spades_assemble_se: input: get_reads_after_qc, output: - "results/{date}/assembly/{sample}/spades-se/{sample}.contigs.fasta", + temp("results/{date}/assembly/{sample}/spades-se/{sample}.contigs.fasta"), log: "logs/{date}/spades/se/{sample}.log", conda: @@ -112,7 +112,7 @@ rule check_contigs: input: get_contigs, output: - "results/{date}/contigs/checked/{sample}.fasta", + temp("results/{date}/contigs/checked/{sample}.fasta"), log: "logs/{date}/check_contigs/{sample}.log", conda: @@ -146,7 +146,7 @@ rule filter_chr0: input: "results/{date}/contigs/ordered-unfiltered/{sample}.fasta", output: - "results/{date}/contigs/ordered/{sample}.fasta", + temp("results/{date}/contigs/ordered/{sample}.fasta"), log: "logs/{date}/ragoo/{sample}_cleaned.log", conda: @@ -206,7 +206,7 @@ rule aggregate_polished_de_novo_sequences: input: get_polished_sequence, output: - "results/{date}/contigs/polished/{sample}.fasta", + temp("results/{date}/contigs/polished/{sample}.fasta"), log: "logs/{date}/aggregate_polished_de_novo_sequences/{sample}.log", conda: @@ -219,7 +219,7 @@ rule aggregate_fallback_sequences: input: get_fallback_sequence, output: - "results/{date}/contigs/fallback/{sample}.fasta", + temp("results/{date}/contigs/fallback/{sample}.fasta"), log: "logs/{date}/aggregate_fallback_sequences/{sample}.log", conda: @@ -233,7 +233,7 @@ rule align_contigs: target="resources/genomes/main.fasta", query=get_quast_fastas, output: - "results/{date}/aligned/ref~main/{stage}~{sample}.bam", + temp("results/{date}/aligned/ref~main/{stage}~{sample}.bam"), log: "results/{date}/aligned/ref~main/{stage}~{sample}.log", conda: diff --git a/workflow/rules/long_read.smk b/workflow/rules/long_read.smk index 11a401868..29f3aa4d0 100644 --- a/workflow/rules/long_read.smk +++ b/workflow/rules/long_read.smk @@ -8,7 +8,7 @@ rule nanoQC: input: get_reads_by_stage, output: - "results/{date}/qc/nanoQC/{sample}/{stage}/nanoQC.html", + temp("results/{date}/qc/nanoQC/{sample}/{stage}/nanoQC.html"), log: "logs/{date}/nanoQC/{sample}_{stage}.log", params: @@ -23,7 +23,7 @@ rule count_fastq_reads: input: get_reads_by_stage, output: - "results/{date}/tables/fastq-read-counts/{stage}~{sample}.txt", + temp("results/{date}/tables/fastq-read-counts/{stage}~{sample}.txt"), log: "logs/{date}/count_reads/{stage}~{sample}.log", conda: @@ -36,7 +36,7 @@ rule porechop_adapter_barcode_trimming: input: get_fastqs, output: - "results/{date}/trimmed/porechop/adapter_barcode_trimming/{sample}.fastq", + temp("results/{date}/trimmed/porechop/adapter_barcode_trimming/{sample}.fastq"), conda: "../envs/porechop.yaml" log: @@ -68,7 +68,7 @@ rule porechop_primer_trimming: ), repl_flag="results/.indicators/replacement_notice.txt", output: - "results/{date}/trimmed/porechop/primer_clipped/{sample}.fastq", + temp("results/{date}/trimmed/porechop/primer_clipped/{sample}.fastq"), conda: "../envs/primechop.yaml" log: @@ -82,7 +82,7 @@ rule nanofilt: input: "results/{date}/trimmed/porechop/primer_clipped/{sample}.fastq", output: - "results/{date}/trimmed/nanofilt/{sample}.fastq", + temp("results/{date}/trimmed/nanofilt/{sample}.fastq"), log: "logs/{date}/nanofilt/{sample}.log", params: @@ -126,7 +126,7 @@ use rule assembly_polishing_ont as medaka_consensus_reference with: fasta="results/{date}/corrected/{sample}/{sample}.correctedReads.fasta.gz", reference="resources/genomes/main.fasta", output: - "results/{date}/consensus/medaka/{sample}/{sample}.fasta", + temp("results/{date}/consensus/medaka/{sample}/{sample}.fasta"), # polish consensus @@ -136,7 +136,7 @@ rule bcftools_consensus_ont: bcf="results/{date}/filtered-calls/ref~{sample}/{sample}.subclonal.high+moderate-impact.bcf", # clonal vs. subclonal? bcfidx="results/{date}/filtered-calls/ref~{sample}/{sample}.subclonal.high+moderate-impact.bcf.csi", output: - "results/{date}/consensus/bcftools/{sample}.fasta", + temp("results/{date}/consensus/bcftools/{sample}.fasta"), log: "logs/{date}/bcftools-consensus-ont/{sample}.log", conda: diff --git a/workflow/rules/pseudoassembly.smk b/workflow/rules/pseudoassembly.smk index 66af0821a..b3b45fb12 100644 --- a/workflow/rules/pseudoassembly.smk +++ b/workflow/rules/pseudoassembly.smk @@ -35,7 +35,7 @@ rule compare_assemblies: assembly="results/{date}/contigs/polished/{sample}.fasta", pseudoassembly="results/{date}/contigs/pseudoassembled/{sample}.fasta", output: - "results/{date}/aligned/assemblies/{sample}.bam", + temp("results/{date}/aligned/assemblies/{sample}.bam"), log: "logs/{date}/aligned/assemblies/{sample}log", conda: diff --git a/workflow/rules/read_clipping.smk b/workflow/rules/read_clipping.smk index 7a0896b1a..d528fb1a7 100644 --- a/workflow/rules/read_clipping.smk +++ b/workflow/rules/read_clipping.smk @@ -8,7 +8,7 @@ rule samtools_sort: input: get_samtools_sort_input, output: - "results/{date}/read-sorted/{read_type}~{sorted_by}/{sample}.{stage}.bam", + temp("results/{date}/read-sorted/{read_type}~{sorted_by}/{sample}.{stage}.bam"), params: extra=( lambda wildcards: "-n -m 4G" if wildcards.sorted_by == "name" else "-m 4G" @@ -56,7 +56,9 @@ rule fgbio: reference=config["adapters"]["amplicon-reference"] ), output: - "results/{date}/read-clipping/hardclipped/{read_type}/{sample}/{sample}.bam", + temp( + "results/{date}/read-clipping/hardclipped/{read_type}/{sample}/{sample}.bam" + ), log: "logs/{date}/fgbio/{read_type}/{sample}.log", conda: @@ -69,8 +71,8 @@ rule samtools_fastq_pe: input: bam="results/{date}/read-sorted/pe~name/{sample}.hardclipped.bam", output: - fq1="results/{date}/read-clipping/fastq/pe/{sample}.1.fastq.gz", - fq2="results/{date}/read-clipping/fastq/pe/{sample}.2.fastq.gz", + fq1=temp("results/{date}/read-clipping/fastq/pe/{sample}.1.fastq.gz"), + fq2=temp("results/{date}/read-clipping/fastq/pe/{sample}.2.fastq.gz"), log: "logs/{date}/samtools_fastq/pe/{sample}.log", conda: @@ -84,7 +86,7 @@ rule samtools_fastq_se: input: bam="results/{date}/read-sorted/se~name/{sample}.hardclipped.bam", output: - "results/{date}/read-clipping/fastq/se/{sample}.fastq", + temp("results/{date}/read-clipping/fastq/se/{sample}.fastq"), log: "logs/{date}/samtools_fastq/se/{sample}.log", conda: diff --git a/workflow/rules/read_mapping.smk b/workflow/rules/read_mapping.smk index d8f67b156..f13f246a8 100644 --- a/workflow/rules/read_mapping.smk +++ b/workflow/rules/read_mapping.smk @@ -81,7 +81,7 @@ rule samtools_calmd: aln=get_recal_input, ref=get_reference(), output: - "results/{date}/recal/ref~{reference}/{sample}.bam", + temp("results/{date}/recal/ref~{reference}/{sample}.bam"), log: "logs/{date}/samtools-calmd/ref~{reference}/{sample}.log", params: diff --git a/workflow/rules/read_trimming.smk b/workflow/rules/read_trimming.smk index 5cbd45252..465dbcb0b 100644 --- a/workflow/rules/read_trimming.smk +++ b/workflow/rules/read_trimming.smk @@ -35,9 +35,9 @@ rule fastp_se: input: sample=get_fastqs, output: - trimmed="results/{date}/trimmed/fastp-se/{sample}.fastq.gz", - html="results/{date}/trimmed/fastp-se/{sample}.html", - json="results/{date}/trimmed/fastp-se/{sample}.fastp.json", + trimmed=temp("results/{date}/trimmed/fastp-se/{sample}.fastq.gz"), + html=temp("results/{date}/trimmed/fastp-se/{sample}.html"), + json=temp("results/{date}/trimmed/fastp-se/{sample}.fastp.json"), params: adapters=get_adapters, extra="--qualified_quality_phred {} ".format( diff --git a/workflow/rules/strain_calling.smk b/workflow/rules/strain_calling.smk index a61772b17..8aadfe384 100644 --- a/workflow/rules/strain_calling.smk +++ b/workflow/rules/strain_calling.smk @@ -50,8 +50,8 @@ rule kallisto_metrics: input: get_reads_after_qc, output: - avg_read_length="results/{date}/tables/avg_read_length/{sample}.txt", - standard_deviation="results/{date}/tables/standard_deviation/{sample}.txt", + avg_read_length=temp("results/{date}/tables/avg_read_length/{sample}.txt"), + standard_deviation=temp("results/{date}/tables/standard_deviation/{sample}.txt"), log: "logs/{date}/kallisto/metrics/{sample}.log", conda: diff --git a/workflow/rules/variant_calling.smk b/workflow/rules/variant_calling.smk index 1526ee167..05ddef8b9 100644 --- a/workflow/rules/variant_calling.smk +++ b/workflow/rules/variant_calling.smk @@ -94,7 +94,7 @@ rule vcf_2_bcf: input: "results/{date}/candidate-calls/ref~{reference}/{sample}.{varrange}.vcf", output: - "results/{date}/candidate-calls/ref~{reference}/{sample}.{varrange}.bcf", + temp("results/{date}/candidate-calls/ref~{reference}/{sample}.{varrange}.bcf"), log: "logs/{date}/vcf_2_bcf/ref~{reference}/{sample}.{varrange}.log", conda: @@ -122,7 +122,7 @@ rule varlociraptor_alignment_properties: ref_idx=get_reference(".fai"), bam="results/{date}/recal/ref~{reference}/{sample}.bam", output: - "results/{date}/alignment-properties/ref~{reference}/{sample}.json", + temp("results/{date}/alignment-properties/ref~{reference}/{sample}.json"), log: "logs/{date}/varlociraptor/estimate-alignment-properties/ref~{reference}/{sample}.log", conda: @@ -141,7 +141,7 @@ rule varlociraptor_preprocess: bam="results/{date}/recal/ref~{reference}/{sample}.bam", bai="results/{date}/recal/ref~{reference}/{sample}.bam.bai", output: - "results/{date}/observations/ref~{reference}/{sample}.{varrange}.bcf", + temp("results/{date}/observations/ref~{reference}/{sample}.{varrange}.bcf"), params: depth=config["variant-calling"]["max-read-depth"], log: