From 60b598f9330afb4e8b40446db38f6bb76e2db655 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Thu, 8 Aug 2024 14:35:24 +0200 Subject: [PATCH 1/7] add error handling --- configs/node.config | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/configs/node.config b/configs/node.config index f1bdb73..24c5a45 100644 --- a/configs/node.config +++ b/configs/node.config @@ -1,11 +1,11 @@ process { - withLabel: minimap2 { cpus = 24; memory = 24.GB } - withLabel: bbmap { cpus = 24; memory = 24.GB } - withLabel: smallTask { cpus = 1; memory = 2.GB } - withLabel: pysam { cpus = 2; memory = 4.GB } - withLabel: fastqc { cpus = {2 * task.attempt}; memory = {4.GB * task.attempt } ; maxRetries = 3 ; errorStrategy = { task.exitStatus in 130..140 ? 'retry' : 'terminate' } } - withLabel: multiqc { cpus = 4; memory = 4.GB } - withLabel: nanoplot{ cpus = 8; memory = 8.GB } - withLabel: quast{ cpus = 8; memory = 8.GB } + withLabel: minimap2 { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 3 ; errorStrategy = { task.exitStatus in 1 ? 'retry' : 'terminate' }; } + withLabel: bbmap { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 3 ; errorStrategy = { task.exitStatus in 1 ? 'retry' : 'terminate' }; } + withLabel: smallTask { cpus = 1; memory = 2.GB } + withLabel: pysam { cpus = 2; memory = 4.GB } + withLabel: fastqc { cpus = {2 * task.attempt}; memory = {4.GB * task.attempt } ; maxRetries = 3 ; errorStrategy = { task.exitStatus in 130..140 ? 'retry' : 'terminate' }; } + withLabel: multiqc { cpus = 4; memory = 4.GB } + withLabel: nanoplot { cpus = 8; memory = 8.GB } + withLabel: quast { cpus = 8; memory = 8.GB } } From 24d6493bf4090313745de62bfa07b1297ae2d6f6 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Mon, 12 Aug 2024 16:41:35 +0200 Subject: [PATCH 2/7] increase RAM further if fails --- configs/node.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/node.config b/configs/node.config index 24c5a45..d207d91 100644 --- a/configs/node.config +++ b/configs/node.config @@ -1,6 +1,6 @@ process { - withLabel: minimap2 { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 3 ; errorStrategy = { task.exitStatus in 1 ? 'retry' : 'terminate' }; } - withLabel: bbmap { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 3 ; errorStrategy = { task.exitStatus in 1 ? 'retry' : 'terminate' }; } + withLabel: minimap2 { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 4 ; errorStrategy = { task.exitStatus in 1 ? 'retry' : 'terminate' }; } + withLabel: bbmap { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 4 ; errorStrategy = { task.exitStatus in 1 ? 'retry' : 'terminate' }; } withLabel: smallTask { cpus = 1; memory = 2.GB } withLabel: pysam { cpus = 2; memory = 4.GB } withLabel: fastqc { cpus = {2 * task.attempt}; memory = {4.GB * task.attempt } ; maxRetries = 3 ; errorStrategy = { task.exitStatus in 130..140 ? 'retry' : 'terminate' }; } From 78f15e116836ed9921a8b51ad2d4eba390811f9c Mon Sep 17 00:00:00 2001 From: hoelzer Date: Tue, 13 Aug 2024 10:47:55 +0200 Subject: [PATCH 3/7] incr ram further when err --- configs/node.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/node.config b/configs/node.config index d207d91..556a4ce 100644 --- a/configs/node.config +++ b/configs/node.config @@ -1,6 +1,6 @@ process { withLabel: minimap2 { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 4 ; errorStrategy = { task.exitStatus in 1 ? 'retry' : 'terminate' }; } - withLabel: bbmap { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 4 ; errorStrategy = { task.exitStatus in 1 ? 'retry' : 'terminate' }; } + withLabel: bbmap { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 4 ; errorStrategy = { task.exitStatus in 1 || 137 ? 'retry' : 'terminate' }; } withLabel: smallTask { cpus = 1; memory = 2.GB } withLabel: pysam { cpus = 2; memory = 4.GB } withLabel: fastqc { cpus = {2 * task.attempt}; memory = {4.GB * task.attempt } ; maxRetries = 3 ; errorStrategy = { task.exitStatus in 130..140 ? 'retry' : 'terminate' }; } From a21402b4e1d5b3eec244f69fa06395b06ef9aee6 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Tue, 13 Aug 2024 11:21:14 +0200 Subject: [PATCH 4/7] incr ram further when err --- configs/node.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/node.config b/configs/node.config index 556a4ce..799869a 100644 --- a/configs/node.config +++ b/configs/node.config @@ -1,6 +1,6 @@ process { withLabel: minimap2 { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 4 ; errorStrategy = { task.exitStatus in 1 ? 'retry' : 'terminate' }; } - withLabel: bbmap { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 4 ; errorStrategy = { task.exitStatus in 1 || 137 ? 'retry' : 'terminate' }; } + withLabel: bbmap { cpus = {24 * task.attempt}; memory = {12.GB * task.attempt}; maxRetries = 5 ; errorStrategy = { task.exitStatus in 1 || 137 ? 'retry' : 'terminate' }; } withLabel: smallTask { cpus = 1; memory = 2.GB } withLabel: pysam { cpus = 2; memory = 4.GB } withLabel: fastqc { cpus = {2 * task.attempt}; memory = {4.GB * task.attempt } ; maxRetries = 3 ; errorStrategy = { task.exitStatus in 130..140 ? 'retry' : 'terminate' }; } From db5366d79c6e8f904913ef8a75e9940f59df2eb1 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Tue, 13 Aug 2024 11:48:50 +0200 Subject: [PATCH 5/7] incr ram further when err --- configs/node.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/node.config b/configs/node.config index 799869a..fca5709 100644 --- a/configs/node.config +++ b/configs/node.config @@ -1,6 +1,6 @@ process { withLabel: minimap2 { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 4 ; errorStrategy = { task.exitStatus in 1 ? 'retry' : 'terminate' }; } - withLabel: bbmap { cpus = {24 * task.attempt}; memory = {12.GB * task.attempt}; maxRetries = 5 ; errorStrategy = { task.exitStatus in 1 || 137 ? 'retry' : 'terminate' }; } + withLabel: bbmap { cpus = {250 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 5 ; errorStrategy = { task.exitStatus in 1 || 137 ? 'retry' : 'terminate' }; } withLabel: smallTask { cpus = 1; memory = 2.GB } withLabel: pysam { cpus = 2; memory = 4.GB } withLabel: fastqc { cpus = {2 * task.attempt}; memory = {4.GB * task.attempt } ; maxRetries = 3 ; errorStrategy = { task.exitStatus in 130..140 ? 'retry' : 'terminate' }; } From fc43bc526f8fa28e0b7196d15ea493072f1cb710 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Tue, 13 Aug 2024 11:49:57 +0200 Subject: [PATCH 6/7] incr ram further when err --- configs/node.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/node.config b/configs/node.config index fca5709..52b8b7b 100644 --- a/configs/node.config +++ b/configs/node.config @@ -1,6 +1,6 @@ process { withLabel: minimap2 { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 4 ; errorStrategy = { task.exitStatus in 1 ? 'retry' : 'terminate' }; } - withLabel: bbmap { cpus = {250 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 5 ; errorStrategy = { task.exitStatus in 1 || 137 ? 'retry' : 'terminate' }; } + withLabel: bbmap { cpus = {24 * task.attempt}; memory = {240.GB * task.attempt}; maxRetries = 5 ; errorStrategy = { task.exitStatus in 1 || 137 ? 'retry' : 'terminate' }; } withLabel: smallTask { cpus = 1; memory = 2.GB } withLabel: pysam { cpus = 2; memory = 4.GB } withLabel: fastqc { cpus = {2 * task.attempt}; memory = {4.GB * task.attempt } ; maxRetries = 3 ; errorStrategy = { task.exitStatus in 130..140 ? 'retry' : 'terminate' }; } From ed1b37e4f1cd1aee94cc081254def25d20dec7b1 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Tue, 13 Aug 2024 12:56:47 +0200 Subject: [PATCH 7/7] incr ram further when err --- configs/node.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/node.config b/configs/node.config index 52b8b7b..2c718bc 100644 --- a/configs/node.config +++ b/configs/node.config @@ -1,6 +1,6 @@ process { - withLabel: minimap2 { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 4 ; errorStrategy = { task.exitStatus in 1 ? 'retry' : 'terminate' }; } - withLabel: bbmap { cpus = {24 * task.attempt}; memory = {240.GB * task.attempt}; maxRetries = 5 ; errorStrategy = { task.exitStatus in 1 || 137 ? 'retry' : 'terminate' }; } + withLabel: minimap2 { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 4 ; errorStrategy = { task.exitStatus in 1 || 130..140 ? 'retry' : 'terminate' }; } + withLabel: bbmap { cpus = {24 * task.attempt}; memory = {24.GB * task.attempt}; maxRetries = 6 ; errorStrategy = { task.exitStatus in 1 || 130..140 ? 'retry' : 'terminate' }; } withLabel: smallTask { cpus = 1; memory = 2.GB } withLabel: pysam { cpus = 2; memory = 4.GB } withLabel: fastqc { cpus = {2 * task.attempt}; memory = {4.GB * task.attempt } ; maxRetries = 3 ; errorStrategy = { task.exitStatus in 130..140 ? 'retry' : 'terminate' }; }