Merge pull request #128 from cta-observatory/env-v093

Specify the sbatch walltime
cta-observatory · Feb 25, 2022 · 08047e7 · 08047e7
2 parents 6b3c82d + 153572c
commit 08047e7
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 13 deletions.
diff --git a/cfg/sequencer.cfg b/cfg/sequencer.cfg
@@ -67,10 +67,11 @@ proton: /path/to/DL2/proton_mc_testing.h5
 electron: /path/to/DL2/electron_mc_testing.h5
 
 [SLURM]
-PARTITION_PEDCALIB: short
-PARTITION_DATA: long
+PARTITION_PEDCALIB: short, long
+PARTITION_DATA: short, long
 MEMSIZE_PEDCALIB: 3GB
 MEMSIZE_DATA: 16GB
+WALLTIME: 1:15:00
 # Days from current day up to which the jobs are fetched from the queue.
 # Default is None (left empty).
 STARTTIME_DAYS_SACCT:

diff --git a/osa/job.py b/osa/job.py
@@ -343,7 +343,7 @@ def scheduler_env_variables(sequence, scheduler="slurm"):
 
     sbatch_parameters = [
         f"--job-name={sequence.jobname}",
-        "--cpus-per-task=1",
+        f"--time={cfg.get('SLURM', 'WALLTIME')}",
         f"--chdir={options.directory}",
         f"--output=log/Run{sequence.run:05d}.%4a_jobid_%A.out",
         f"--error=log/Run{sequence.run:05d}.%4a_jobid_%A.err",

diff --git a/osa/tests/test_jobs.py b/osa/tests/test_jobs.py
@@ -69,24 +69,24 @@ def test_scheduler_env_variables(sequence_list, running_analysis_dir):
     env_variables = scheduler_env_variables(first_sequence)
     assert env_variables == [
         '#SBATCH --job-name=LST1_01805',
-        '#SBATCH --cpus-per-task=1',
+        '#SBATCH --time=1:15:00',
         f'#SBATCH --chdir={running_analysis_dir}',
         '#SBATCH --output=log/Run01805.%4a_jobid_%A.out',
         '#SBATCH --error=log/Run01805.%4a_jobid_%A.err',
-        '#SBATCH --partition=short',
+        f'#SBATCH --partition={cfg.get("SLURM", "PARTITION_PEDCALIB")}',
         '#SBATCH --mem-per-cpu=3GB'
     ]
     # Extract the second sequence
     second_sequence = sequence_list[1]
     env_variables = scheduler_env_variables(second_sequence)
     assert env_variables == [
         '#SBATCH --job-name=LST1_01807',
-        '#SBATCH --cpus-per-task=1',
+        '#SBATCH --time=1:15:00',
         f'#SBATCH --chdir={running_analysis_dir}',
         '#SBATCH --output=log/Run01807.%4a_jobid_%A.out',
         '#SBATCH --error=log/Run01807.%4a_jobid_%A.err',
         '#SBATCH --array=0-10',
-        '#SBATCH --partition=long',
+        f'#SBATCH --partition={cfg.get("SLURM", "PARTITION_DATA")}',
         '#SBATCH --mem-per-cpu=16GB'
     ]
 
@@ -101,11 +101,11 @@ def test_job_header_template(sequence_list, running_analysis_dir):
     #!/bin/env python
 
     #SBATCH --job-name=LST1_01805
-    #SBATCH --cpus-per-task=1
+    #SBATCH --time=1:15:00
     #SBATCH --chdir={running_analysis_dir}
     #SBATCH --output=log/Run01805.%4a_jobid_%A.out
     #SBATCH --error=log/Run01805.%4a_jobid_%A.err
-    #SBATCH --partition=short
+    #SBATCH --partition={cfg.get('SLURM', 'PARTITION_PEDCALIB')}
     #SBATCH --mem-per-cpu=3GB""")
     assert header == output_string1
 
@@ -116,12 +116,12 @@ def test_job_header_template(sequence_list, running_analysis_dir):
     #!/bin/env python
     
     #SBATCH --job-name=LST1_01807
-    #SBATCH --cpus-per-task=1
+    #SBATCH --time=1:15:00
     #SBATCH --chdir={running_analysis_dir}
     #SBATCH --output=log/Run01807.%4a_jobid_%A.out
     #SBATCH --error=log/Run01807.%4a_jobid_%A.err
     #SBATCH --array=0-10
-    #SBATCH --partition=long
+    #SBATCH --partition={cfg.get('SLURM', 'PARTITION_DATA')}
     #SBATCH --mem-per-cpu=16GB""")
     assert header == output_string2
 
@@ -145,7 +145,7 @@ def test_create_job_template_scheduler(
     #!/bin/env python
 
     #SBATCH --job-name=LST1_01807
-    #SBATCH --cpus-per-task=1
+    #SBATCH --time=1:15:00
     #SBATCH --chdir={Path.cwd()}/test_osa/test_files0/running_analysis/20200117/v0.1.0
     #SBATCH --output=log/Run01807.%4a_jobid_%A.out
     #SBATCH --error=log/Run01807.%4a_jobid_%A.err
@@ -188,7 +188,7 @@ def test_create_job_template_scheduler(
         #!/bin/env python
 
         #SBATCH --job-name=LST1_01808
-        #SBATCH --cpus-per-task=1
+        #SBATCH --time=1:15:00
         #SBATCH --chdir={Path.cwd()}/test_osa/test_files0/running_analysis/20200117/v0.1.0
         #SBATCH --output=log/Run01808.%4a_jobid_%A.out
         #SBATCH --error=log/Run01808.%4a_jobid_%A.err