-
Notifications
You must be signed in to change notification settings - Fork 2
/
RA_SLURM.py
executable file
·121 lines (98 loc) · 4.39 KB
/
RA_SLURM.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
#!/imaging/local/software/miniconda/envs/mne0.23/bin/python
"""
==========================================
Submit sbatch jobs for WH Resolution analysis
==========================================
"""
print(__doc__)
import subprocess
from os import path as op
# wrapper to run python script via qsub. Python3
fname_wrap = op.join('/', 'home', 'olaf', 'MEG', 'WakemanHensonEMEG', 'Python2SLURM_ResolutionAtlas.sh')
# indices of subjects to process
subjs = range(0, 16)
job_list = \
[
#####
# COMPUTING and MORPHING
# #####
# Compute resolution metrics
{'N': 'RA_Metrics', # job name
'Py': 'RA_ResolutionMetrics', # Python script
'Cf': 'RA_config', # configuration script
'Ss': subjs, # subject indices
'mem': '4G', # memory for qsub process
'dep': ''},
# Compute contrasts among methods
{'N': 'RA_Diff', # job name
'Py': 'RA_ResolutionMetrics_differences', # Python script
'Cf': 'RA_config', # configuration script
'Ss': subjs, # subject indices
'mem': '1G', # memory for qsub process
'dep': 'RA_Metrics'},
# Morph STCs to fsaverage
{'N': 'RA_Mph', # job name
'Py': 'RA_MorphSTC', # Python script
'Cf': 'RA_config', # configuration script
'Ss': subjs, # subject indices
'mem': '1G', # memory for qsub process
'dep': 'RA_Diff'},
# Grand-average STCs
# {'N': 'RA_Avg', # job name
# 'Py': 'RA_AvgSTCs', # Python script
# 'Cf': 'RA_config', # configuration script
# 'Ss': [99], # subject indices
# 'mem': '1G', # memory for qsub process
# 'dep': ''},
]
# directory where python scripts are
dir_py = op.join('/', 'home', 'olaf', 'MEG', 'WakemanHensonEMEG', 'ResolutionAtlas')
# directory for qsub output
dir_qsub = op.join('/', 'home', 'olaf', 'MEG', 'WakemanHensonEMEG', 'ResolutionAtlas', 'sbatch_out')
# keep track of qsub Job IDs
Job_IDs = {}
for job in job_list:
for Ss in job['Ss']:
Ss = str(Ss) # turn into string for filenames etc.
N = Ss + job['N']
Py = op.join(dir_py, job['Py'])
Cf = job['Cf']
mem = job['mem']
# files for sbatch output
file_out = op.join(dir_qsub, job['N'] + '_' + Cf + '-%s.out' % str(Ss))
file_err = op.join(dir_qsub, job['N'] + '_' + Cf + '-%s.err' % str(Ss))
# if job dependent of previous job, get Job ID and produce command
if 'dep' in job: # if dependency on previous job specified
if job['dep'] == '':
dep_str = ''
else:
job_id = Job_IDs[Ss + job['dep'], Ss]
dep_str = '--dependency=afterok:%s' % (job_id)
else:
dep_str = ''
if 'node' in job: # if node constraint present (e.g. Maxfilter)
node_str = job['node']
else:
node_str = ''
if 'var' in job: # if variables for python script specified
var_str = job['var']
else:
var_str = ''
# sbatch command string to be executed
sbatch_cmd = 'sbatch \
-o %s \
-e %s \
--export=pycmd="%s.py %s",subj_idx=%s,var=%s \
--mem-per-cpu=%s -t 1-00:00:00 -J %s %s\
%s' \
% (file_out, file_err, Py, Cf, Ss, var_str, mem, N, dep_str, fname_wrap)
# format string for display
print_str = sbatch_cmd.replace(' ' * 25, ' ')
print('\n%s\n' % print_str)
# execute qsub command
proc = subprocess.Popen(sbatch_cmd, stdout=subprocess.PIPE, shell=True)
# get linux output
(out, err) = proc.communicate()
# keep track of Job IDs from sbatch, for dependencies
Job_IDs[N, Ss] = str(int(out.split()[-1]))
# Done