From 7388d3d5d6d60e5304ae98f7d356f82bdd528e93 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 16 Apr 2020 16:53:06 -0400 Subject: [PATCH 01/24] Initial work. --- heudiconv/cli/run.py | 156 +++++++++++++++++++++++-------------------- 1 file changed, 83 insertions(+), 73 deletions(-) diff --git a/heudiconv/cli/run.py b/heudiconv/cli/run.py index f6af3d1d..7a516f17 100644 --- a/heudiconv/cli/run.py +++ b/heudiconv/cli/run.py @@ -46,7 +46,8 @@ def _pdb_excepthook(type, value, tb): sys.excepthook = _pdb_excepthook -def process_extra_commands(outdir, args): +def process_extra_commands(outdir, command, files, dicom_dir_template, + heuristic, session, subjs, grouping): """ Perform custom command instead of regular operations. Supported commands: ['treat-json', 'ls', 'populate-templates'] @@ -58,17 +59,17 @@ def process_extra_commands(outdir, args): args : Namespace arguments """ - if args.command == 'treat-jsons': - for f in args.files: + if command == 'treat-jsons': + for f in files: treat_infofile(f) - elif args.command == 'ls': - ensure_heuristic_arg(args) - heuristic = load_heuristic(args.heuristic) + elif command == 'ls': + ensure_heuristic_arg(heuristic) + heuristic = load_heuristic(heuristic) heuristic_ls = getattr(heuristic, 'ls', None) - for f in args.files: + for f in files: study_sessions = get_study_sessions( - args.dicom_dir_template, [f], heuristic, outdir, - args.session, args.subjs, grouping=args.grouping) + dicom_dir_template, [f], heuristic, outdir, + session, subjs, grouping=grouping) print(f) for study_session, sequences in study_sessions.items(): suf = '' @@ -78,29 +79,29 @@ def process_extra_commands(outdir, args): "\t%s %d sequences%s" % (str(study_session), len(sequences), suf) ) - elif args.command == 'populate-templates': - ensure_heuristic_arg(args) - heuristic = load_heuristic(args.heuristic) - for f in args.files: + elif command == 'populate-templates': + ensure_heuristic_arg(heuristic) + heuristic = load_heuristic(heuristic) + for f in files: populate_bids_templates(f, getattr(heuristic, 'DEFAULT_FIELDS', {})) - elif args.command == 'sanitize-jsons': - tuneup_bids_json_files(args.files) - elif args.command == 'heuristics': + elif command == 'sanitize-jsons': + tuneup_bids_json_files(files) + elif command == 'heuristics': from ..utils import get_known_heuristics_with_descriptions for name_desc in get_known_heuristics_with_descriptions().items(): print("- %s: %s" % name_desc) - elif args.command == 'heuristic-info': - ensure_heuristic_arg(args) + elif command == 'heuristic-info': + ensure_heuristic_arg(heuristic) from ..utils import get_heuristic_description - print(get_heuristic_description(args.heuristic, full=True)) + print(get_heuristic_description(heuristic, full=True)) else: - raise ValueError("Unknown command %s", args.command) + raise ValueError("Unknown command %s", command) return -def ensure_heuristic_arg(args): +def ensure_heuristic_arg(heuristic=None): from ..utils import get_known_heuristic_names - if not args.heuristic: + if not heuristic: raise ValueError("Specify heuristic using -f. Known are: %s" % ', '.join(get_known_heuristic_names())) @@ -113,25 +114,9 @@ def main(argv=None): lgr.warning("Nothing to be done - displaying usage help") parser.print_help() sys.exit(1) - # To be done asap so anything random is deterministic - if args.random_seed is not None: - import random - random.seed(args.random_seed) - import numpy - numpy.random.seed(args.random_seed) - # Ensure only supported bids options are passed - if args.debug: - lgr.setLevel(logging.DEBUG) - # Should be possible but only with a single subject -- will be used to - # override subject deduced from the DICOMs - if args.files and args.subjs and len(args.subjs) > 1: - raise ValueError( - "Unable to processes multiple `--subjects` with files" - ) - if args.debug: - setup_exceptionhook() - process_args(args) + kwargs = vars(args) + process_args(**kwargs) def get_parser(): @@ -244,14 +229,38 @@ def get_parser(): return parser -def process_args(args): +def process_args(outdir, command=None, heuristic=None, queue=None, files=None, + subjs=None, queue_args=None, dicom_dir_template=None, + session=None, grouping=None, locator=None, anon_cmd=None, + conv_outdir=None, converter=None, with_prov=None, + bids_options=None, minmeta=None, overwrite=False, + dcmconfig=None, datalad=False, random_seed=None, debug=False): """Given a structure of arguments from the parser perform computation""" + # To be done asap so anything random is deterministic + if random_seed is not None: + import random + random.seed(random_seed) + import numpy + numpy.random.seed(random_seed) + # Ensure only supported bids options are passed + if debug: + lgr.setLevel(logging.DEBUG) + # Should be possible but only with a single subject -- will be used to + # override subject deduced from the DICOMs + if files and subjs and len(subjs) > 1: + raise ValueError( + "Unable to processes multiple `--subjects` with files" + ) + + if debug: + setup_exceptionhook() + # Deal with provided files or templates # pre-process provided list of files and possibly sort into groups/sessions # Group files per each study/sid/session - outdir = op.abspath(args.outdir) + outdir = op.abspath(outdir) try: import etelemetry @@ -264,27 +273,28 @@ def process_args(args): version=__version__, latest=latest["version"])) - if args.command: - process_extra_commands(outdir, args) + if command: + process_extra_commands(outdir, command, files, dicom_dir_template, + heuristic, session, subjs, grouping) return # # Load heuristic -- better do it asap to make sure it loads correctly # - if not args.heuristic: + if not heuristic: raise RuntimeError("No heuristic specified - add to arguments and rerun") - if args.queue: - lgr.info("Queuing %s conversion", args.queue) - iterarg, iterables = ("files", len(args.files)) if args.files else \ - ("subjects", len(args.subjs)) - queue_conversion(args.queue, iterarg, iterables, args.queue_args) + if queue: + lgr.info("Queuing %s conversion", queue) + iterarg, iterables = ("files", len(files)) if files else \ + ("subjects", len(subjs)) + queue_conversion(queue, iterarg, iterables, queue_args) sys.exit(0) - heuristic = load_heuristic(args.heuristic) + heuristic = load_heuristic(heuristic) - study_sessions = get_study_sessions(args.dicom_dir_template, args.files, - heuristic, outdir, args.session, - args.subjs, grouping=args.grouping) + study_sessions = get_study_sessions(dicom_dir_template, files, + heuristic, outdir, session, + subjs, grouping=grouping) # extract tarballs, and replace their entries with expanded lists of files # TODO: we might need to sort so sessions are ordered??? @@ -295,10 +305,10 @@ def process_args(args): for (locator, session, sid), files_or_seqinfo in study_sessions.items(): # Allow for session to be overloaded from command line - if args.session is not None: - session = args.session - if args.locator is not None: - locator = args.locator + if session is not None: + session = session + if locator is not None: + locator = locator if not len(files_or_seqinfo): raise ValueError("nothing to process?") # that is how life is ATM :-/ since we don't do sorting if subj @@ -315,22 +325,22 @@ def process_args(args): lgr.warning("Skipping unknown locator dataset") continue - anon_sid = anonymize_sid(sid, args.anon_cmd) if args.anon_cmd else None - if args.anon_cmd: + anon_sid = anonymize_sid(sid, anon_cmd) if anon_cmd else None + if anon_cmd: lgr.info('Anonymized {} to {}'.format(sid, anon_sid)) study_outdir = op.join(outdir, locator or '') - anon_outdir = args.conv_outdir or outdir + anon_outdir = conv_outdir or outdir anon_study_outdir = op.join(anon_outdir, locator or '') # TODO: --datalad cmdline option, which would take care about initiating # the outdir -> study_outdir datasets if not yet there - if args.datalad: + if datalad: from ..external.dlad import prepare_datalad dlad_sid = sid if not anon_sid else anon_sid dl_msg = prepare_datalad(anon_study_outdir, anon_outdir, dlad_sid, session, seqinfo, dicoms, - args.bids_options) + bids_options) lgr.info("PROCESSING STARTS: {0}".format( str(dict(subject=sid, outdir=study_outdir, session=session)))) @@ -339,22 +349,22 @@ def process_args(args): dicoms, study_outdir, heuristic, - converter=args.converter, + converter=converter, anon_sid=anon_sid, anon_outdir=anon_study_outdir, - with_prov=args.with_prov, + with_prov=with_prov, ses=session, - bids_options=args.bids_options, + bids_options=bids_options, seqinfo=seqinfo, - min_meta=args.minmeta, - overwrite=args.overwrite, - dcmconfig=args.dcmconfig, - grouping=args.grouping,) + min_meta=minmeta, + overwrite=overwrite, + dcmconfig=dcmconfig, + grouping=grouping,) lgr.info("PROCESSING DONE: {0}".format( str(dict(subject=sid, outdir=study_outdir, session=session)))) - if args.datalad: + if datalad: from ..external.dlad import add_to_datalad msg = "Converted subject %s" % dl_msg # TODO: whenever propagate to supers work -- do just @@ -362,9 +372,9 @@ def process_args(args): # also in batch mode might fail since we have no locking ATM # and theoretically no need actually to save entire study # we just need that - add_to_datalad(outdir, study_outdir, msg, args.bids_options) + add_to_datalad(outdir, study_outdir, msg, bids_options) - # if args.bids: + # if bids: # # Let's populate BIDS templates for folks to take care about # for study_outdir in processed_studydirs: # populate_bids_templates(study_outdir) From 5ab426bde8c53ec2b0913f9ae954e145a4daef66 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 16 Apr 2020 17:09:15 -0400 Subject: [PATCH 02/24] Reorder arguments and rename function. --- heudiconv/cli/run.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/heudiconv/cli/run.py b/heudiconv/cli/run.py index 7a516f17..d516bf8c 100644 --- a/heudiconv/cli/run.py +++ b/heudiconv/cli/run.py @@ -116,7 +116,7 @@ def main(argv=None): sys.exit(1) kwargs = vars(args) - process_args(**kwargs) + heudiconv_workflow(**kwargs) def get_parser(): @@ -229,12 +229,14 @@ def get_parser(): return parser -def process_args(outdir, command=None, heuristic=None, queue=None, files=None, - subjs=None, queue_args=None, dicom_dir_template=None, - session=None, grouping=None, locator=None, anon_cmd=None, - conv_outdir=None, converter=None, with_prov=None, - bids_options=None, minmeta=None, overwrite=False, - dcmconfig=None, datalad=False, random_seed=None, debug=False): +def heudiconv_workflow(dicom_dir_template=None, files=None, + subjs=None, converter='dcm2niix', outdir='.', + locator=None, conv_outdir=None, anon_cmd=None, + heuristic=None, with_prov=False, session=None, + bids_options=None, overwrite=False, datalad=False, + debug=False, command=None, grouping='studyUID', + minmeta=False, random_seed=None, dcmconfig=None, + queue=None, queue_args=None): """Given a structure of arguments from the parser perform computation""" # To be done asap so anything random is deterministic From b7d19a1f6ef873b5fdec73d0fb9ad3e68efb4617 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 16 Apr 2020 17:54:16 -0400 Subject: [PATCH 03/24] Document workflow function. --- heudiconv/cli/run.py | 86 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/heudiconv/cli/run.py b/heudiconv/cli/run.py index d516bf8c..19c80074 100644 --- a/heudiconv/cli/run.py +++ b/heudiconv/cli/run.py @@ -237,7 +237,91 @@ def heudiconv_workflow(dicom_dir_template=None, files=None, debug=False, command=None, grouping='studyUID', minmeta=False, random_seed=None, dcmconfig=None, queue=None, queue_args=None): - """Given a structure of arguments from the parser perform computation""" + """Run the HeuDiConv conversion workflow. + + Parameters + ---------- + dicom_dir_template : str or None, optional + Location of dicomdir that can be indexed with subject id + {subject} and session {session}. Tarballs (can be compressed) + are supported in addition to directory. All matching tarballs + for a subject are extracted and their content processed in a + single pass. If multiple tarballs are found, each is assumed to + be a separate session and the 'session' argument is ignored. + Mutually exclusive with 'files'. Default is None. + files : list or None, optional + Files (tarballs, dicoms) or directories containing files to + process. Mutually exclusive with 'dicom_dir_template'. Default is None. + subjs : list or None, optional + List of subjects - required for dicom template. If not + provided, DICOMS would first be "sorted" and subject IDs + deduced by the heuristic. Default is None. + converter : {'dcm2niix', None}, optional + Tool to use for DICOM conversion. Setting to None disables + the actual conversion step -- useful for testing heuristics. + Default is None. + outdir : str, optional + Output directory for conversion setup (for further + customization and future reference. This directory will refer + to non-anonymized subject IDs. + Default is '.' (current working directory). + locator : str or 'unknown' or None, optional + Study path under outdir. If provided, it overloads the value + provided by the heuristic. If 'datalad=True', every + directory within locator becomes a super-dataset thus + establishing a hierarchy. Setting to "unknown" will skip that + dataset. Default is None. + conv_outdir : str or None, optional + Output directory for converted files. By default this is + identical to --outdir. This option is most useful in + combination with 'anon_cmd'. Default is None. + anon_cmd : str or None, optional + Command to run to convert subject IDs used for DICOMs to + anonymized IDs. Such command must take a single argument and + return a single anonymized ID. Also see 'conv_outdir'. Default is None. + heuristic : str or None, optional + Name of a known heuristic or path to the Python script containing + heuristic. Default is None. + with_prov : bool, optional + Store additional provenance information. Requires python-rdflib. + Default is False. + session : str or None, optional + Session for longitudinal study_sessions. Default is None. + bids_options : str or None, optional + Flag for output into BIDS structure. Can also take BIDS- + specific options, e.g., --bids notop. The only currently + supported options is "notop", which skips creation of + top-level BIDS files. This is useful when running in batch + mode to prevent possible race conditions. Default is None. + overwrite : bool, optional + Overwrite existing converted files. Default is False. + datalad : bool, optional + Store the entire collection as DataLad dataset(s). Small files + will be committed directly to git, while large to annex. New + version (6) of annex repositories will be used in a "thin" + mode so it would look to mortals as just any other regular + directory (i.e. no symlinks to under .git/annex). For now just + for BIDS mode. Default is False. + debug : bool, optional + Do not catch exceptions and show exception traceback. Default is False. + command : {'heuristics', 'heuristic-info', 'ls', 'populate-templates', + 'sanitize-jsons', 'treat-jsons', None}, optional + Custom action to be performed on provided files instead of regular + operation. Default is None. + grouping : {'studyUID', 'accession_number', 'all', 'custom'}, optional + How to group dicoms. Default is 'studyUID'. + minmeta : bool, optional + Exclude dcmstack meta information in sidecar jsons. Default is False. + random_seed : int or None, optional + Random seed to initialize RNG. Default is None. + dcmconfig : str or None, optional + JSON file for additional dcm2niix configuration. Default is None. + queue : {'SLURM', None}, optional + Batch system to submit jobs in parallel. Default is None. + queue_args : str or None, optional + Additional queue arguments passed as single string of space-separated + Argument=Value pairs. Default is None. + """ # To be done asap so anything random is deterministic if random_seed is not None: From 61f7d48ce928b1b8c733e16e5a17dbf915014e68 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 16 Apr 2020 17:54:31 -0400 Subject: [PATCH 04/24] Fix spacing. --- heudiconv/cli/run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/heudiconv/cli/run.py b/heudiconv/cli/run.py index 19c80074..bbb06714 100644 --- a/heudiconv/cli/run.py +++ b/heudiconv/cli/run.py @@ -20,9 +20,9 @@ def is_interactive(): - """Return True if all in/outs are tty""" - # TODO: check on windows if hasattr check would work correctly and add value: - return sys.stdin.isatty() and sys.stdout.isatty() and sys.stderr.isatty() + """Return True if all in/outs are tty""" + # TODO: check on windows if hasattr check would work correctly and add value: + return sys.stdin.isatty() and sys.stdout.isatty() and sys.stderr.isatty() def setup_exceptionhook(): From 38f323ee38449caddf1051c952deca46467e686d Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 16 Apr 2020 18:04:42 -0400 Subject: [PATCH 05/24] Move workflow into new workflows module. --- heudiconv/cli/run.py | 579 +++++++++---------------------------- heudiconv/workflows/run.py | 343 ++++++++++++++++++++++ 2 files changed, 484 insertions(+), 438 deletions(-) create mode 100644 heudiconv/workflows/run.py diff --git a/heudiconv/cli/run.py b/heudiconv/cli/run.py index bbb06714..15c9588b 100644 --- a/heudiconv/cli/run.py +++ b/heudiconv/cli/run.py @@ -1,109 +1,14 @@ #!/usr/bin/env python import os -import os.path as op from argparse import ArgumentParser import sys - -from .. import __version__, __packagename__ -from ..parser import get_study_sessions -from ..utils import load_heuristic, anonymize_sid, treat_infofile, SeqInfo -from ..convert import prep_conversion -from ..bids import populate_bids_templates, tuneup_bids_json_files -from ..queue import queue_conversion - -import inspect import logging -lgr = logging.getLogger(__name__) - -INIT_MSG = "Running {packname} version {version} latest {latest}".format - - -def is_interactive(): - """Return True if all in/outs are tty""" - # TODO: check on windows if hasattr check would work correctly and add value: - return sys.stdin.isatty() and sys.stdout.isatty() and sys.stderr.isatty() - - -def setup_exceptionhook(): - """ - Overloads default sys.excepthook with our exceptionhook handler. - - If interactive, our exceptionhook handler will invoke pdb.post_mortem; - if not interactive, then invokes default handler. - """ - def _pdb_excepthook(type, value, tb): - if is_interactive(): - import traceback - import pdb - traceback.print_exception(type, value, tb) - # print() - pdb.post_mortem(tb) - else: - lgr.warning( - "We cannot setup exception hook since not in interactive mode") - - sys.excepthook = _pdb_excepthook +from .. import __version__ +from ..workflows import heudiconv_workflow -def process_extra_commands(outdir, command, files, dicom_dir_template, - heuristic, session, subjs, grouping): - """ - Perform custom command instead of regular operations. Supported commands: - ['treat-json', 'ls', 'populate-templates'] - - Parameters - ---------- - outdir : String - Output directory - args : Namespace - arguments - """ - if command == 'treat-jsons': - for f in files: - treat_infofile(f) - elif command == 'ls': - ensure_heuristic_arg(heuristic) - heuristic = load_heuristic(heuristic) - heuristic_ls = getattr(heuristic, 'ls', None) - for f in files: - study_sessions = get_study_sessions( - dicom_dir_template, [f], heuristic, outdir, - session, subjs, grouping=grouping) - print(f) - for study_session, sequences in study_sessions.items(): - suf = '' - if heuristic_ls: - suf += heuristic_ls(study_session, sequences) - print( - "\t%s %d sequences%s" - % (str(study_session), len(sequences), suf) - ) - elif command == 'populate-templates': - ensure_heuristic_arg(heuristic) - heuristic = load_heuristic(heuristic) - for f in files: - populate_bids_templates(f, getattr(heuristic, 'DEFAULT_FIELDS', {})) - elif command == 'sanitize-jsons': - tuneup_bids_json_files(files) - elif command == 'heuristics': - from ..utils import get_known_heuristics_with_descriptions - for name_desc in get_known_heuristics_with_descriptions().items(): - print("- %s: %s" % name_desc) - elif command == 'heuristic-info': - ensure_heuristic_arg(heuristic) - from ..utils import get_heuristic_description - print(get_heuristic_description(heuristic, full=True)) - else: - raise ValueError("Unknown command %s", command) - return - - -def ensure_heuristic_arg(heuristic=None): - from ..utils import get_known_heuristic_names - if not heuristic: - raise ValueError("Specify heuristic using -f. Known are: %s" - % ', '.join(get_known_heuristic_names())) +lgr = logging.getLogger(__name__) def main(argv=None): @@ -125,349 +30,147 @@ def get_parser(): parser = ArgumentParser(description=docstr) parser.add_argument('--version', action='version', version=__version__) group = parser.add_mutually_exclusive_group() - group.add_argument('-d', '--dicom_dir_template', dest='dicom_dir_template', - help='location of dicomdir that can be indexed with ' - 'subject id {subject} and session {session}. Tarballs ' - '(can be compressed) are supported in addition to ' - 'directory. All matching tarballs for a subject are ' - 'extracted and their content processed in a single ' - 'pass. If multiple tarballs are found, each is ' - 'assumed to be a separate session and the --ses ' - 'argument is ignored. Note that you might need to ' - 'surround the value with quotes to avoid {...} being ' - 'considered by shell') - group.add_argument('--files', nargs='*', - help='Files (tarballs, dicoms) or directories ' - 'containing files to process. Cannot be provided if ' - 'using --dicom_dir_template.') - parser.add_argument('-s', '--subjects', dest='subjs', type=str, nargs='*', - help='list of subjects - required for dicom template. ' - 'If not provided, DICOMS would first be "sorted" and ' - 'subject IDs deduced by the heuristic') - parser.add_argument('-c', '--converter', - choices=('dcm2niix', 'none'), default='dcm2niix', - help='tool to use for DICOM conversion. Setting to ' - '"none" disables the actual conversion step -- useful' - 'for testing heuristics.') - parser.add_argument('-o', '--outdir', default=os.getcwd(), - help='output directory for conversion setup (for ' - 'further customization and future reference. This ' - 'directory will refer to non-anonymized subject IDs') - parser.add_argument('-l', '--locator', default=None, - help='study path under outdir. If provided, ' - 'it overloads the value provided by the heuristic. ' - 'If --datalad is enabled, every directory within ' - 'locator becomes a super-dataset thus establishing a ' - 'hierarchy. Setting to "unknown" will skip that dataset') - parser.add_argument('-a', '--conv-outdir', default=None, - help='output directory for converted files. By default ' - 'this is identical to --outdir. This option is most ' - 'useful in combination with --anon-cmd') - parser.add_argument('--anon-cmd', default=None, - help='command to run to convert subject IDs used for ' - 'DICOMs to anonymized IDs. Such command must take a ' - 'single argument and return a single anonymized ID. ' - 'Also see --conv-outdir') - parser.add_argument('-f', '--heuristic', dest='heuristic', - help='Name of a known heuristic or path to the Python' - 'script containing heuristic') - parser.add_argument('-p', '--with-prov', action='store_true', - help='Store additional provenance information. ' - 'Requires python-rdflib.') - parser.add_argument('-ss', '--ses', dest='session', default=None, - help='session for longitudinal study_sessions, default ' - 'is none') - parser.add_argument('-b', '--bids', nargs='*', - metavar=('BIDSOPTION1', 'BIDSOPTION2'), - choices=['notop'], - dest='bids_options', - help='flag for output into BIDS structure. Can also ' - 'take bids specific options, e.g., --bids notop.' - 'The only currently supported options is' - '"notop", which skips creation of top-level bids ' - 'files. This is useful when running in batch mode to ' - 'prevent possible race conditions.') - parser.add_argument('--overwrite', action='store_true', default=False, - help='flag to allow overwriting existing converted files') - parser.add_argument('--datalad', action='store_true', - help='Store the entire collection as DataLad ' - 'dataset(s). Small files will be committed directly to ' - 'git, while large to annex. New version (6) of annex ' - 'repositories will be used in a "thin" mode so it ' - 'would look to mortals as just any other regular ' - 'directory (i.e. no symlinks to under .git/annex). ' - 'For now just for BIDS mode.') - parser.add_argument('--dbg', action='store_true', dest='debug', - help='Do not catch exceptions and show exception ' - 'traceback') - parser.add_argument('--command', - choices=( - 'heuristics', 'heuristic-info', - 'ls', 'populate-templates', - 'sanitize-jsons', 'treat-jsons', - ), - help='custom actions to be performed on provided ' - 'files instead of regular operation.') - parser.add_argument('-g', '--grouping', default='studyUID', - choices=('studyUID', 'accession_number', 'all', 'custom'), - help='How to group dicoms (default: by studyUID)') - parser.add_argument('--minmeta', action='store_true', - help='Exclude dcmstack meta information in sidecar ' - 'jsons') - parser.add_argument('--random-seed', type=int, default=None, - help='Random seed to initialize RNG') - parser.add_argument('--dcmconfig', default=None, - help='JSON file for additional dcm2niix configuration') + group.add_argument( + '-d', '--dicom_dir_template', + dest='dicom_dir_template', + help='Location of dicomdir that can be indexed with subject id ' + '{subject} and session {session}. Tarballs (can be compressed) ' + 'are supported in addition to directory. All matching tarballs ' + 'for a subject are extracted and their content processed in a ' + 'single pass. If multiple tarballs are found, each is assumed to ' + 'be a separate session and the --ses argument is ignored. Note ' + 'that you might need to surround the value with quotes to avoid ' + '{...} being considered by shell') + group.add_argument( + '--files', + nargs='*', + help='Files (tarballs, dicoms) or directories containing files to ' + 'process. Cannot be provided if using --dicom_dir_template.') + parser.add_argument( + '-s', '--subjects', + dest='subjs', + type=str, + nargs='*', + help='List of subjects - required for dicom template. If not ' + 'provided, DICOMS would first be "sorted" and subject IDs ' + 'deduced by the heuristic.') + parser.add_argument( + '-c', '--converter', + choices=('dcm2niix', 'none'), + default='dcm2niix', + help='Tool to use for DICOM conversion. Setting to "none" disables ' + 'the actual conversion step -- useful for testing heuristics.') + parser.add_argument( + '-o', '--outdir', + default=os.getcwd(), + help='Output directory for conversion setup (for further ' + 'customization and future reference. This directory will refer ' + 'to non-anonymized subject IDs.') + parser.add_argument( + '-l', '--locator', + default=None, + help='Study path under outdir. If provided, it overloads the value ' + 'provided by the heuristic. If --datalad is enabled, every ' + 'directory within locator becomes a super-dataset thus ' + 'establishing a hierarchy. Setting to "unknown" will skip that ' + 'dataset.') + parser.add_argument( + '-a', '--conv-outdir', + default=None, + help='Output directory for converted files. By default this is ' + 'identical to --outdir. This option is most useful in ' + 'combination with --anon-cmd.') + parser.add_argument( + '--anon-cmd', + default=None, + help='Command to run to convert subject IDs used for DICOMs to ' + 'anonymized IDs. Such command must take a single argument and ' + 'return a single anonymized ID. Also see --conv-outdir.') + parser.add_argument( + '-f', '--heuristic', + dest='heuristic', + help='Name of a known heuristic or path to the Python script ' + 'containing heuristic.') + parser.add_argument( + '-p', '--with-prov', + action='store_true', + help='Store additional provenance information. Requires python-rdflib.') + parser.add_argument( + '-ss', '--ses', + dest='session', + default=None, + help='Session for longitudinal study_sessions. Default is None.') + parser.add_argument( + '-b', '--bids', + nargs='*', + metavar=('BIDSOPTION1', 'BIDSOPTION2'), + choices=['notop'], + dest='bids_options', + help='Flag for output into BIDS structure. Can also take BIDS-' + 'specific options, e.g., --bids notop. The only currently ' + 'supported options is "notop", which skips creation of ' + 'top-level BIDS files. This is useful when running in batch ' + 'mode to prevent possible race conditions.') + parser.add_argument( + '--overwrite', + action='store_true', + default=False, + help='Overwrite existing converted files.') + parser.add_argument( + '--datalad', + action='store_true', + help='Store the entire collection as DataLad dataset(s). Small files ' + 'will be committed directly to git, while large to annex. New ' + 'version (6) of annex repositories will be used in a "thin" ' + 'mode so it would look to mortals as just any other regular ' + 'directory (i.e. no symlinks to under .git/annex). For now just ' + 'for BIDS mode.') + parser.add_argument( + '--dbg', + action='store_true', + dest='debug', + help='Do not catch exceptions and show exception traceback.') + parser.add_argument( + '--command', + choices=( + 'heuristics', 'heuristic-info', + 'ls', 'populate-templates', + 'sanitize-jsons', 'treat-jsons', + ), + help='Custom action to be performed on provided files instead of ' + 'regular operation.') + parser.add_argument( + '-g', '--grouping', + default='studyUID', + choices=('studyUID', 'accession_number', 'all', 'custom'), + help='How to group dicoms (default: by studyUID).') + parser.add_argument( + '--minmeta', + action='store_true', + help='Exclude dcmstack meta information in sidecar jsons.') + parser.add_argument( + '--random-seed', + type=int, + default=None, + help='Random seed to initialize RNG.') + parser.add_argument( + '--dcmconfig', + default=None, + help='JSON file for additional dcm2niix configuration.') submission = parser.add_argument_group('Conversion submission options') - submission.add_argument('-q', '--queue', choices=("SLURM", None), - default=None, - help='batch system to submit jobs in parallel') - submission.add_argument('--queue-args', dest='queue_args', default=None, - help='Additional queue arguments passed as ' - 'single string of Argument=Value pairs space ' - 'separated.') + submission.add_argument( + '-q', '--queue', + choices=("SLURM", None), + default=None, + help='Batch system to submit jobs in parallel.') + submission.add_argument( + '--queue-args', + dest='queue_args', + default=None, + help='Additional queue arguments passed as a single string of ' + 'space-separated Argument=Value pairs.') return parser -def heudiconv_workflow(dicom_dir_template=None, files=None, - subjs=None, converter='dcm2niix', outdir='.', - locator=None, conv_outdir=None, anon_cmd=None, - heuristic=None, with_prov=False, session=None, - bids_options=None, overwrite=False, datalad=False, - debug=False, command=None, grouping='studyUID', - minmeta=False, random_seed=None, dcmconfig=None, - queue=None, queue_args=None): - """Run the HeuDiConv conversion workflow. - - Parameters - ---------- - dicom_dir_template : str or None, optional - Location of dicomdir that can be indexed with subject id - {subject} and session {session}. Tarballs (can be compressed) - are supported in addition to directory. All matching tarballs - for a subject are extracted and their content processed in a - single pass. If multiple tarballs are found, each is assumed to - be a separate session and the 'session' argument is ignored. - Mutually exclusive with 'files'. Default is None. - files : list or None, optional - Files (tarballs, dicoms) or directories containing files to - process. Mutually exclusive with 'dicom_dir_template'. Default is None. - subjs : list or None, optional - List of subjects - required for dicom template. If not - provided, DICOMS would first be "sorted" and subject IDs - deduced by the heuristic. Default is None. - converter : {'dcm2niix', None}, optional - Tool to use for DICOM conversion. Setting to None disables - the actual conversion step -- useful for testing heuristics. - Default is None. - outdir : str, optional - Output directory for conversion setup (for further - customization and future reference. This directory will refer - to non-anonymized subject IDs. - Default is '.' (current working directory). - locator : str or 'unknown' or None, optional - Study path under outdir. If provided, it overloads the value - provided by the heuristic. If 'datalad=True', every - directory within locator becomes a super-dataset thus - establishing a hierarchy. Setting to "unknown" will skip that - dataset. Default is None. - conv_outdir : str or None, optional - Output directory for converted files. By default this is - identical to --outdir. This option is most useful in - combination with 'anon_cmd'. Default is None. - anon_cmd : str or None, optional - Command to run to convert subject IDs used for DICOMs to - anonymized IDs. Such command must take a single argument and - return a single anonymized ID. Also see 'conv_outdir'. Default is None. - heuristic : str or None, optional - Name of a known heuristic or path to the Python script containing - heuristic. Default is None. - with_prov : bool, optional - Store additional provenance information. Requires python-rdflib. - Default is False. - session : str or None, optional - Session for longitudinal study_sessions. Default is None. - bids_options : str or None, optional - Flag for output into BIDS structure. Can also take BIDS- - specific options, e.g., --bids notop. The only currently - supported options is "notop", which skips creation of - top-level BIDS files. This is useful when running in batch - mode to prevent possible race conditions. Default is None. - overwrite : bool, optional - Overwrite existing converted files. Default is False. - datalad : bool, optional - Store the entire collection as DataLad dataset(s). Small files - will be committed directly to git, while large to annex. New - version (6) of annex repositories will be used in a "thin" - mode so it would look to mortals as just any other regular - directory (i.e. no symlinks to under .git/annex). For now just - for BIDS mode. Default is False. - debug : bool, optional - Do not catch exceptions and show exception traceback. Default is False. - command : {'heuristics', 'heuristic-info', 'ls', 'populate-templates', - 'sanitize-jsons', 'treat-jsons', None}, optional - Custom action to be performed on provided files instead of regular - operation. Default is None. - grouping : {'studyUID', 'accession_number', 'all', 'custom'}, optional - How to group dicoms. Default is 'studyUID'. - minmeta : bool, optional - Exclude dcmstack meta information in sidecar jsons. Default is False. - random_seed : int or None, optional - Random seed to initialize RNG. Default is None. - dcmconfig : str or None, optional - JSON file for additional dcm2niix configuration. Default is None. - queue : {'SLURM', None}, optional - Batch system to submit jobs in parallel. Default is None. - queue_args : str or None, optional - Additional queue arguments passed as single string of space-separated - Argument=Value pairs. Default is None. - """ - - # To be done asap so anything random is deterministic - if random_seed is not None: - import random - random.seed(random_seed) - import numpy - numpy.random.seed(random_seed) - # Ensure only supported bids options are passed - if debug: - lgr.setLevel(logging.DEBUG) - # Should be possible but only with a single subject -- will be used to - # override subject deduced from the DICOMs - if files and subjs and len(subjs) > 1: - raise ValueError( - "Unable to processes multiple `--subjects` with files" - ) - - if debug: - setup_exceptionhook() - - # Deal with provided files or templates - # pre-process provided list of files and possibly sort into groups/sessions - # Group files per each study/sid/session - - outdir = op.abspath(outdir) - - try: - import etelemetry - latest = etelemetry.get_project("nipy/heudiconv") - except Exception as e: - lgr.warning("Could not check for version updates: %s", str(e)) - latest = {"version": 'Unknown'} - - lgr.info(INIT_MSG(packname=__packagename__, - version=__version__, - latest=latest["version"])) - - if command: - process_extra_commands(outdir, command, files, dicom_dir_template, - heuristic, session, subjs, grouping) - return - # - # Load heuristic -- better do it asap to make sure it loads correctly - # - if not heuristic: - raise RuntimeError("No heuristic specified - add to arguments and rerun") - - if queue: - lgr.info("Queuing %s conversion", queue) - iterarg, iterables = ("files", len(files)) if files else \ - ("subjects", len(subjs)) - queue_conversion(queue, iterarg, iterables, queue_args) - sys.exit(0) - - heuristic = load_heuristic(heuristic) - - study_sessions = get_study_sessions(dicom_dir_template, files, - heuristic, outdir, session, - subjs, grouping=grouping) - - # extract tarballs, and replace their entries with expanded lists of files - # TODO: we might need to sort so sessions are ordered??? - lgr.info("Need to process %d study sessions", len(study_sessions)) - - # processed_studydirs = set() - - for (locator, session, sid), files_or_seqinfo in study_sessions.items(): - - # Allow for session to be overloaded from command line - if session is not None: - session = session - if locator is not None: - locator = locator - if not len(files_or_seqinfo): - raise ValueError("nothing to process?") - # that is how life is ATM :-/ since we don't do sorting if subj - # template is provided - if isinstance(files_or_seqinfo, dict): - assert(isinstance(list(files_or_seqinfo.keys())[0], SeqInfo)) - dicoms = None - seqinfo = files_or_seqinfo - else: - dicoms = files_or_seqinfo - seqinfo = None - - if locator == 'unknown': - lgr.warning("Skipping unknown locator dataset") - continue - - anon_sid = anonymize_sid(sid, anon_cmd) if anon_cmd else None - if anon_cmd: - lgr.info('Anonymized {} to {}'.format(sid, anon_sid)) - - study_outdir = op.join(outdir, locator or '') - anon_outdir = conv_outdir or outdir - anon_study_outdir = op.join(anon_outdir, locator or '') - - # TODO: --datalad cmdline option, which would take care about initiating - # the outdir -> study_outdir datasets if not yet there - if datalad: - from ..external.dlad import prepare_datalad - dlad_sid = sid if not anon_sid else anon_sid - dl_msg = prepare_datalad(anon_study_outdir, anon_outdir, dlad_sid, - session, seqinfo, dicoms, - bids_options) - - lgr.info("PROCESSING STARTS: {0}".format( - str(dict(subject=sid, outdir=study_outdir, session=session)))) - - prep_conversion(sid, - dicoms, - study_outdir, - heuristic, - converter=converter, - anon_sid=anon_sid, - anon_outdir=anon_study_outdir, - with_prov=with_prov, - ses=session, - bids_options=bids_options, - seqinfo=seqinfo, - min_meta=minmeta, - overwrite=overwrite, - dcmconfig=dcmconfig, - grouping=grouping,) - - lgr.info("PROCESSING DONE: {0}".format( - str(dict(subject=sid, outdir=study_outdir, session=session)))) - - if datalad: - from ..external.dlad import add_to_datalad - msg = "Converted subject %s" % dl_msg - # TODO: whenever propagate to supers work -- do just - # ds.save(msg=msg) - # also in batch mode might fail since we have no locking ATM - # and theoretically no need actually to save entire study - # we just need that - add_to_datalad(outdir, study_outdir, msg, bids_options) - - # if bids: - # # Let's populate BIDS templates for folks to take care about - # for study_outdir in processed_studydirs: - # populate_bids_templates(study_outdir) - # - # TODO: record_collection of the sid/session although that information - # is pretty much present in .heudiconv/SUBJECT/info so we could just poke there - - if __name__ == "__main__": main() diff --git a/heudiconv/workflows/run.py b/heudiconv/workflows/run.py new file mode 100644 index 00000000..2bcac67c --- /dev/null +++ b/heudiconv/workflows/run.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python +import os.path as op +import sys +import logging + +from .. import __version__, __packagename__ +from ..parser import get_study_sessions +from ..utils import load_heuristic, anonymize_sid, treat_infofile, SeqInfo +from ..convert import prep_conversion +from ..bids import populate_bids_templates, tuneup_bids_json_files +from ..queue import queue_conversion + +lgr = logging.getLogger(__name__) + + +INIT_MSG = "Running {packname} version {version} latest {latest}".format + + +def is_interactive(): + """Return True if all in/outs are tty""" + # TODO: check on windows if hasattr check would work correctly and add value: + return sys.stdin.isatty() and sys.stdout.isatty() and sys.stderr.isatty() + + +def setup_exceptionhook(): + """ + Overloads default sys.excepthook with our exceptionhook handler. + + If interactive, our exceptionhook handler will invoke pdb.post_mortem; + if not interactive, then invokes default handler. + """ + def _pdb_excepthook(type, value, tb): + if is_interactive(): + import traceback + import pdb + traceback.print_exception(type, value, tb) + # print() + pdb.post_mortem(tb) + else: + lgr.warning( + "We cannot setup exception hook since not in interactive mode") + + sys.excepthook = _pdb_excepthook + + +def process_extra_commands(outdir, command, files, dicom_dir_template, + heuristic, session, subjs, grouping): + """ + Perform custom command instead of regular operations. Supported commands: + ['treat-json', 'ls', 'populate-templates'] + + Parameters + ---------- + outdir : String + Output directory + args : Namespace + arguments + """ + if command == 'treat-jsons': + for f in files: + treat_infofile(f) + elif command == 'ls': + ensure_heuristic_arg(heuristic) + heuristic = load_heuristic(heuristic) + heuristic_ls = getattr(heuristic, 'ls', None) + for f in files: + study_sessions = get_study_sessions( + dicom_dir_template, [f], heuristic, outdir, + session, subjs, grouping=grouping) + print(f) + for study_session, sequences in study_sessions.items(): + suf = '' + if heuristic_ls: + suf += heuristic_ls(study_session, sequences) + print( + "\t%s %d sequences%s" + % (str(study_session), len(sequences), suf) + ) + elif command == 'populate-templates': + ensure_heuristic_arg(heuristic) + heuristic = load_heuristic(heuristic) + for f in files: + populate_bids_templates(f, getattr(heuristic, 'DEFAULT_FIELDS', {})) + elif command == 'sanitize-jsons': + tuneup_bids_json_files(files) + elif command == 'heuristics': + from ..utils import get_known_heuristics_with_descriptions + for name_desc in get_known_heuristics_with_descriptions().items(): + print("- %s: %s" % name_desc) + elif command == 'heuristic-info': + ensure_heuristic_arg(heuristic) + from ..utils import get_heuristic_description + print(get_heuristic_description(heuristic, full=True)) + else: + raise ValueError("Unknown command %s", command) + return + + +def ensure_heuristic_arg(heuristic=None): + from ..utils import get_known_heuristic_names + if not heuristic: + raise ValueError("Specify heuristic using -f. Known are: %s" + % ', '.join(get_known_heuristic_names())) + + +def heudiconv_workflow(dicom_dir_template=None, files=None, + subjs=None, converter='dcm2niix', outdir='.', + locator=None, conv_outdir=None, anon_cmd=None, + heuristic=None, with_prov=False, session=None, + bids_options=None, overwrite=False, datalad=False, + debug=False, command=None, grouping='studyUID', + minmeta=False, random_seed=None, dcmconfig=None, + queue=None, queue_args=None): + """Run the HeuDiConv conversion workflow. + + Parameters + ---------- + dicom_dir_template : str or None, optional + Location of dicomdir that can be indexed with subject id + {subject} and session {session}. Tarballs (can be compressed) + are supported in addition to directory. All matching tarballs + for a subject are extracted and their content processed in a + single pass. If multiple tarballs are found, each is assumed to + be a separate session and the 'session' argument is ignored. + Mutually exclusive with 'files'. Default is None. + files : list or None, optional + Files (tarballs, dicoms) or directories containing files to + process. Mutually exclusive with 'dicom_dir_template'. Default is None. + subjs : list or None, optional + List of subjects - required for dicom template. If not + provided, DICOMS would first be "sorted" and subject IDs + deduced by the heuristic. Default is None. + converter : {'dcm2niix', None}, optional + Tool to use for DICOM conversion. Setting to None disables + the actual conversion step -- useful for testing heuristics. + Default is None. + outdir : str, optional + Output directory for conversion setup (for further + customization and future reference. This directory will refer + to non-anonymized subject IDs. + Default is '.' (current working directory). + locator : str or 'unknown' or None, optional + Study path under outdir. If provided, it overloads the value + provided by the heuristic. If 'datalad=True', every + directory within locator becomes a super-dataset thus + establishing a hierarchy. Setting to "unknown" will skip that + dataset. Default is None. + conv_outdir : str or None, optional + Output directory for converted files. By default this is + identical to --outdir. This option is most useful in + combination with 'anon_cmd'. Default is None. + anon_cmd : str or None, optional + Command to run to convert subject IDs used for DICOMs to + anonymized IDs. Such command must take a single argument and + return a single anonymized ID. Also see 'conv_outdir'. Default is None. + heuristic : str or None, optional + Name of a known heuristic or path to the Python script containing + heuristic. Default is None. + with_prov : bool, optional + Store additional provenance information. Requires python-rdflib. + Default is False. + session : str or None, optional + Session for longitudinal study_sessions. Default is None. + bids_options : str or None, optional + Flag for output into BIDS structure. Can also take BIDS- + specific options, e.g., --bids notop. The only currently + supported options is "notop", which skips creation of + top-level BIDS files. This is useful when running in batch + mode to prevent possible race conditions. Default is None. + overwrite : bool, optional + Overwrite existing converted files. Default is False. + datalad : bool, optional + Store the entire collection as DataLad dataset(s). Small files + will be committed directly to git, while large to annex. New + version (6) of annex repositories will be used in a "thin" + mode so it would look to mortals as just any other regular + directory (i.e. no symlinks to under .git/annex). For now just + for BIDS mode. Default is False. + debug : bool, optional + Do not catch exceptions and show exception traceback. Default is False. + command : {'heuristics', 'heuristic-info', 'ls', 'populate-templates', + 'sanitize-jsons', 'treat-jsons', None}, optional + Custom action to be performed on provided files instead of regular + operation. Default is None. + grouping : {'studyUID', 'accession_number', 'all', 'custom'}, optional + How to group dicoms. Default is 'studyUID'. + minmeta : bool, optional + Exclude dcmstack meta information in sidecar jsons. Default is False. + random_seed : int or None, optional + Random seed to initialize RNG. Default is None. + dcmconfig : str or None, optional + JSON file for additional dcm2niix configuration. Default is None. + queue : {'SLURM', None}, optional + Batch system to submit jobs in parallel. Default is None. + queue_args : str or None, optional + Additional queue arguments passed as single string of space-separated + Argument=Value pairs. Default is None. + """ + + # To be done asap so anything random is deterministic + if random_seed is not None: + import random + random.seed(random_seed) + import numpy + numpy.random.seed(random_seed) + # Ensure only supported bids options are passed + if debug: + lgr.setLevel(logging.DEBUG) + # Should be possible but only with a single subject -- will be used to + # override subject deduced from the DICOMs + if files and subjs and len(subjs) > 1: + raise ValueError( + "Unable to processes multiple `--subjects` with files" + ) + + if debug: + setup_exceptionhook() + + # Deal with provided files or templates + # pre-process provided list of files and possibly sort into groups/sessions + # Group files per each study/sid/session + + outdir = op.abspath(outdir) + + try: + import etelemetry + latest = etelemetry.get_project("nipy/heudiconv") + except Exception as e: + lgr.warning("Could not check for version updates: %s", str(e)) + latest = {"version": 'Unknown'} + + lgr.info(INIT_MSG(packname=__packagename__, + version=__version__, + latest=latest["version"])) + + if command: + process_extra_commands(outdir, command, files, dicom_dir_template, + heuristic, session, subjs, grouping) + return + # + # Load heuristic -- better do it asap to make sure it loads correctly + # + if not heuristic: + raise RuntimeError("No heuristic specified - add to arguments and rerun") + + if queue: + lgr.info("Queuing %s conversion", queue) + iterarg, iterables = ("files", len(files)) if files else \ + ("subjects", len(subjs)) + queue_conversion(queue, iterarg, iterables, queue_args) + sys.exit(0) + + heuristic = load_heuristic(heuristic) + + study_sessions = get_study_sessions(dicom_dir_template, files, + heuristic, outdir, session, + subjs, grouping=grouping) + + # extract tarballs, and replace their entries with expanded lists of files + # TODO: we might need to sort so sessions are ordered??? + lgr.info("Need to process %d study sessions", len(study_sessions)) + + # processed_studydirs = set() + + for (locator, session, sid), files_or_seqinfo in study_sessions.items(): + + # Allow for session to be overloaded from command line + if session is not None: + session = session + if locator is not None: + locator = locator + if not len(files_or_seqinfo): + raise ValueError("nothing to process?") + # that is how life is ATM :-/ since we don't do sorting if subj + # template is provided + if isinstance(files_or_seqinfo, dict): + assert(isinstance(list(files_or_seqinfo.keys())[0], SeqInfo)) + dicoms = None + seqinfo = files_or_seqinfo + else: + dicoms = files_or_seqinfo + seqinfo = None + + if locator == 'unknown': + lgr.warning("Skipping unknown locator dataset") + continue + + anon_sid = anonymize_sid(sid, anon_cmd) if anon_cmd else None + if anon_cmd: + lgr.info('Anonymized {} to {}'.format(sid, anon_sid)) + + study_outdir = op.join(outdir, locator or '') + anon_outdir = conv_outdir or outdir + anon_study_outdir = op.join(anon_outdir, locator or '') + + # TODO: --datalad cmdline option, which would take care about initiating + # the outdir -> study_outdir datasets if not yet there + if datalad: + from ..external.dlad import prepare_datalad + dlad_sid = sid if not anon_sid else anon_sid + dl_msg = prepare_datalad(anon_study_outdir, anon_outdir, dlad_sid, + session, seqinfo, dicoms, + bids_options) + + lgr.info("PROCESSING STARTS: {0}".format( + str(dict(subject=sid, outdir=study_outdir, session=session)))) + + prep_conversion(sid, + dicoms, + study_outdir, + heuristic, + converter=converter, + anon_sid=anon_sid, + anon_outdir=anon_study_outdir, + with_prov=with_prov, + ses=session, + bids_options=bids_options, + seqinfo=seqinfo, + min_meta=minmeta, + overwrite=overwrite, + dcmconfig=dcmconfig, + grouping=grouping,) + + lgr.info("PROCESSING DONE: {0}".format( + str(dict(subject=sid, outdir=study_outdir, session=session)))) + + if datalad: + from ..external.dlad import add_to_datalad + msg = "Converted subject %s" % dl_msg + # TODO: whenever propagate to supers work -- do just + # ds.save(msg=msg) + # also in batch mode might fail since we have no locking ATM + # and theoretically no need actually to save entire study + # we just need that + add_to_datalad(outdir, study_outdir, msg, bids_options) + + # if bids: + # # Let's populate BIDS templates for folks to take care about + # for study_outdir in processed_studydirs: + # populate_bids_templates(study_outdir) + # + # TODO: record_collection of the sid/session although that information + # is pretty much present in .heudiconv/SUBJECT/info so we could just poke there From f773516e9c8cf7b78964a6a4559940906dd17905 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 16 Apr 2020 18:15:14 -0400 Subject: [PATCH 06/24] Add init file for workflows module. --- heudiconv/workflows/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 heudiconv/workflows/__init__.py diff --git a/heudiconv/workflows/__init__.py b/heudiconv/workflows/__init__.py new file mode 100644 index 00000000..7b4b829e --- /dev/null +++ b/heudiconv/workflows/__init__.py @@ -0,0 +1,3 @@ +from .run import heudiconv_workflow + +__all__ = ['heudiconv_workflow'] From 31b586a9fee9018674673be432fb582d4522b406 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Sat, 18 Apr 2020 12:48:32 -0400 Subject: [PATCH 07/24] Fix test. --- heudiconv/tests/test_main.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/heudiconv/tests/test_main.py b/heudiconv/tests/test_main.py index 91291378..4c32db1e 100644 --- a/heudiconv/tests/test_main.py +++ b/heudiconv/tests/test_main.py @@ -1,9 +1,7 @@ # TODO: break this up by modules -from heudiconv.cli.run import ( - main as runner, - process_args, -) +from heudiconv.cli.run import main as runner +from heudiconv.workflows.run import process_args from heudiconv import __version__ from heudiconv.utils import (create_file_if_missing, set_readonly, From 83760510f9d9b63e8e235456469020d4b5740128 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Sat, 18 Apr 2020 14:07:53 -0400 Subject: [PATCH 08/24] Fix test. --- heudiconv/tests/test_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/heudiconv/tests/test_main.py b/heudiconv/tests/test_main.py index 4c32db1e..e27d4f5b 100644 --- a/heudiconv/tests/test_main.py +++ b/heudiconv/tests/test_main.py @@ -1,7 +1,7 @@ # TODO: break this up by modules from heudiconv.cli.run import main as runner -from heudiconv.workflows.run import process_args +from heudiconv.workflows.run import heudiconv_workflow from heudiconv import __version__ from heudiconv.utils import (create_file_if_missing, set_readonly, @@ -294,4 +294,4 @@ class args: # must not fail if etelemetry no found with patch.dict('sys.modules', {'etelemetry': None}): - process_args(args) + heudiconv_workflow(**vars(args)) From c60b50f0488667229fcad920a026683657818890 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Sat, 18 Apr 2020 17:03:56 -0400 Subject: [PATCH 09/24] Drops "args" and just use parameters. --- heudiconv/tests/test_main.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/heudiconv/tests/test_main.py b/heudiconv/tests/test_main.py index e27d4f5b..0508ec82 100644 --- a/heudiconv/tests/test_main.py +++ b/heudiconv/tests/test_main.py @@ -286,12 +286,7 @@ def test_cache(tmpdir): def test_no_etelemetry(): # smoke test at large - just verifying that no crash if no etelemetry - class args: - outdir = '/dev/null' - command = 'ls' - heuristic = 'reproin' - files = [] # Nothing to list - # must not fail if etelemetry no found with patch.dict('sys.modules', {'etelemetry': None}): - heudiconv_workflow(**vars(args)) + heudiconv_workflow(outdir='/dev/null', command='ls', + heuristic='reproin', files=[]) From 86d9864dd803902b31649ede35a7b15050941529 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 23 Apr 2020 12:33:17 -0400 Subject: [PATCH 10/24] Remove shebang. --- heudiconv/workflows/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/heudiconv/workflows/run.py b/heudiconv/workflows/run.py index 2bcac67c..1d674fb6 100644 --- a/heudiconv/workflows/run.py +++ b/heudiconv/workflows/run.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import os.path as op import sys import logging From 330404a770dff8465e0da317562ed1f7c148240c Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 23 Apr 2020 12:35:13 -0400 Subject: [PATCH 11/24] Sort imports. --- heudiconv/cli/run.py | 4 ++-- heudiconv/workflows/run.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/heudiconv/cli/run.py b/heudiconv/cli/run.py index 15c9588b..7e89b99e 100644 --- a/heudiconv/cli/run.py +++ b/heudiconv/cli/run.py @@ -1,9 +1,9 @@ #!/usr/bin/env python +import logging import os -from argparse import ArgumentParser import sys -import logging +from argparse import ArgumentParser from .. import __version__ from ..workflows import heudiconv_workflow diff --git a/heudiconv/workflows/run.py b/heudiconv/workflows/run.py index 1d674fb6..90584eb5 100644 --- a/heudiconv/workflows/run.py +++ b/heudiconv/workflows/run.py @@ -1,13 +1,13 @@ +import logging import os.path as op import sys -import logging from .. import __version__, __packagename__ -from ..parser import get_study_sessions -from ..utils import load_heuristic, anonymize_sid, treat_infofile, SeqInfo -from ..convert import prep_conversion from ..bids import populate_bids_templates, tuneup_bids_json_files +from ..convert import prep_conversion +from ..parser import get_study_sessions from ..queue import queue_conversion +from ..utils import anonymize_sid, load_heuristic, treat_infofile, SeqInfo lgr = logging.getLogger(__name__) From 893d008e993b72f209d10f7ddce62285a91cd34e Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 23 Apr 2020 13:29:33 -0400 Subject: [PATCH 12/24] Eliminate workflows module. --- heudiconv/cli/run.py | 4 ++-- heudiconv/{workflows/run.py => main.py} | 14 ++++++-------- heudiconv/workflows/__init__.py | 3 --- 3 files changed, 8 insertions(+), 13 deletions(-) rename heudiconv/{workflows/run.py => main.py} (96%) delete mode 100644 heudiconv/workflows/__init__.py diff --git a/heudiconv/cli/run.py b/heudiconv/cli/run.py index 7e89b99e..8a381829 100644 --- a/heudiconv/cli/run.py +++ b/heudiconv/cli/run.py @@ -6,7 +6,7 @@ from argparse import ArgumentParser from .. import __version__ -from ..workflows import heudiconv_workflow +from ..main import workflow lgr = logging.getLogger(__name__) @@ -21,7 +21,7 @@ def main(argv=None): sys.exit(1) kwargs = vars(args) - heudiconv_workflow(**kwargs) + workflow(**kwargs) def get_parser(): diff --git a/heudiconv/workflows/run.py b/heudiconv/main.py similarity index 96% rename from heudiconv/workflows/run.py rename to heudiconv/main.py index 90584eb5..376a75a2 100644 --- a/heudiconv/workflows/run.py +++ b/heudiconv/main.py @@ -102,14 +102,12 @@ def ensure_heuristic_arg(heuristic=None): % ', '.join(get_known_heuristic_names())) -def heudiconv_workflow(dicom_dir_template=None, files=None, - subjs=None, converter='dcm2niix', outdir='.', - locator=None, conv_outdir=None, anon_cmd=None, - heuristic=None, with_prov=False, session=None, - bids_options=None, overwrite=False, datalad=False, - debug=False, command=None, grouping='studyUID', - minmeta=False, random_seed=None, dcmconfig=None, - queue=None, queue_args=None): +def workflow(dicom_dir_template=None, files=None, subjs=None, + converter='dcm2niix', outdir='.', locator=None, conv_outdir=None, + anon_cmd=None, heuristic=None, with_prov=False, session=None, + bids_options=None, overwrite=False, datalad=False, debug=False, + command=None, grouping='studyUID', minmeta=False, + random_seed=None, dcmconfig=None, queue=None, queue_args=None): """Run the HeuDiConv conversion workflow. Parameters diff --git a/heudiconv/workflows/__init__.py b/heudiconv/workflows/__init__.py deleted file mode 100644 index 7b4b829e..00000000 --- a/heudiconv/workflows/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .run import heudiconv_workflow - -__all__ = ['heudiconv_workflow'] From b7340be33287b93823ecb11fc12c1e0acea6335c Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Fri, 24 Apr 2020 16:03:57 -0400 Subject: [PATCH 13/24] Fix imports after refactor. --- heudiconv/main.py | 12 ++++++------ heudiconv/tests/test_main.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/heudiconv/main.py b/heudiconv/main.py index 376a75a2..b55d0405 100644 --- a/heudiconv/main.py +++ b/heudiconv/main.py @@ -2,12 +2,12 @@ import os.path as op import sys -from .. import __version__, __packagename__ -from ..bids import populate_bids_templates, tuneup_bids_json_files -from ..convert import prep_conversion -from ..parser import get_study_sessions -from ..queue import queue_conversion -from ..utils import anonymize_sid, load_heuristic, treat_infofile, SeqInfo +from . import __version__, __packagename__ +from .bids import populate_bids_templates, tuneup_bids_json_files +from .convert import prep_conversion +from .parser import get_study_sessions +from .queue import queue_conversion +from .utils import anonymize_sid, load_heuristic, treat_infofile, SeqInfo lgr = logging.getLogger(__name__) diff --git a/heudiconv/tests/test_main.py b/heudiconv/tests/test_main.py index 0508ec82..0274f99f 100644 --- a/heudiconv/tests/test_main.py +++ b/heudiconv/tests/test_main.py @@ -1,7 +1,7 @@ # TODO: break this up by modules from heudiconv.cli.run import main as runner -from heudiconv.workflows.run import heudiconv_workflow +from heudiconv.main import workflow from heudiconv import __version__ from heudiconv.utils import (create_file_if_missing, set_readonly, @@ -288,5 +288,5 @@ def test_no_etelemetry(): # smoke test at large - just verifying that no crash if no etelemetry # must not fail if etelemetry no found with patch.dict('sys.modules', {'etelemetry': None}): - heudiconv_workflow(outdir='/dev/null', command='ls', - heuristic='reproin', files=[]) + workflow(outdir='/dev/null', command='ls', + heuristic='reproin', files=[]) From 3a2545849646ea9eb8a1462094b13a6e47cf1bac Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Fri, 24 Apr 2020 16:13:39 -0400 Subject: [PATCH 14/24] Fix many more imports. --- heudiconv/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/heudiconv/main.py b/heudiconv/main.py index b55d0405..a9e0d8a3 100644 --- a/heudiconv/main.py +++ b/heudiconv/main.py @@ -83,12 +83,12 @@ def process_extra_commands(outdir, command, files, dicom_dir_template, elif command == 'sanitize-jsons': tuneup_bids_json_files(files) elif command == 'heuristics': - from ..utils import get_known_heuristics_with_descriptions + from .utils import get_known_heuristics_with_descriptions for name_desc in get_known_heuristics_with_descriptions().items(): print("- %s: %s" % name_desc) elif command == 'heuristic-info': ensure_heuristic_arg(heuristic) - from ..utils import get_heuristic_description + from .utils import get_heuristic_description print(get_heuristic_description(heuristic, full=True)) else: raise ValueError("Unknown command %s", command) @@ -96,7 +96,7 @@ def process_extra_commands(outdir, command, files, dicom_dir_template, def ensure_heuristic_arg(heuristic=None): - from ..utils import get_known_heuristic_names + from .utils import get_known_heuristic_names if not heuristic: raise ValueError("Specify heuristic using -f. Known are: %s" % ', '.join(get_known_heuristic_names())) @@ -293,7 +293,7 @@ def workflow(dicom_dir_template=None, files=None, subjs=None, # TODO: --datalad cmdline option, which would take care about initiating # the outdir -> study_outdir datasets if not yet there if datalad: - from ..external.dlad import prepare_datalad + from .external.dlad import prepare_datalad dlad_sid = sid if not anon_sid else anon_sid dl_msg = prepare_datalad(anon_study_outdir, anon_outdir, dlad_sid, session, seqinfo, dicoms, @@ -322,7 +322,7 @@ def workflow(dicom_dir_template=None, files=None, subjs=None, str(dict(subject=sid, outdir=study_outdir, session=session)))) if datalad: - from ..external.dlad import add_to_datalad + from .external.dlad import add_to_datalad msg = "Converted subject %s" % dl_msg # TODO: whenever propagate to supers work -- do just # ds.save(msg=msg) From 7070c49228008c179e638440cc98e3ce58851fd8 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Tue, 12 May 2020 22:42:52 -0400 Subject: [PATCH 15/24] Disable positional arguments in workflow. --- heudiconv/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/heudiconv/main.py b/heudiconv/main.py index a9e0d8a3..1b1c5da9 100644 --- a/heudiconv/main.py +++ b/heudiconv/main.py @@ -102,7 +102,7 @@ def ensure_heuristic_arg(heuristic=None): % ', '.join(get_known_heuristic_names())) -def workflow(dicom_dir_template=None, files=None, subjs=None, +def workflow(*, dicom_dir_template=None, files=None, subjs=None, converter='dcm2niix', outdir='.', locator=None, conv_outdir=None, anon_cmd=None, heuristic=None, with_prov=False, session=None, bids_options=None, overwrite=False, datalad=False, debug=False, @@ -192,6 +192,10 @@ def workflow(dicom_dir_template=None, files=None, subjs=None, queue_args : str or None, optional Additional queue arguments passed as single string of space-separated Argument=Value pairs. Default is None. + + Notes + ----- + All parameters in this function must be called as keyword arguments. """ # To be done asap so anything random is deterministic From 4950336ce4e20d6d507309bf6d17c21acd98b848 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Tue, 12 May 2020 22:43:21 -0400 Subject: [PATCH 16/24] Fix circular variable assignment. --- heudiconv/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/heudiconv/main.py b/heudiconv/main.py index 1b1c5da9..54adaec2 100644 --- a/heudiconv/main.py +++ b/heudiconv/main.py @@ -263,13 +263,13 @@ def workflow(*, dicom_dir_template=None, files=None, subjs=None, # processed_studydirs = set() - for (locator, session, sid), files_or_seqinfo in study_sessions.items(): + for (locator_ext, session_ext, sid), files_or_seqinfo in study_sessions.items(): - # Allow for session to be overloaded from command line - if session is not None: - session = session - if locator is not None: - locator = locator + # Only use extracted session/locator if not provided explicitly + if session is None: + session = session_ext + if locator is None: + locator = locator_ext if not len(files_or_seqinfo): raise ValueError("nothing to process?") # that is how life is ATM :-/ since we don't do sorting if subj From f9cf5692fb8fe34bd433a8c65e87bdc3dd6e870a Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 14 May 2020 10:06:08 -0400 Subject: [PATCH 17/24] Add docstrings. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also kind of hoping that the test failures were random and that they’ll pass now. --- heudiconv/main.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/heudiconv/main.py b/heudiconv/main.py index 54adaec2..3b54a7c9 100644 --- a/heudiconv/main.py +++ b/heudiconv/main.py @@ -50,10 +50,27 @@ def process_extra_commands(outdir, command, files, dicom_dir_template, Parameters ---------- - outdir : String + outdir : str Output directory - args : Namespace - arguments + command : {'treat-json', 'ls', 'populate-templates'} + Heudiconv command to run + files : list of str + List of files + dicom_dir_template : str + Location of dicomdir that can be indexed with subject id + {subject} and session {session}. Tarballs (can be compressed) + are supported in addition to directory. All matching tarballs + for a subject are extracted and their content processed in a + single pass. If multiple tarballs are found, each is assumed to + be a separate session and the 'session' argument is ignored. + heuristic : str + Path to heuristic file or name of builtin heuristic. + session : str + Session identifier + subjs : list of str + List of subject identifiers + grouping : {'studyUID', 'accession_number', 'all', 'custom'} + How to group dicoms. """ if command == 'treat-jsons': for f in files: @@ -96,6 +113,9 @@ def process_extra_commands(outdir, command, files, dicom_dir_template, def ensure_heuristic_arg(heuristic=None): + """ + Check that the heuristic argument was provided. + """ from .utils import get_known_heuristic_names if not heuristic: raise ValueError("Specify heuristic using -f. Known are: %s" From 59430c04ac4a5534318e7268a88234d654e58f32 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Fri, 15 May 2020 13:58:31 -0400 Subject: [PATCH 18/24] Drop kwarg-only to test CI. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit My local testing in Docker isn’t working for some reason. --- heudiconv/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/main.py b/heudiconv/main.py index 3b54a7c9..e832e6f9 100644 --- a/heudiconv/main.py +++ b/heudiconv/main.py @@ -122,7 +122,7 @@ def ensure_heuristic_arg(heuristic=None): % ', '.join(get_known_heuristic_names())) -def workflow(*, dicom_dir_template=None, files=None, subjs=None, +def workflow(dicom_dir_template=None, files=None, subjs=None, converter='dcm2niix', outdir='.', locator=None, conv_outdir=None, anon_cmd=None, heuristic=None, with_prov=False, session=None, bids_options=None, overwrite=False, datalad=False, debug=False, From 54da90b16a50f0a623cd8dd0ebcab51abe37694b Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Fri, 15 May 2020 14:04:02 -0400 Subject: [PATCH 19/24] Revert 59430c0. --- heudiconv/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/main.py b/heudiconv/main.py index e832e6f9..3b54a7c9 100644 --- a/heudiconv/main.py +++ b/heudiconv/main.py @@ -122,7 +122,7 @@ def ensure_heuristic_arg(heuristic=None): % ', '.join(get_known_heuristic_names())) -def workflow(dicom_dir_template=None, files=None, subjs=None, +def workflow(*, dicom_dir_template=None, files=None, subjs=None, converter='dcm2niix', outdir='.', locator=None, conv_outdir=None, anon_cmd=None, heuristic=None, with_prov=False, session=None, bids_options=None, overwrite=False, datalad=False, debug=False, From ea2aea701ef5c55f1a2152e027a5f008ded5d2b8 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Fri, 15 May 2020 14:05:01 -0400 Subject: [PATCH 20/24] Drop circularity fix to test CI. --- heudiconv/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/heudiconv/main.py b/heudiconv/main.py index 3b54a7c9..3b907976 100644 --- a/heudiconv/main.py +++ b/heudiconv/main.py @@ -283,13 +283,13 @@ def workflow(*, dicom_dir_template=None, files=None, subjs=None, # processed_studydirs = set() - for (locator_ext, session_ext, sid), files_or_seqinfo in study_sessions.items(): + for (locator, session, sid), files_or_seqinfo in study_sessions.items(): - # Only use extracted session/locator if not provided explicitly - if session is None: - session = session_ext - if locator is None: - locator = locator_ext + # Allow for session to be overloaded from command line + if session is not None: + session = session + if locator is not None: + locator = locator if not len(files_or_seqinfo): raise ValueError("nothing to process?") # that is how life is ATM :-/ since we don't do sorting if subj From f8d8ec1fc1dd78ea0c97a40a3ad1ccbbaf4ef5ea Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Fri, 15 May 2020 14:17:01 -0400 Subject: [PATCH 21/24] Try a different approach for the circularity issue. --- heudiconv/main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/heudiconv/main.py b/heudiconv/main.py index 3b907976..2df64954 100644 --- a/heudiconv/main.py +++ b/heudiconv/main.py @@ -283,13 +283,14 @@ def workflow(*, dicom_dir_template=None, files=None, subjs=None, # processed_studydirs = set() + locator_manual, session_manual = locator, session for (locator, session, sid), files_or_seqinfo in study_sessions.items(): # Allow for session to be overloaded from command line - if session is not None: - session = session - if locator is not None: - locator = locator + if session_manual is not None: + session = session_manual + if locator_manual is not None: + locator = locator_manual if not len(files_or_seqinfo): raise ValueError("nothing to process?") # that is how life is ATM :-/ since we don't do sorting if subj From c530d297a5d521b17d74034c794f24fac22ed66f Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Fri, 15 May 2020 14:17:22 -0400 Subject: [PATCH 22/24] Remove datalad comment. --- heudiconv/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/heudiconv/main.py b/heudiconv/main.py index 2df64954..292ebd1b 100644 --- a/heudiconv/main.py +++ b/heudiconv/main.py @@ -315,8 +315,6 @@ def workflow(*, dicom_dir_template=None, files=None, subjs=None, anon_outdir = conv_outdir or outdir anon_study_outdir = op.join(anon_outdir, locator or '') - # TODO: --datalad cmdline option, which would take care about initiating - # the outdir -> study_outdir datasets if not yet there if datalad: from .external.dlad import prepare_datalad dlad_sid = sid if not anon_sid else anon_sid From e79df64b8acc6f79e5bcc0e546f8123575adb0af Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 16 Jun 2020 14:38:13 -0400 Subject: [PATCH 23/24] BF: do interpolate the string msg in the exception I guess it was a mental side effect from using lgr.error initially or smth like that --- heudiconv/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/main.py b/heudiconv/main.py index 292ebd1b..886f6c86 100644 --- a/heudiconv/main.py +++ b/heudiconv/main.py @@ -108,7 +108,7 @@ def process_extra_commands(outdir, command, files, dicom_dir_template, from .utils import get_heuristic_description print(get_heuristic_description(heuristic, full=True)) else: - raise ValueError("Unknown command %s", command) + raise ValueError("Unknown command %s" % command) return From c49134ea5db10bc60dd7b9f9d8edfb1d813a02b6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 16 Jun 2020 14:50:50 -0400 Subject: [PATCH 24/24] RF: do not sys.exit(0) - just return from main upon queueing up conversion --- heudiconv/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/heudiconv/main.py b/heudiconv/main.py index 886f6c86..56401def 100644 --- a/heudiconv/main.py +++ b/heudiconv/main.py @@ -209,6 +209,8 @@ def workflow(*, dicom_dir_template=None, files=None, subjs=None, JSON file for additional dcm2niix configuration. Default is None. queue : {'SLURM', None}, optional Batch system to submit jobs in parallel. Default is None. + If set, will cause scheduling of conversion and return without performing + any further action. queue_args : str or None, optional Additional queue arguments passed as single string of space-separated Argument=Value pairs. Default is None. @@ -269,7 +271,7 @@ def workflow(*, dicom_dir_template=None, files=None, subjs=None, iterarg, iterables = ("files", len(files)) if files else \ ("subjects", len(subjs)) queue_conversion(queue, iterarg, iterables, queue_args) - sys.exit(0) + return heuristic = load_heuristic(heuristic)