From e21a9ba0e4914a96e212eb1885ed3aa36a881c8b Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Sun, 23 Feb 2020 17:53:06 -0500 Subject: [PATCH 01/11] Add functions for renaming complex, multi-echo, and uncombined files. --- heudiconv/convert.py | 185 ++++++++++++++++++++++++++++++------------- 1 file changed, 129 insertions(+), 56 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index e3390593..4a410de4 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -230,6 +230,95 @@ def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid, getattr(heuristic, 'DEFAULT_FIELDS', {})) +def update_complex_name(fileinfo, this_prefix_basename, suffix): + """ + Insert `_part-` entity into filename if data are from a sequence + with magnitude/phase reconstruction. + """ + unsupported_types = ['_bold', '_phase'] + if suffix in unsupported_types: + return this_prefix_basename + + # Check to see if it is magnitude or phase reconstruction: + if 'M' in fileinfo.get('ImageType'): + mag_or_phase = 'mag' + elif 'P' in fileinfo.get('ImageType'): + mag_or_phase = 'phase' + else: + mag_or_phase = suffix + + # Insert reconstruction label + if not ('_part-%s' % mag_or_phase) in this_prefix_basename: + + # If "_part-" is specified, prepend the 'mag_or_phase' value. + if '_part-' in this_prefix_basename: + raise BIDSError( + "Part label for images will be automatically set, remove " + "from heuristic" + ) + + # If not, insert "_part-" + 'mag_or_phase' into the prefix_basename + # **before** "_run", "_echo" or "_sbref", whichever appears first: + for label in ['_run', '_echo', '_sbref']: + if (label in this_prefix_basename): + this_prefix_basename = this_prefix_basename.replace( + label, "_part-%s%s" % (mag_or_phase, label) + ) + break + return this_prefix_basename + + +def update_multiecho_name(fileinfo, this_prefix_basename, echo_times, suffix): + """ + Insert `_echo-` entity into filename if data are from a multi-echo + sequence. + """ + unsupported_types = ['_magnitude1', '_magnitude2', '_phasediff', '_phase1', '_phase2'] + if suffix in unsupported_types: + return this_prefix_basename + + # Get the EchoNumber from json file info. If not present, use EchoTime + if 'EchoNumber' in fileinfo.keys(): + echo_number = fileinfo['EchoNumber'] + else: + echo_number = echo_times.index(fileinfo['EchoTime']) + 1 + + # Now, decide where to insert it. + # Insert it **before** the following string(s), whichever appears first. + for imgtype in supported_multiecho: + if (imgtype in this_prefix_basename): + this_prefix_basename = this_prefix_basename.replace( + imgtype, "_echo-%d%s" % (echo_number, imgtype) + ) + break + return this_prefix_basename + + +def update_uncombined_name(fileinfo, this_prefix_basename, coil_names, suffix): + """ + Insert `_channel-` entity into filename if data are from a sequence + with "save uncombined". + """ + # Determine the channel number + channel_number = coil_names.index(fileinfo['CoilString']) + 1 + + # Insert it **before** the following string(s), whichever appears first. + for label in ['_run', '_echo', suffix]: + if label == suffix: + prefix_suffix = this_prefix_basename.split('_')[-1] + this_prefix_basename = this_prefix_basename.replace( + prefix_suffix, "_channel-%s_%s" % (channel_number, prefix_suffix) + ) + break + + if (label in this_prefix_basename): + this_prefix_basename = this_prefix_basename.replace( + label, "_channel-%s%s" % (coil_number, label) + ) + break + return this_prefix_basename + + def convert(items, converter, scaninfo_suffix, custom_callable, with_prov, bids_options, outdir, min_meta, overwrite, symlink=True, prov_file=None, dcmconfig=None): @@ -527,14 +616,29 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # series. To do that, the most straightforward way is to read the # echo times for all bids_files and see if they are all the same or not. - # Check for varying echo times - echo_times = sorted(list(set( - load_json(b).get('EchoTime', None) - for b in bids_files - if b - ))) - - is_multiecho = len(echo_times) > 1 + # Collect some metadata across all images + echo_times, channel_names, image_types = [], [], [] + for b in bids_files: + if not b: + continue + metadata = load_json(b) + echo_times.append(metadata.get('EchoTime', None)) + channel_names.append(metadata.get('CoilString', None)) + image_types.append(metadata.get('ImageType', None)) + echo_times = [v for v in echo_times if v] + echo_times = sorted(list(set(echo_times))) + channel_names = [v for v in channel_names if v] + channel_names = sorted(list(set(channel_names))) + image_types = [v for v in image_types if v] + image_types = sorted(list(set(image_types))) + + is_multiecho = len(echo_times) > 1 # Check for varying echo times + is_uncombined = len(coil_names) > 1 # Check for uncombined data + + # Determine if data are complex (magnitude + phase) + magnitude_found = ['M' in it for it in image_types] + phase_found = ['P' in it for it in image_types] + is_complex = magnitude_found and phase_found ### Loop through the bids_files, set the output name and save files for fl, suffix, bids_file in zip(res_files, suffixes, bids_files): @@ -542,63 +646,32 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # TODO: monitor conversion duration if bids_file: fileinfo = load_json(bids_file) + print(suffix) # set the prefix basename for this specific file (we'll modify it, # and we don't want to modify it for all the bids_files): this_prefix_basename = prefix_basename - # _sbref sequences reconstructing magnitude and phase generate - # two NIfTI files IN THE SAME SERIES, so we cannot just add - # the suffix, if we want to be bids compliant: - if bids_file and this_prefix_basename.endswith('_sbref'): - # Check to see if it is magnitude or phase reconstruction: - if 'M' in fileinfo.get('ImageType'): - mag_or_phase = 'magnitude' - elif 'P' in fileinfo.get('ImageType'): - mag_or_phase = 'phase' - else: - mag_or_phase = suffix - - # Insert reconstruction label - if not ("_rec-%s" % mag_or_phase) in this_prefix_basename: - - # If "_rec-" is specified, prepend the 'mag_or_phase' value. - if ('_rec-' in this_prefix_basename): - raise BIDSError( - "Reconstruction label for multi-echo single-band" - " reference images will be automatically set, remove" - " from heuristic" - ) - - # If not, insert "_rec-" + 'mag_or_phase' into the prefix_basename - # **before** "_run", "_echo" or "_sbref", whichever appears first: - for label in ['_run', '_echo', '_sbref']: - if (label in this_prefix_basename): - this_prefix_basename = this_prefix_basename.replace( - label, "_rec-%s%s" % (mag_or_phase, label) - ) - break - - # Now check if this run is multi-echo + # Update name if complex data + if bids_file and is_complex: + this_prefix_basename = update_complex_name( + fileinfo, this_prefix_basename, suffix + ) + + # Update name if multi-echo # (Note: it can be _sbref and multiecho, so don't use "elif"): # For multi-echo sequences, we have to specify the echo number in # the file name: if bids_file and is_multiecho: - # Get the EchoNumber from json file info. If not present, use EchoTime - if 'EchoNumber' in fileinfo.keys(): - echo_number = fileinfo['EchoNumber'] - else: - echo_number = echo_times.index(fileinfo['EchoTime']) + 1 - - supported_multiecho = ['_bold', '_phase', '_epi', '_sbref', '_T1w', '_PDT2'] - # Now, decide where to insert it. - # Insert it **before** the following string(s), whichever appears first. - for imgtype in supported_multiecho: - if (imgtype in this_prefix_basename): - this_prefix_basename = this_prefix_basename.replace( - imgtype, "_echo-%d%s" % (echo_number, imgtype) - ) - break + this_prefix_basename = update_multiecho_name( + fileinfo, this_prefix_basename, echo_times, suffix + ) + + # Update name if uncombined (channel-level) data + if bids_file and is_uncombined: + this_prefix_basename = update_uncombined_name( + fileinfo, this_prefix_basename, channel_names + ) # Fallback option: # If we have failed to modify this_prefix_basename, because it didn't fall From 95e574be1726526cf48b1f576039a7771f3f0f59 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Mon, 24 Feb 2020 09:12:00 -0500 Subject: [PATCH 02/11] Fix bugs. --- heudiconv/convert.py | 62 +++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index 4a410de4..b5b81110 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -236,7 +236,7 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): with magnitude/phase reconstruction. """ unsupported_types = ['_bold', '_phase'] - if suffix in unsupported_types: + if any(ut in this_prefix_basename for ut in unsupported_types): return this_prefix_basename # Check to see if it is magnitude or phase reconstruction: @@ -249,7 +249,6 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): # Insert reconstruction label if not ('_part-%s' % mag_or_phase) in this_prefix_basename: - # If "_part-" is specified, prepend the 'mag_or_phase' value. if '_part-' in this_prefix_basename: raise BIDSError( @@ -258,7 +257,7 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): ) # If not, insert "_part-" + 'mag_or_phase' into the prefix_basename - # **before** "_run", "_echo" or "_sbref", whichever appears first: + # **before** "_run", "_echo" or "_sbref", whichever appears first: for label in ['_run', '_echo', '_sbref']: if (label in this_prefix_basename): this_prefix_basename = this_prefix_basename.replace( @@ -274,7 +273,7 @@ def update_multiecho_name(fileinfo, this_prefix_basename, echo_times, suffix): sequence. """ unsupported_types = ['_magnitude1', '_magnitude2', '_phasediff', '_phase1', '_phase2'] - if suffix in unsupported_types: + if any(ut in this_prefix_basename for ut in unsupported_types): return this_prefix_basename # Get the EchoNumber from json file info. If not present, use EchoTime @@ -282,38 +281,45 @@ def update_multiecho_name(fileinfo, this_prefix_basename, echo_times, suffix): echo_number = fileinfo['EchoNumber'] else: echo_number = echo_times.index(fileinfo['EchoTime']) + 1 + filetype = '_' + this_prefix_basename.split('_')[-1] - # Now, decide where to insert it. # Insert it **before** the following string(s), whichever appears first. - for imgtype in supported_multiecho: - if (imgtype in this_prefix_basename): + for label in ['_run', filetype]: + if label == filetype: this_prefix_basename = this_prefix_basename.replace( - imgtype, "_echo-%d%s" % (echo_number, imgtype) + filetype, "_echo-%s%s" % (echo_number, filetype) ) break + elif (label in this_prefix_basename): + this_prefix_basename = this_prefix_basename.replace( + label, "_echo-%s%s" % (echo_number, label) + ) + break + return this_prefix_basename -def update_uncombined_name(fileinfo, this_prefix_basename, coil_names, suffix): +def update_uncombined_name(fileinfo, this_prefix_basename, channel_names, suffix): """ Insert `_channel-` entity into filename if data are from a sequence with "save uncombined". """ # Determine the channel number - channel_number = coil_names.index(fileinfo['CoilString']) + 1 + channel_number = ''.join([c for c in fileinfo['CoilString'] if c.isdigit()]) + if not channel_number: + channel_number = channel_names.index(fileinfo['CoilString']) + 1 + filetype = '_' + this_prefix_basename.split('_')[-1] # Insert it **before** the following string(s), whichever appears first. - for label in ['_run', '_echo', suffix]: - if label == suffix: - prefix_suffix = this_prefix_basename.split('_')[-1] + for label in ['_run', '_echo', filetype]: + if label == filetype: this_prefix_basename = this_prefix_basename.replace( - prefix_suffix, "_channel-%s_%s" % (channel_number, prefix_suffix) + filetype, "_channel-%s%s" % (channel_number, filetype) ) break - - if (label in this_prefix_basename): + elif (label in this_prefix_basename): this_prefix_basename = this_prefix_basename.replace( - label, "_channel-%s%s" % (coil_number, label) + label, "_channel-%s%s" % (channel_number, label) ) break return this_prefix_basename @@ -630,14 +636,13 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam channel_names = [v for v in channel_names if v] channel_names = sorted(list(set(channel_names))) image_types = [v for v in image_types if v] - image_types = sorted(list(set(image_types))) is_multiecho = len(echo_times) > 1 # Check for varying echo times - is_uncombined = len(coil_names) > 1 # Check for uncombined data + is_uncombined = len(channel_names) > 1 # Check for uncombined data # Determine if data are complex (magnitude + phase) - magnitude_found = ['M' in it for it in image_types] - phase_found = ['P' in it for it in image_types] + magnitude_found = any(['M' in it for it in image_types]) + phase_found = any(['P' in it for it in image_types]) is_complex = magnitude_found and phase_found ### Loop through the bids_files, set the output name and save files @@ -646,18 +651,11 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # TODO: monitor conversion duration if bids_file: fileinfo = load_json(bids_file) - print(suffix) # set the prefix basename for this specific file (we'll modify it, # and we don't want to modify it for all the bids_files): this_prefix_basename = prefix_basename - # Update name if complex data - if bids_file and is_complex: - this_prefix_basename = update_complex_name( - fileinfo, this_prefix_basename, suffix - ) - # Update name if multi-echo # (Note: it can be _sbref and multiecho, so don't use "elif"): # For multi-echo sequences, we have to specify the echo number in @@ -667,10 +665,16 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam fileinfo, this_prefix_basename, echo_times, suffix ) + # Update name if complex data + if bids_file and is_complex: + this_prefix_basename = update_complex_name( + fileinfo, this_prefix_basename, suffix + ) + # Update name if uncombined (channel-level) data if bids_file and is_uncombined: this_prefix_basename = update_uncombined_name( - fileinfo, this_prefix_basename, channel_names + fileinfo, this_prefix_basename, channel_names, suffix ) # Fallback option: From 7858bbc1227d60d587bb39c5bc965bbd09d41d92 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Mon, 24 Feb 2020 09:16:36 -0500 Subject: [PATCH 03/11] Change part back to rec. And mag back to magnitude. --- heudiconv/convert.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index b5b81110..79cdeb21 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -232,47 +232,52 @@ def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid, def update_complex_name(fileinfo, this_prefix_basename, suffix): """ - Insert `_part-` entity into filename if data are from a sequence + Insert `_rec-` entity into filename if data are from a sequence with magnitude/phase reconstruction. """ + # Functional scans separate magnitude/phase differently unsupported_types = ['_bold', '_phase'] if any(ut in this_prefix_basename for ut in unsupported_types): return this_prefix_basename # Check to see if it is magnitude or phase reconstruction: if 'M' in fileinfo.get('ImageType'): - mag_or_phase = 'mag' + mag_or_phase = 'magnitude' elif 'P' in fileinfo.get('ImageType'): mag_or_phase = 'phase' else: mag_or_phase = suffix # Insert reconstruction label - if not ('_part-%s' % mag_or_phase) in this_prefix_basename: - # If "_part-" is specified, prepend the 'mag_or_phase' value. - if '_part-' in this_prefix_basename: + if not ('_rec-%s' % mag_or_phase) in this_prefix_basename: + # If "_rec-" is specified, prepend the 'mag_or_phase' value. + if '_rec-' in this_prefix_basename: raise BIDSError( - "Part label for images will be automatically set, remove " + "Rec label for images will be automatically set, remove " "from heuristic" ) - # If not, insert "_part-" + 'mag_or_phase' into the prefix_basename + # If not, insert "_rec-" + 'mag_or_phase' into the prefix_basename # **before** "_run", "_echo" or "_sbref", whichever appears first: for label in ['_run', '_echo', '_sbref']: if (label in this_prefix_basename): this_prefix_basename = this_prefix_basename.replace( - label, "_part-%s%s" % (mag_or_phase, label) + label, "_rec-%s%s" % (mag_or_phase, label) ) break return this_prefix_basename -def update_multiecho_name(fileinfo, this_prefix_basename, echo_times, suffix): +def update_multiecho_name(fileinfo, this_prefix_basename, echo_times): """ Insert `_echo-` entity into filename if data are from a multi-echo sequence. """ - unsupported_types = ['_magnitude1', '_magnitude2', '_phasediff', '_phase1', '_phase2'] + # Field maps separate echoes differently + unsupported_types = [ + '_magnitude', '_magnitude1', '_magnitude2', + '_phasediff', '_phase1', '_phase2', '_fieldmap' + ] if any(ut in this_prefix_basename for ut in unsupported_types): return this_prefix_basename @@ -299,11 +304,16 @@ def update_multiecho_name(fileinfo, this_prefix_basename, echo_times, suffix): return this_prefix_basename -def update_uncombined_name(fileinfo, this_prefix_basename, channel_names, suffix): +def update_uncombined_name(fileinfo, this_prefix_basename, channel_names): """ Insert `_channel-` entity into filename if data are from a sequence with "save uncombined". """ + # In case any scan types separate channels differently + unsupported_types = [] + if any(ut in this_prefix_basename for ut in unsupported_types): + return this_prefix_basename + # Determine the channel number channel_number = ''.join([c for c in fileinfo['CoilString'] if c.isdigit()]) if not channel_number: @@ -657,12 +667,9 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam this_prefix_basename = prefix_basename # Update name if multi-echo - # (Note: it can be _sbref and multiecho, so don't use "elif"): - # For multi-echo sequences, we have to specify the echo number in - # the file name: if bids_file and is_multiecho: this_prefix_basename = update_multiecho_name( - fileinfo, this_prefix_basename, echo_times, suffix + fileinfo, this_prefix_basename, echo_times ) # Update name if complex data @@ -674,7 +681,7 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # Update name if uncombined (channel-level) data if bids_file and is_uncombined: this_prefix_basename = update_uncombined_name( - fileinfo, this_prefix_basename, channel_names, suffix + fileinfo, this_prefix_basename, channel_names ) # Fallback option: From 788bd96362b33523574e51982bfd8101c4086ebe Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Mon, 24 Feb 2020 09:22:56 -0500 Subject: [PATCH 04/11] Update entity orders based on entity table. --- heudiconv/convert.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index 79cdeb21..a3f0421e 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -248,6 +248,9 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): else: mag_or_phase = suffix + # Determine scan suffix + filetype = '_' + this_prefix_basename.split('_')[-1] + # Insert reconstruction label if not ('_rec-%s' % mag_or_phase) in this_prefix_basename: # If "_rec-" is specified, prepend the 'mag_or_phase' value. @@ -259,12 +262,18 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): # If not, insert "_rec-" + 'mag_or_phase' into the prefix_basename # **before** "_run", "_echo" or "_sbref", whichever appears first: - for label in ['_run', '_echo', '_sbref']: - if (label in this_prefix_basename): + for label in ['_dir', '_run', '_mod', '_echo', '_recording', '_proc', '_space', filetype]: + if label == filetype: + this_prefix_basename = this_prefix_basename.replace( + filetype, "_rec-%s%s" % (mag_or_phase, filetype) + ) + break + elif (label in this_prefix_basename): this_prefix_basename = this_prefix_basename.replace( label, "_rec-%s%s" % (mag_or_phase, label) ) break + return this_prefix_basename @@ -286,10 +295,12 @@ def update_multiecho_name(fileinfo, this_prefix_basename, echo_times): echo_number = fileinfo['EchoNumber'] else: echo_number = echo_times.index(fileinfo['EchoTime']) + 1 + + # Determine scan suffix filetype = '_' + this_prefix_basename.split('_')[-1] # Insert it **before** the following string(s), whichever appears first. - for label in ['_run', filetype]: + for label in ['_recording', '_proc', '_space', filetype]: if label == filetype: this_prefix_basename = this_prefix_basename.replace( filetype, "_echo-%s%s" % (echo_number, filetype) @@ -318,10 +329,13 @@ def update_uncombined_name(fileinfo, this_prefix_basename, channel_names): channel_number = ''.join([c for c in fileinfo['CoilString'] if c.isdigit()]) if not channel_number: channel_number = channel_names.index(fileinfo['CoilString']) + 1 + + # Determine scan suffix filetype = '_' + this_prefix_basename.split('_')[-1] # Insert it **before** the following string(s), whichever appears first. - for label in ['_run', '_echo', filetype]: + # Choosing to put channel near the end since it's not in the specification yet. + for label in ['_recording', '_proc', '_space', filetype]: if label == filetype: this_prefix_basename = this_prefix_basename.replace( filetype, "_channel-%s%s" % (channel_number, filetype) From bdcbf2ef07324884c4412f70e38e4b4ca284b61b Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Sun, 23 Feb 2020 17:53:06 -0500 Subject: [PATCH 05/11] Add functions for renaming complex, multi-echo, and uncombined files. --- heudiconv/convert.py | 192 ++++++++++++++++++++++++++++--------------- 1 file changed, 128 insertions(+), 64 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index 7a0ea36f..bbf1455f 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -233,6 +233,95 @@ def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid, getattr(heuristic, 'DEFAULT_FIELDS', {})) +def update_complex_name(fileinfo, this_prefix_basename, suffix): + """ + Insert `_part-` entity into filename if data are from a sequence + with magnitude/phase reconstruction. + """ + unsupported_types = ['_bold', '_phase'] + if suffix in unsupported_types: + return this_prefix_basename + + # Check to see if it is magnitude or phase reconstruction: + if 'M' in fileinfo.get('ImageType'): + mag_or_phase = 'mag' + elif 'P' in fileinfo.get('ImageType'): + mag_or_phase = 'phase' + else: + mag_or_phase = suffix + + # Insert reconstruction label + if not ('_part-%s' % mag_or_phase) in this_prefix_basename: + + # If "_part-" is specified, prepend the 'mag_or_phase' value. + if '_part-' in this_prefix_basename: + raise BIDSError( + "Part label for images will be automatically set, remove " + "from heuristic" + ) + + # If not, insert "_part-" + 'mag_or_phase' into the prefix_basename + # **before** "_run", "_echo" or "_sbref", whichever appears first: + for label in ['_run', '_echo', '_sbref']: + if (label in this_prefix_basename): + this_prefix_basename = this_prefix_basename.replace( + label, "_part-%s%s" % (mag_or_phase, label) + ) + break + return this_prefix_basename + + +def update_multiecho_name(fileinfo, this_prefix_basename, echo_times, suffix): + """ + Insert `_echo-` entity into filename if data are from a multi-echo + sequence. + """ + unsupported_types = ['_magnitude1', '_magnitude2', '_phasediff', '_phase1', '_phase2'] + if suffix in unsupported_types: + return this_prefix_basename + + # Get the EchoNumber from json file info. If not present, use EchoTime + if 'EchoNumber' in fileinfo.keys(): + echo_number = fileinfo['EchoNumber'] + else: + echo_number = echo_times.index(fileinfo['EchoTime']) + 1 + + # Now, decide where to insert it. + # Insert it **before** the following string(s), whichever appears first. + for imgtype in supported_multiecho: + if (imgtype in this_prefix_basename): + this_prefix_basename = this_prefix_basename.replace( + imgtype, "_echo-%d%s" % (echo_number, imgtype) + ) + break + return this_prefix_basename + + +def update_uncombined_name(fileinfo, this_prefix_basename, coil_names, suffix): + """ + Insert `_channel-` entity into filename if data are from a sequence + with "save uncombined". + """ + # Determine the channel number + channel_number = coil_names.index(fileinfo['CoilString']) + 1 + + # Insert it **before** the following string(s), whichever appears first. + for label in ['_run', '_echo', suffix]: + if label == suffix: + prefix_suffix = this_prefix_basename.split('_')[-1] + this_prefix_basename = this_prefix_basename.replace( + prefix_suffix, "_channel-%s_%s" % (channel_number, prefix_suffix) + ) + break + + if (label in this_prefix_basename): + this_prefix_basename = this_prefix_basename.replace( + label, "_channel-%s%s" % (coil_number, label) + ) + break + return this_prefix_basename + + def convert(items, converter, scaninfo_suffix, custom_callable, with_prov, bids_options, outdir, min_meta, overwrite, symlink=True, prov_file=None, dcmconfig=None): @@ -534,14 +623,28 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # series. To do that, the most straightforward way is to read the # echo times for all bids_files and see if they are all the same or not. - # Check for varying echo times - echo_times = sorted(list(set( - b.get('EchoTime', nan) - for b in bids_metas - if b - ))) - - is_multiecho = len(echo_times) > 1 + # Collect some metadata across all images + echo_times, channel_names, image_types = [], [], [] + for metadata in bids_metas: + if not metadata: + continue + echo_times.append(metadata.get('EchoTime', None)) + channel_names.append(metadata.get('CoilString', None)) + image_types.append(metadata.get('ImageType', None)) + echo_times = [v for v in echo_times if v] + echo_times = sorted(list(set(echo_times))) + channel_names = [v for v in channel_names if v] + channel_names = sorted(list(set(channel_names))) + image_types = [v for v in image_types if v] + image_types = sorted(list(set(image_types))) + + is_multiecho = len(echo_times) > 1 # Check for varying echo times + is_uncombined = len(coil_names) > 1 # Check for uncombined data + + # Determine if data are complex (magnitude + phase) + magnitude_found = ['M' in it for it in image_types] + phase_found = ['P' in it for it in image_types] + is_complex = magnitude_found and phase_found ### Loop through the bids_files, set the output name and save files for fl, suffix, bids_file, bids_meta in zip(res_files, suffixes, bids_files, bids_metas): @@ -552,65 +655,26 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # and we don't want to modify it for all the bids_files): this_prefix_basename = prefix_basename - # _sbref sequences reconstructing magnitude and phase generate - # two NIfTI files IN THE SAME SERIES, so we cannot just add - # the suffix, if we want to be bids compliant: - if bids_meta and this_prefix_basename.endswith('_sbref') \ - and len(suffixes) > len(echo_times): - if len(suffixes) != len(echo_times)*2: - lgr.warning( - "Got %d suffixes for %d echo times, which isn't " - "multiple of two as if it was magnitude + phase pairs", - len(suffixes), len(echo_times) - ) - # Check to see if it is magnitude or phase reconstruction: - if 'M' in bids_meta.get('ImageType'): - mag_or_phase = 'magnitude' - elif 'P' in bids_meta.get('ImageType'): - mag_or_phase = 'phase' - else: - mag_or_phase = suffix - - # Insert reconstruction label - if not ("_rec-%s" % mag_or_phase) in this_prefix_basename: - - # If "_rec-" is specified, prepend the 'mag_or_phase' value. - if ('_rec-' in this_prefix_basename): - raise BIDSError( - "Reconstruction label for multi-echo single-band" - " reference images will be automatically set, remove" - " from heuristic" - ) - - # If not, insert "_rec-" + 'mag_or_phase' into the prefix_basename - # **before** "_run", "_echo" or "_sbref", whichever appears first: - for label in ['_run', '_echo', '_sbref']: - if (label in this_prefix_basename): - this_prefix_basename = this_prefix_basename.replace( - label, "_rec-%s%s" % (mag_or_phase, label) - ) - break - - # Now check if this run is multi-echo + # Update name if complex data + if bids_file and is_complex: + this_prefix_basename = update_complex_name( + bids_meta, this_prefix_basename, suffix + ) + + # Update name if multi-echo # (Note: it can be _sbref and multiecho, so don't use "elif"): # For multi-echo sequences, we have to specify the echo number in # the file name: - if bids_meta and is_multiecho: - # Get the EchoNumber from json file info. If not present, use EchoTime - if 'EchoNumber' in bids_meta: - echo_number = bids_meta['EchoNumber'] - else: - echo_number = echo_times.index(bids_meta['EchoTime']) + 1 - - supported_multiecho = ['_bold', '_phase', '_epi', '_sbref', '_T1w', '_PDT2'] - # Now, decide where to insert it. - # Insert it **before** the following string(s), whichever appears first. - for imgtype in supported_multiecho: - if (imgtype in this_prefix_basename): - this_prefix_basename = this_prefix_basename.replace( - imgtype, "_echo-%d%s" % (echo_number, imgtype) - ) - break + if bids_file and is_multiecho: + this_prefix_basename = update_multiecho_name( + bids_meta, this_prefix_basename, echo_times, suffix + ) + + # Update name if uncombined (channel-level) data + if bids_file and is_uncombined: + this_prefix_basename = update_uncombined_name( + bids_meta, this_prefix_basename, channel_names + ) # Fallback option: # If we have failed to modify this_prefix_basename, because it didn't fall From 85189f1e7784d2f75131e367bb4b085eb3d8fbe9 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Mon, 24 Feb 2020 09:12:00 -0500 Subject: [PATCH 06/11] Fix bugs. --- heudiconv/convert.py | 61 ++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index bbf1455f..59878eb2 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -239,7 +239,7 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): with magnitude/phase reconstruction. """ unsupported_types = ['_bold', '_phase'] - if suffix in unsupported_types: + if any(ut in this_prefix_basename for ut in unsupported_types): return this_prefix_basename # Check to see if it is magnitude or phase reconstruction: @@ -252,7 +252,6 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): # Insert reconstruction label if not ('_part-%s' % mag_or_phase) in this_prefix_basename: - # If "_part-" is specified, prepend the 'mag_or_phase' value. if '_part-' in this_prefix_basename: raise BIDSError( @@ -261,7 +260,7 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): ) # If not, insert "_part-" + 'mag_or_phase' into the prefix_basename - # **before** "_run", "_echo" or "_sbref", whichever appears first: + # **before** "_run", "_echo" or "_sbref", whichever appears first: for label in ['_run', '_echo', '_sbref']: if (label in this_prefix_basename): this_prefix_basename = this_prefix_basename.replace( @@ -277,7 +276,7 @@ def update_multiecho_name(fileinfo, this_prefix_basename, echo_times, suffix): sequence. """ unsupported_types = ['_magnitude1', '_magnitude2', '_phasediff', '_phase1', '_phase2'] - if suffix in unsupported_types: + if any(ut in this_prefix_basename for ut in unsupported_types): return this_prefix_basename # Get the EchoNumber from json file info. If not present, use EchoTime @@ -285,38 +284,45 @@ def update_multiecho_name(fileinfo, this_prefix_basename, echo_times, suffix): echo_number = fileinfo['EchoNumber'] else: echo_number = echo_times.index(fileinfo['EchoTime']) + 1 + filetype = '_' + this_prefix_basename.split('_')[-1] - # Now, decide where to insert it. # Insert it **before** the following string(s), whichever appears first. - for imgtype in supported_multiecho: - if (imgtype in this_prefix_basename): + for label in ['_run', filetype]: + if label == filetype: this_prefix_basename = this_prefix_basename.replace( - imgtype, "_echo-%d%s" % (echo_number, imgtype) + filetype, "_echo-%s%s" % (echo_number, filetype) ) break + elif (label in this_prefix_basename): + this_prefix_basename = this_prefix_basename.replace( + label, "_echo-%s%s" % (echo_number, label) + ) + break + return this_prefix_basename -def update_uncombined_name(fileinfo, this_prefix_basename, coil_names, suffix): +def update_uncombined_name(fileinfo, this_prefix_basename, channel_names, suffix): """ Insert `_channel-` entity into filename if data are from a sequence with "save uncombined". """ # Determine the channel number - channel_number = coil_names.index(fileinfo['CoilString']) + 1 + channel_number = ''.join([c for c in fileinfo['CoilString'] if c.isdigit()]) + if not channel_number: + channel_number = channel_names.index(fileinfo['CoilString']) + 1 + filetype = '_' + this_prefix_basename.split('_')[-1] # Insert it **before** the following string(s), whichever appears first. - for label in ['_run', '_echo', suffix]: - if label == suffix: - prefix_suffix = this_prefix_basename.split('_')[-1] + for label in ['_run', '_echo', filetype]: + if label == filetype: this_prefix_basename = this_prefix_basename.replace( - prefix_suffix, "_channel-%s_%s" % (channel_number, prefix_suffix) + filetype, "_channel-%s%s" % (channel_number, filetype) ) break - - if (label in this_prefix_basename): + elif (label in this_prefix_basename): this_prefix_basename = this_prefix_basename.replace( - label, "_channel-%s%s" % (coil_number, label) + label, "_channel-%s%s" % (channel_number, label) ) break return this_prefix_basename @@ -636,14 +642,13 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam channel_names = [v for v in channel_names if v] channel_names = sorted(list(set(channel_names))) image_types = [v for v in image_types if v] - image_types = sorted(list(set(image_types))) is_multiecho = len(echo_times) > 1 # Check for varying echo times - is_uncombined = len(coil_names) > 1 # Check for uncombined data + is_uncombined = len(channel_names) > 1 # Check for uncombined data # Determine if data are complex (magnitude + phase) - magnitude_found = ['M' in it for it in image_types] - phase_found = ['P' in it for it in image_types] + magnitude_found = any(['M' in it for it in image_types]) + phase_found = any(['P' in it for it in image_types]) is_complex = magnitude_found and phase_found ### Loop through the bids_files, set the output name and save files @@ -655,12 +660,6 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # and we don't want to modify it for all the bids_files): this_prefix_basename = prefix_basename - # Update name if complex data - if bids_file and is_complex: - this_prefix_basename = update_complex_name( - bids_meta, this_prefix_basename, suffix - ) - # Update name if multi-echo # (Note: it can be _sbref and multiecho, so don't use "elif"): # For multi-echo sequences, we have to specify the echo number in @@ -670,10 +669,16 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam bids_meta, this_prefix_basename, echo_times, suffix ) + # Update name if complex data + if bids_file and is_complex: + this_prefix_basename = update_complex_name( + bids_meta, this_prefix_basename, suffix + ) + # Update name if uncombined (channel-level) data if bids_file and is_uncombined: this_prefix_basename = update_uncombined_name( - bids_meta, this_prefix_basename, channel_names + bids_meta, this_prefix_basename, channel_names, suffix ) # Fallback option: From aaf199ec6f4062c63be879503f5ef42eb72110af Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Mon, 24 Feb 2020 09:16:36 -0500 Subject: [PATCH 07/11] Change part back to rec. And mag back to magnitude. --- heudiconv/convert.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index 59878eb2..1a91d87e 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -235,47 +235,52 @@ def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid, def update_complex_name(fileinfo, this_prefix_basename, suffix): """ - Insert `_part-` entity into filename if data are from a sequence + Insert `_rec-` entity into filename if data are from a sequence with magnitude/phase reconstruction. """ + # Functional scans separate magnitude/phase differently unsupported_types = ['_bold', '_phase'] if any(ut in this_prefix_basename for ut in unsupported_types): return this_prefix_basename # Check to see if it is magnitude or phase reconstruction: if 'M' in fileinfo.get('ImageType'): - mag_or_phase = 'mag' + mag_or_phase = 'magnitude' elif 'P' in fileinfo.get('ImageType'): mag_or_phase = 'phase' else: mag_or_phase = suffix # Insert reconstruction label - if not ('_part-%s' % mag_or_phase) in this_prefix_basename: - # If "_part-" is specified, prepend the 'mag_or_phase' value. - if '_part-' in this_prefix_basename: + if not ('_rec-%s' % mag_or_phase) in this_prefix_basename: + # If "_rec-" is specified, prepend the 'mag_or_phase' value. + if '_rec-' in this_prefix_basename: raise BIDSError( - "Part label for images will be automatically set, remove " + "Rec label for images will be automatically set, remove " "from heuristic" ) - # If not, insert "_part-" + 'mag_or_phase' into the prefix_basename + # If not, insert "_rec-" + 'mag_or_phase' into the prefix_basename # **before** "_run", "_echo" or "_sbref", whichever appears first: for label in ['_run', '_echo', '_sbref']: if (label in this_prefix_basename): this_prefix_basename = this_prefix_basename.replace( - label, "_part-%s%s" % (mag_or_phase, label) + label, "_rec-%s%s" % (mag_or_phase, label) ) break return this_prefix_basename -def update_multiecho_name(fileinfo, this_prefix_basename, echo_times, suffix): +def update_multiecho_name(fileinfo, this_prefix_basename, echo_times): """ Insert `_echo-` entity into filename if data are from a multi-echo sequence. """ - unsupported_types = ['_magnitude1', '_magnitude2', '_phasediff', '_phase1', '_phase2'] + # Field maps separate echoes differently + unsupported_types = [ + '_magnitude', '_magnitude1', '_magnitude2', + '_phasediff', '_phase1', '_phase2', '_fieldmap' + ] if any(ut in this_prefix_basename for ut in unsupported_types): return this_prefix_basename @@ -302,11 +307,16 @@ def update_multiecho_name(fileinfo, this_prefix_basename, echo_times, suffix): return this_prefix_basename -def update_uncombined_name(fileinfo, this_prefix_basename, channel_names, suffix): +def update_uncombined_name(fileinfo, this_prefix_basename, channel_names): """ Insert `_channel-` entity into filename if data are from a sequence with "save uncombined". """ + # In case any scan types separate channels differently + unsupported_types = [] + if any(ut in this_prefix_basename for ut in unsupported_types): + return this_prefix_basename + # Determine the channel number channel_number = ''.join([c for c in fileinfo['CoilString'] if c.isdigit()]) if not channel_number: @@ -661,12 +671,9 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam this_prefix_basename = prefix_basename # Update name if multi-echo - # (Note: it can be _sbref and multiecho, so don't use "elif"): - # For multi-echo sequences, we have to specify the echo number in - # the file name: if bids_file and is_multiecho: this_prefix_basename = update_multiecho_name( - bids_meta, this_prefix_basename, echo_times, suffix + bids_meta, this_prefix_basename, echo_times ) # Update name if complex data @@ -678,7 +685,7 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # Update name if uncombined (channel-level) data if bids_file and is_uncombined: this_prefix_basename = update_uncombined_name( - bids_meta, this_prefix_basename, channel_names, suffix + bids_meta, this_prefix_basename, channel_names ) # Fallback option: From a2d9099527a2b807ac6bc3b935b485f110297206 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Mon, 24 Feb 2020 09:22:56 -0500 Subject: [PATCH 08/11] Update entity orders based on entity table. --- heudiconv/convert.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index 1a91d87e..5b93614b 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -251,6 +251,9 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): else: mag_or_phase = suffix + # Determine scan suffix + filetype = '_' + this_prefix_basename.split('_')[-1] + # Insert reconstruction label if not ('_rec-%s' % mag_or_phase) in this_prefix_basename: # If "_rec-" is specified, prepend the 'mag_or_phase' value. @@ -262,12 +265,18 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): # If not, insert "_rec-" + 'mag_or_phase' into the prefix_basename # **before** "_run", "_echo" or "_sbref", whichever appears first: - for label in ['_run', '_echo', '_sbref']: - if (label in this_prefix_basename): + for label in ['_dir', '_run', '_mod', '_echo', '_recording', '_proc', '_space', filetype]: + if label == filetype: + this_prefix_basename = this_prefix_basename.replace( + filetype, "_rec-%s%s" % (mag_or_phase, filetype) + ) + break + elif (label in this_prefix_basename): this_prefix_basename = this_prefix_basename.replace( label, "_rec-%s%s" % (mag_or_phase, label) ) break + return this_prefix_basename @@ -289,10 +298,12 @@ def update_multiecho_name(fileinfo, this_prefix_basename, echo_times): echo_number = fileinfo['EchoNumber'] else: echo_number = echo_times.index(fileinfo['EchoTime']) + 1 + + # Determine scan suffix filetype = '_' + this_prefix_basename.split('_')[-1] # Insert it **before** the following string(s), whichever appears first. - for label in ['_run', filetype]: + for label in ['_recording', '_proc', '_space', filetype]: if label == filetype: this_prefix_basename = this_prefix_basename.replace( filetype, "_echo-%s%s" % (echo_number, filetype) @@ -321,10 +332,13 @@ def update_uncombined_name(fileinfo, this_prefix_basename, channel_names): channel_number = ''.join([c for c in fileinfo['CoilString'] if c.isdigit()]) if not channel_number: channel_number = channel_names.index(fileinfo['CoilString']) + 1 + + # Determine scan suffix filetype = '_' + this_prefix_basename.split('_')[-1] # Insert it **before** the following string(s), whichever appears first. - for label in ['_run', '_echo', filetype]: + # Choosing to put channel near the end since it's not in the specification yet. + for label in ['_recording', '_proc', '_space', filetype]: if label == filetype: this_prefix_basename = this_prefix_basename.replace( filetype, "_channel-%s%s" % (channel_number, filetype) From 3b29db8a566da4fa63d73f60bd3c870f0442a441 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 7 May 2020 16:16:54 -0400 Subject: [PATCH 09/11] Change channel to ch and rec to part. --- heudiconv/convert.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index a3f0421e..fa4e6863 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -232,17 +232,17 @@ def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid, def update_complex_name(fileinfo, this_prefix_basename, suffix): """ - Insert `_rec-` entity into filename if data are from a sequence - with magnitude/phase reconstruction. + Insert `_part-` entity into filename if data are from a sequence + with magnitude/phase part. """ # Functional scans separate magnitude/phase differently unsupported_types = ['_bold', '_phase'] if any(ut in this_prefix_basename for ut in unsupported_types): return this_prefix_basename - # Check to see if it is magnitude or phase reconstruction: + # Check to see if it is magnitude or phase part: if 'M' in fileinfo.get('ImageType'): - mag_or_phase = 'magnitude' + mag_or_phase = 'mag' elif 'P' in fileinfo.get('ImageType'): mag_or_phase = 'phase' else: @@ -251,26 +251,26 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): # Determine scan suffix filetype = '_' + this_prefix_basename.split('_')[-1] - # Insert reconstruction label - if not ('_rec-%s' % mag_or_phase) in this_prefix_basename: - # If "_rec-" is specified, prepend the 'mag_or_phase' value. - if '_rec-' in this_prefix_basename: + # Insert part label + if not ('_part-%s' % mag_or_phase) in this_prefix_basename: + # If "_part-" is specified, prepend the 'mag_or_phase' value. + if '_part-' in this_prefix_basename: raise BIDSError( - "Rec label for images will be automatically set, remove " + "Part label for images will be automatically set, remove " "from heuristic" ) - # If not, insert "_rec-" + 'mag_or_phase' into the prefix_basename + # If not, insert "_part-" + 'mag_or_phase' into the prefix_basename # **before** "_run", "_echo" or "_sbref", whichever appears first: for label in ['_dir', '_run', '_mod', '_echo', '_recording', '_proc', '_space', filetype]: if label == filetype: this_prefix_basename = this_prefix_basename.replace( - filetype, "_rec-%s%s" % (mag_or_phase, filetype) + filetype, "_part-%s%s" % (mag_or_phase, filetype) ) break elif (label in this_prefix_basename): this_prefix_basename = this_prefix_basename.replace( - label, "_rec-%s%s" % (mag_or_phase, label) + label, "_part-%s%s" % (mag_or_phase, label) ) break @@ -317,7 +317,7 @@ def update_multiecho_name(fileinfo, this_prefix_basename, echo_times): def update_uncombined_name(fileinfo, this_prefix_basename, channel_names): """ - Insert `_channel-` entity into filename if data are from a sequence + Insert `_ch-` entity into filename if data are from a sequence with "save uncombined". """ # In case any scan types separate channels differently @@ -329,6 +329,7 @@ def update_uncombined_name(fileinfo, this_prefix_basename, channel_names): channel_number = ''.join([c for c in fileinfo['CoilString'] if c.isdigit()]) if not channel_number: channel_number = channel_names.index(fileinfo['CoilString']) + 1 + channel_number = channel_number.zfill(2) # Determine scan suffix filetype = '_' + this_prefix_basename.split('_')[-1] @@ -338,12 +339,12 @@ def update_uncombined_name(fileinfo, this_prefix_basename, channel_names): for label in ['_recording', '_proc', '_space', filetype]: if label == filetype: this_prefix_basename = this_prefix_basename.replace( - filetype, "_channel-%s%s" % (channel_number, filetype) + filetype, "_ch-%s%s" % (channel_number, filetype) ) break elif (label in this_prefix_basename): this_prefix_basename = this_prefix_basename.replace( - label, "_channel-%s%s" % (channel_number, label) + label, "_ch-%s%s" % (channel_number, label) ) break return this_prefix_basename From b640c16826e8016f3e4cd558bafec236d7789aab Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 7 May 2020 16:54:10 -0400 Subject: [PATCH 10/11] Merge remote-tracking branch 'nipy/master' into ref/modularize-multifile-renamers # Conflicts: # heudiconv/convert.py --- CHANGELOG.md | 80 ++++ Makefile | 6 + docs/conf.py | 2 +- docs/heuristics.rst | 17 + docs/installation.rst | 4 +- docs/tutorials.rst | 8 + docs/usage.rst | 4 +- heudiconv/bids.py | 12 +- heudiconv/cli/run.py | 24 +- heudiconv/convert.py | 82 +++- heudiconv/dicoms.py | 504 +++++++++++---------- heudiconv/external/dlad.py | 4 +- heudiconv/external/tests/test_dlad.py | 7 +- heudiconv/heuristics/reproin.py | 138 ++++-- heudiconv/heuristics/reproin_validator.cfg | 3 +- heudiconv/heuristics/test_reproin.py | 26 +- heudiconv/info.py | 8 +- heudiconv/parser.py | 26 +- heudiconv/tests/data/phantom.dcm | Bin 0 -> 162304 bytes heudiconv/tests/test_dicoms.py | 46 +- heudiconv/tests/test_main.py | 11 + heudiconv/tests/test_regression.py | 77 +++- heudiconv/tests/utils.py | 56 +++ heudiconv/utils.py | 71 +-- setup.py | 4 +- utils/prep_release | 14 + 26 files changed, 807 insertions(+), 427 deletions(-) create mode 100644 Makefile create mode 100644 heudiconv/tests/data/phantom.dcm create mode 100755 utils/prep_release diff --git a/CHANGELOG.md b/CHANGELOG.md index bc9070e4..5d4b1e2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,74 @@ All notable changes to this project will be documented (for humans) in this file The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [0.8.1] - Date + +TODO Summary + +### Added +### Changed +### Deprecated +### Fixed +### Removed +### Security + + +## [0.8.0] - 2020-04-15 + +### Enhancements + +- Centralized saving of .json files. Indentation of some files could + change now from previous versions where it could have used `3` + spaces. Now indentation should be consistently `2` for .json files + we produce/modify ([#436][]) (note: dcm2niix uses tabs for indentation) +- ReproIn heuristic: support SBRef and phase data ([#387][]) +- Set the "TaskName" field in .json sidecar files for multi-echo data + ([#420][]) +- Provide an informative exception if command needs heuristic to be + specified ([#437][]) + +### Refactored + +- `embed_nifti` was refactored into `embed_dicom_and_nifti_metadata` + which would no longer create `.nii` file if it does not exist + already ([#432][]) + +### Fixed + +- Skip datalad-based tests if no datalad available ([#430][]) +- Search heuristic file path first so we do not pick up a python + module if name conflicts ([#434][]) + +## [0.7.0] - 2020-03-20 + +### Removed + +- Python 2 support/testing + +### Enhancement + +- `-g` option obtained two new modes: `all` and `custom`. In case of `all`, + all provided DICOMs will be treated as coming from a single scanning session. + `custom` instructs to use `.grouping` value (could be a DICOM attribute or + a callable)provided by the heuristic ([#359][]). +- Stop before reading pixels data while gathering metadata from DICOMs ([#404][]) +- reproin heuristic: + - In addition to original "md5sum of the study_description" `protocols2fix` + could now have (and applied after md5sum matching ones) + 1). a regular expression searched in study_description, + 2). an empty string as "catch all". + This features could be used to easily provide remapping into reproin + naming (documentation is to come to http://github.com/ReproNim/reproin) + ([#425][]) + +### Fixed + +- Use nan, not None for absent echo value in sorting +- reproin heuristic: case seqinfos into a list to be able to modify from + overloaded heuristic ([#419][]) +- No spurious errors from the logger upon a warning about `etelemetry` + absence ([#407][]) + ## [0.6.0] - 2019-12-16 This is largely a bug fix. Metadata and order of `_key-value` fields in BIDS @@ -271,6 +339,7 @@ TODO Summary [#348]: https://github.com/nipy/heudiconv/issues/348 [#351]: https://github.com/nipy/heudiconv/issues/351 [#352]: https://github.com/nipy/heudiconv/issues/352 +[#359]: https://github.com/nipy/heudiconv/issues/359 [#360]: https://github.com/nipy/heudiconv/issues/360 [#364]: https://github.com/nipy/heudiconv/issues/364 [#369]: https://github.com/nipy/heudiconv/issues/369 @@ -279,4 +348,15 @@ TODO Summary [#376]: https://github.com/nipy/heudiconv/issues/376 [#379]: https://github.com/nipy/heudiconv/issues/379 [#380]: https://github.com/nipy/heudiconv/issues/380 +[#387]: https://github.com/nipy/heudiconv/issues/387 [#390]: https://github.com/nipy/heudiconv/issues/390 +[#404]: https://github.com/nipy/heudiconv/issues/404 +[#407]: https://github.com/nipy/heudiconv/issues/407 +[#419]: https://github.com/nipy/heudiconv/issues/419 +[#420]: https://github.com/nipy/heudiconv/issues/420 +[#425]: https://github.com/nipy/heudiconv/issues/425 +[#430]: https://github.com/nipy/heudiconv/issues/430 +[#432]: https://github.com/nipy/heudiconv/issues/432 +[#434]: https://github.com/nipy/heudiconv/issues/434 +[#436]: https://github.com/nipy/heudiconv/issues/436 +[#437]: https://github.com/nipy/heudiconv/issues/437 diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..e1198a70 --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ +all: + echo 'nothing by default' + +prep_release: + # take previous one, and replace with the next one + utils/prep_release diff --git a/docs/conf.py b/docs/conf.py index 2abf2136..9cb5fe1f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,7 +26,7 @@ # The short X.Y version version = '' # The full version, including alpha/beta/rc tags -release = '0.6.0' +release = '0.8.0' # -- General configuration --------------------------------------------------- diff --git a/docs/heuristics.rst b/docs/heuristics.rst index aee4f354..874cc9b4 100644 --- a/docs/heuristics.rst +++ b/docs/heuristics.rst @@ -68,3 +68,20 @@ DICOMs where this function returns ``True`` will be filtered out. Further processing on ``seqinfos`` to deduce/customize subject, session, and locator. A dictionary of {"locator": locator, "session": session, "subject": subject} is returned. + +--------------------------------------------------------------- +``grouping`` string or ``grouping(files, dcmfilter, seqinfo)`` +--------------------------------------------------------------- + +Whenever ``--grouping custom`` (``-g custom``) is used, this attribute or callable +will be used to inform how to group the DICOMs into separate groups. From +`original PR#359 `_:: + + grouping = 'AcquisitionDate' + +or:: + + def grouping(files, dcmfilter, seqinfo): + seqinfos = collections.OrderedDict() + ... + return seqinfos # ordered dict containing seqinfo objects: list of DICOMs \ No newline at end of file diff --git a/docs/installation.rst b/docs/installation.rst index 4a7be810..86668d38 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -26,7 +26,7 @@ If `Docker `_ is available on your system, you can visit `our page on Docker Hub `_ to view available releases. To pull the latest release, run:: - $ docker pull nipy/heudiconv:0.6.0 + $ docker pull nipy/heudiconv:0.8.0 Singularity @@ -35,4 +35,4 @@ If `Singularity `_ is available on your syst you can use it to pull and convert our Docker images! For example, to pull and build the latest release, you can run:: - $ singularity pull docker://nipy/heudiconv:0.6.0 + $ singularity pull docker://nipy/heudiconv:0.8.0 diff --git a/docs/tutorials.rst b/docs/tutorials.rst index 00aa7174..6074c5c4 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -13,6 +13,14 @@ other users' tutorials covering their experience with ``heudiconv``. - `Sample Conversion: Coastal Coding 2019 `_. +- `A joined DataLad and HeuDiConv tutorial for reproducible fMRI studies `_. + +- `The ReproIn conversion workflow overview `_. + +- `Slides `_ and + `recording `_ + of a ReproNim Webinar on ``heudiconv``. + .. caution:: Some of these tutorials may not be up to date with the latest releases of ``heudiconv``. diff --git a/docs/usage.rst b/docs/usage.rst index df1a14b1..21b5699a 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -82,7 +82,7 @@ The second script processes a DICOM directory with ``heudiconv`` using the built DCMDIR=${DCMDIRS[${SLURM_ARRAY_TASK_ID}]} echo Submitted directory: ${DCMDIR} - IMG="/singularity-images/heudiconv-0.6.0-dev.sif" + IMG="/singularity-images/heudiconv-0.8.0-dev.sif" CMD="singularity run -B ${DCMDIR}:/dicoms:ro -B ${OUTDIR}:/output -e ${IMG} --files /dicoms/ -o /output -f reproin -c dcm2niix -b notop --minmeta -l ." printf "Command:\n${CMD}\n" @@ -97,7 +97,7 @@ This script creates the top-level bids files (e.g., set -eu OUTDIR=${1} - IMG="/singularity-images/heudiconv-0.6.0-dev.sif" + IMG="/singularity-images/heudiconv-0.8.0-dev.sif" CMD="singularity run -B ${OUTDIR}:/output -e ${IMG} --files /output -f reproin --command populate-templates" printf "Command:\n${CMD}\n" diff --git a/heudiconv/bids.py b/heudiconv/bids.py index 3092cbf6..6a9c136c 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -171,7 +171,7 @@ def populate_aggregated_jsons(path): act = "Generating" lgr.debug("%s %s", act, task_file) fields.update(placeholders) - save_json(task_file, fields, indent=2, sort_keys=True, pretty=True) + save_json(task_file, fields, sort_keys=True, pretty=True) def tuneup_bids_json_files(json_files): @@ -193,7 +193,7 @@ def tuneup_bids_json_files(json_files): # Let's hope no word 'Date' comes within a study name or smth like # that raise ValueError("There must be no dates in .json sidecar") - save_json(jsonfile, json_, indent=2) + save_json(jsonfile, json_) # Load the beast seqtype = op.basename(op.dirname(jsonfile)) @@ -223,7 +223,7 @@ def tuneup_bids_json_files(json_files): was_readonly = is_readonly(json_phasediffname) if was_readonly: set_readonly(json_phasediffname, False) - save_json(json_phasediffname, json_, indent=2) + save_json(json_phasediffname, json_) if was_readonly: set_readonly(json_phasediffname) @@ -259,8 +259,7 @@ def add_participant_record(studydir, subject, age, sex): ("Description", "(TODO: adjust - by default everyone is in " "control group)")])), ]), - sort_keys=False, - indent=2) + sort_keys=False) # Add a new participant with open(participants_tsv, 'a') as f: f.write( @@ -373,8 +372,7 @@ def add_rows_to_scans_keys_file(fn, newrows): ("LongName", "Random string"), ("Description", "md5 hash of UIDs")])), ]), - sort_keys=False, - indent=2) + sort_keys=False) header = ['filename', 'acq_time', 'operator', 'randstr'] # prepare all the data rows diff --git a/heudiconv/cli/run.py b/heudiconv/cli/run.py index 516e1630..f6af3d1d 100644 --- a/heudiconv/cli/run.py +++ b/heudiconv/cli/run.py @@ -62,6 +62,7 @@ def process_extra_commands(outdir, args): for f in args.files: treat_infofile(f) elif args.command == 'ls': + ensure_heuristic_arg(args) heuristic = load_heuristic(args.heuristic) heuristic_ls = getattr(heuristic, 'ls', None) for f in args.files: @@ -78,6 +79,7 @@ def process_extra_commands(outdir, args): % (str(study_session), len(sequences), suf) ) elif args.command == 'populate-templates': + ensure_heuristic_arg(args) heuristic = load_heuristic(args.heuristic) for f in args.files: populate_bids_templates(f, getattr(heuristic, 'DEFAULT_FIELDS', {})) @@ -88,16 +90,21 @@ def process_extra_commands(outdir, args): for name_desc in get_known_heuristics_with_descriptions().items(): print("- %s: %s" % name_desc) elif args.command == 'heuristic-info': - from ..utils import get_heuristic_description, get_known_heuristic_names - if not args.heuristic: - raise ValueError("Specify heuristic using -f. Known are: %s" - % ', '.join(get_known_heuristic_names())) + ensure_heuristic_arg(args) + from ..utils import get_heuristic_description print(get_heuristic_description(args.heuristic, full=True)) else: raise ValueError("Unknown command %s", args.command) return +def ensure_heuristic_arg(args): + from ..utils import get_known_heuristic_names + if not args.heuristic: + raise ValueError("Specify heuristic using -f. Known are: %s" + % ', '.join(get_known_heuristic_names())) + + def main(argv=None): parser = get_parser() args = parser.parse_args(argv) @@ -124,7 +131,6 @@ def main(argv=None): if args.debug: setup_exceptionhook() - process_args(args) @@ -154,8 +160,7 @@ def get_parser(): 'If not provided, DICOMS would first be "sorted" and ' 'subject IDs deduced by the heuristic') parser.add_argument('-c', '--converter', - default='dcm2niix', - choices=('dcm2niix', 'none'), + choices=('dcm2niix', 'none'), default='dcm2niix', help='tool to use for DICOM conversion. Setting to ' '"none" disables the actual conversion step -- useful' 'for testing heuristics.') @@ -219,7 +224,7 @@ def get_parser(): help='custom actions to be performed on provided ' 'files instead of regular operation.') parser.add_argument('-g', '--grouping', default='studyUID', - choices=('studyUID', 'accession_number'), + choices=('studyUID', 'accession_number', 'all', 'custom'), help='How to group dicoms (default: by studyUID)') parser.add_argument('--minmeta', action='store_true', help='Exclude dcmstack meta information in sidecar ' @@ -343,7 +348,8 @@ def process_args(args): seqinfo=seqinfo, min_meta=args.minmeta, overwrite=args.overwrite, - dcmconfig=args.dcmconfig,) + dcmconfig=args.dcmconfig, + grouping=args.grouping,) lgr.info("PROCESSING DONE: {0}".format( str(dict(subject=sid, outdir=study_outdir, session=session)))) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index fa4e6863..f6efc06d 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -2,8 +2,10 @@ import os import os.path as op import logging +from math import nan import shutil import sys +import re from .utils import ( read_config, @@ -80,8 +82,8 @@ def conversion_info(subject, outdir, info, filegroup, ses): def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid, - anon_outdir, with_prov, ses, bids_options, seqinfo, min_meta, - overwrite, dcmconfig): + anon_outdir, with_prov, ses, bids_options, seqinfo, + min_meta, overwrite, dcmconfig, grouping): if dicoms: lgr.info("Processing %d dicoms", len(dicoms)) elif seqinfo: @@ -157,16 +159,17 @@ def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid, # So either it would need to be brought back or reconsidered altogether # (since no sample data to test on etc) else: - # TODO -- might have been done outside already! - # MG -- will have to try with both dicom template, files assure_no_file_exists(target_heuristic_filename) safe_copyfile(heuristic.filename, target_heuristic_filename) if dicoms: seqinfo = group_dicoms_into_seqinfos( dicoms, + grouping, file_filter=getattr(heuristic, 'filter_files', None), dcmfilter=getattr(heuristic, 'filter_dicom', None), - grouping=None) + flatten=True, + custom_grouping=getattr(heuristic, 'grouping', None)) + seqinfo_list = list(seqinfo.keys()) filegroup = {si.series_id: x for si, x in seqinfo.items()} dicominfo_file = op.join(idir, 'dicominfo%s.tsv' % ses_suffix) @@ -230,7 +233,7 @@ def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid, getattr(heuristic, 'DEFAULT_FIELDS', {})) -def update_complex_name(fileinfo, this_prefix_basename, suffix): +def update_complex_name(metadata, this_prefix_basename, suffix): """ Insert `_part-` entity into filename if data are from a sequence with magnitude/phase part. @@ -241,9 +244,9 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): return this_prefix_basename # Check to see if it is magnitude or phase part: - if 'M' in fileinfo.get('ImageType'): + if 'M' in metadata.get('ImageType'): mag_or_phase = 'mag' - elif 'P' in fileinfo.get('ImageType'): + elif 'P' in metadata.get('ImageType'): mag_or_phase = 'phase' else: mag_or_phase = suffix @@ -277,7 +280,7 @@ def update_complex_name(fileinfo, this_prefix_basename, suffix): return this_prefix_basename -def update_multiecho_name(fileinfo, this_prefix_basename, echo_times): +def update_multiecho_name(metadata, this_prefix_basename, echo_times): """ Insert `_echo-` entity into filename if data are from a multi-echo sequence. @@ -291,10 +294,10 @@ def update_multiecho_name(fileinfo, this_prefix_basename, echo_times): return this_prefix_basename # Get the EchoNumber from json file info. If not present, use EchoTime - if 'EchoNumber' in fileinfo.keys(): - echo_number = fileinfo['EchoNumber'] + if 'EchoNumber' in metadata.keys(): + echo_number = metadata['EchoNumber'] else: - echo_number = echo_times.index(fileinfo['EchoTime']) + 1 + echo_number = echo_times.index(metadata['EchoTime']) + 1 # Determine scan suffix filetype = '_' + this_prefix_basename.split('_')[-1] @@ -315,7 +318,7 @@ def update_multiecho_name(fileinfo, this_prefix_basename, echo_times): return this_prefix_basename -def update_uncombined_name(fileinfo, this_prefix_basename, channel_names): +def update_uncombined_name(metadata, this_prefix_basename, channel_names): """ Insert `_ch-` entity into filename if data are from a sequence with "save uncombined". @@ -326,9 +329,9 @@ def update_uncombined_name(fileinfo, this_prefix_basename, channel_names): return this_prefix_basename # Determine the channel number - channel_number = ''.join([c for c in fileinfo['CoilString'] if c.isdigit()]) + channel_number = ''.join([c for c in metadata['CoilString'] if c.isdigit()]) if not channel_number: - channel_number = channel_names.index(fileinfo['CoilString']) + 1 + channel_number = channel_names.index(metadata['CoilString']) + 1 channel_number = channel_number.zfill(2) # Determine scan suffix @@ -441,16 +444,18 @@ def convert(items, converter, scaninfo_suffix, custom_callable, with_prov, % (outname) ) + # add the taskname field to the json file(s): + add_taskname_to_infofile(bids_outfiles) + if len(bids_outfiles) > 1: lgr.warning("For now not embedding BIDS and info generated " ".nii.gz itself since sequence produced " "multiple files") elif not bids_outfiles: lgr.debug("No BIDS files were produced, nothing to embed to then") - elif outname: + elif outname and not min_meta: embed_metadata_from_dicoms(bids_options, item_dicoms, outname, outname_bids, - prov_file, scaninfo, tempdirs, with_prov, - min_meta) + prov_file, scaninfo, tempdirs, with_prov) if scaninfo and op.exists(scaninfo): lgr.info("Post-treating %s file", scaninfo) treat_infofile(scaninfo) @@ -636,6 +641,8 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam bids_files = (sorted(res.outputs.bids) if len(res.outputs.bids) == len(res_files) else [None] * len(res_files)) + # preload since will be used in multiple spots + bids_metas = [load_json(b) for b in bids_files if b] ### Do we have a multi-echo series? ### # Some Siemens sequences (e.g. CMRR's MB-EPI) set the label 'TE1', @@ -671,11 +678,9 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam is_complex = magnitude_found and phase_found ### Loop through the bids_files, set the output name and save files - for fl, suffix, bids_file in zip(res_files, suffixes, bids_files): + for fl, suffix, bids_file, bids_meta in zip(res_files, suffixes, bids_files, bids_metas): # TODO: monitor conversion duration - if bids_file: - fileinfo = load_json(bids_file) # set the prefix basename for this specific file (we'll modify it, # and we don't want to modify it for all the bids_files): @@ -684,19 +689,19 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # Update name if multi-echo if bids_file and is_multiecho: this_prefix_basename = update_multiecho_name( - fileinfo, this_prefix_basename, echo_times + bids_meta, this_prefix_basename, echo_times ) # Update name if complex data if bids_file and is_complex: this_prefix_basename = update_complex_name( - fileinfo, this_prefix_basename, suffix + bids_meta, this_prefix_basename, suffix ) # Update name if uncombined (channel-level) data if bids_file and is_uncombined: this_prefix_basename = update_uncombined_name( - fileinfo, this_prefix_basename, channel_names + bids_meta, this_prefix_basename, channel_names ) # Fallback option: @@ -727,3 +732,32 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam except TypeError as exc: ##catch lists raise TypeError("Multiple BIDS sidecars detected.") return bids_outfiles + + +def add_taskname_to_infofile(infofiles): + """Add the "TaskName" field to json files corresponding to func images. + + Parameters + ---------- + infofiles : list with json filenames or single filename + + Returns + ------- + """ + + # in case they pass a string with a path: + if not isinstance(infofiles, list): + infofiles = [infofiles] + + for infofile in infofiles: + meta_info = load_json(infofile) + try: + meta_info['TaskName'] = (re.search('(?<=_task-)\w+', + op.basename(infofile)) + .group(0).split('_')[0]) + except AttributeError: + lgr.warning("Failed to find task field in {0}.".format(infofile)) + continue + + # write to outfile + save_json(infofile, meta_info) diff --git a/heudiconv/dicoms.py b/heudiconv/dicoms.py index 7c8c450a..647ad8ca 100644 --- a/heudiconv/dicoms.py +++ b/heudiconv/dicoms.py @@ -6,25 +6,162 @@ import tarfile from .external.pydicom import dcm -from .utils import SeqInfo, load_json, set_readonly +from .utils import ( + get_typed_attr, + load_json, + save_json, + SeqInfo, + set_readonly, +) + +import warnings +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + # suppress warning + import nibabel.nicom.dicomwrappers as dw lgr = logging.getLogger(__name__) +total_files = 0 -def group_dicoms_into_seqinfos(files, file_filter, dcmfilter, grouping): + +def create_seqinfo(mw, series_files, series_id): + """Generate sequence info + + Parameters + ---------- + mw: MosaicWrapper + series_files: list + series_id: str + """ + dcminfo = mw.dcm_data + accession_number = dcminfo.get('AccessionNumber') + + # TODO: do not group echoes by default + size = list(mw.image_shape) + [len(series_files)] + if len(size) < 4: + size.append(1) + + # parse DICOM for seqinfo fields + TR = get_typed_attr(dcminfo, "RepetitionTime", float, -1000) / 1000 + TE = get_typed_attr(dcminfo, "EchoTime", float, -1) + refphys = get_typed_attr(dcminfo, "ReferringPhysicianName", str, "") + image_type = get_typed_attr(dcminfo, "ImageType", tuple, ()) + is_moco = 'MOCO' in image_type + series_desc = get_typed_attr(dcminfo, "SeriesDescription", str, "") + + if dcminfo.get([0x18, 0x24]): + # GE and Philips + sequence_name = dcminfo[0x18, 0x24].value + elif dcminfo.get([0x19, 0x109c]): + # Siemens + sequence_name = dcminfo[0x19, 0x109c].value + else: + sequence_name = "" + + # initialized in `group_dicoms_to_seqinfos` + global total_files + total_files += len(series_files) + + seqinfo = SeqInfo( + total_files_till_now=total_files, + example_dcm_file=op.basename(series_files[0]), + series_id=series_id, + dcm_dir_name=op.basename(op.dirname(series_files[0])), + series_files=len(series_files), + unspecified="", + dim1=size[0], + dim2=size[1], + dim3=size[2], + dim4=size[3], + TR=TR, + TE=TE, + protocol_name=dcminfo.ProtocolName, + is_motion_corrected=is_moco, + is_derived='derived' in [x.lower() for x in image_type], + patient_id=dcminfo.get('PatientID'), + study_description=dcminfo.get('StudyDescription'), + referring_physician_name=refphys, + series_description=series_desc, + sequence_name=sequence_name, + image_type=image_type, + accession_number=accession_number, + # For demographics to populate BIDS participants.tsv + patient_age=dcminfo.get('PatientAge'), + patient_sex=dcminfo.get('PatientSex'), + date=dcminfo.get('AcquisitionDate'), + series_uid=dcminfo.get('SeriesInstanceUID') + ) + return seqinfo + + +def validate_dicom(fl, dcmfilter): + """ + Parse DICOM attributes. Returns None if not valid. + """ + mw = dw.wrapper_from_file(fl, force=True, stop_before_pixels=True) + # clean series signature + for sig in ('iop', 'ICE_Dims', 'SequenceName'): + try: + del mw.series_signature[sig] + except KeyError: + pass + # Workaround for protocol name in private siemens csa header + if not getattr(mw.dcm_data, 'ProtocolName', '').strip(): + mw.dcm_data.ProtocolName = parse_private_csa_header( + mw.dcm_data, 'ProtocolName', 'tProtocolName' + ) if mw.is_csa else '' + try: + series_id = ( + int(mw.dcm_data.SeriesNumber), mw.dcm_data.ProtocolName + ) + except AttributeError as e: + lgr.warning( + 'Ignoring %s since not quite a "normal" DICOM: %s', fl, e + ) + return + if dcmfilter is not None and dcmfilter(mw.dcm_data): + lgr.warning("Ignoring %s because of DICOM filter", fl) + return + if mw.dcm_data[0x0008, 0x0016].repval in ( + 'Raw Data Storage', + 'GrayscaleSoftcopyPresentationStateStorage' + ): + return + try: + file_studyUID = mw.dcm_data.StudyInstanceUID + except AttributeError: + lgr.info("File {} is missing any StudyInstanceUID".format(fl)) + file_studyUID = None + return mw, series_id, file_studyUID + + +def group_dicoms_into_seqinfos(files, grouping, file_filter=None, + dcmfilter=None, flatten=False, + custom_grouping=None): """Process list of dicoms and return seqinfo and file group `seqinfo` contains per-sequence extract of fields from DICOMs which will be later provided into heuristics to decide on filenames + Parameters ---------- files : list of str List of files to consider + grouping : {'studyUID', 'accession_number', 'all', 'custom'} + How to group DICOMs for conversion. If 'custom', see `custom_grouping` + parameter. file_filter : callable, optional Applied to each item of filenames. Should return True if file needs to be kept, False otherwise. dcmfilter : callable, optional If called on dcm_data and returns True, it is used to set series_id - grouping : {'studyUID', 'accession_number', None}, optional - what to group by: studyUID or accession_number + flatten : bool, optional + Creates a flattened `seqinfo` with corresponding DICOM files. True when + invoked with `dicom_dir_template`. + custom_grouping: str or callable, optional + grouping key defined within heuristic. Can be a string of a + DICOM attribute, or a method that handles more complex groupings. + + Returns ------- seqinfo : list of list @@ -33,103 +170,74 @@ def group_dicoms_into_seqinfos(files, file_filter, dcmfilter, grouping): filegrp : dict `filegrp` is a dictionary with files groupped per each sequence """ - allowed_groupings = ['studyUID', 'accession_number', None] + allowed_groupings = ['studyUID', 'accession_number', 'all', 'custom'] if grouping not in allowed_groupings: raise ValueError('I do not know how to group by {0}'.format(grouping)) per_studyUID = grouping == 'studyUID' - per_accession_number = grouping == 'accession_number' + # per_accession_number = grouping == 'accession_number' lgr.info("Analyzing %d dicoms", len(files)) groups = [[], []] mwgroup = [] - studyUID = None - # for sanity check that all DICOMs came from the same - # "study". If not -- what is the use-case? (interrupted acquisition?) - # and how would then we deal with series numbers - # which would differ already + if file_filter: nfl_before = len(files) files = list(filter(file_filter, files)) nfl_after = len(files) lgr.info('Filtering out {0} dicoms based on their filename'.format( nfl_before-nfl_after)) - for fidx, filename in enumerate(files): - import nibabel.nicom.dicomwrappers as dw - # TODO after getting a regression test check if the same behavior - # with stop_before_pixels=True - mw = dw.wrapper_from_data( - dcm.read_file(filename, stop_before_pixels=True, force=True) - ) - - for sig in ('iop', 'ICE_Dims', 'SequenceName'): - try: - del mw.series_signature[sig] - except: - pass - - try: - file_studyUID = mw.dcm_data.StudyInstanceUID - except AttributeError: - lgr.info("File {} is missing any StudyInstanceUID".format(filename)) - file_studyUID = None - # Workaround for protocol name in private siemens csa header - try: - mw.dcm_data.ProtocolName - except AttributeError: - if not getattr(mw.dcm_data, 'ProtocolName', '').strip(): - mw.dcm_data.ProtocolName = parse_private_csa_header( - mw.dcm_data, 'ProtocolName', 'tProtocolName' - ) if mw.is_csa else '' - - try: - series_id = (int(mw.dcm_data.SeriesNumber), - mw.dcm_data.ProtocolName) - file_studyUID = mw.dcm_data.StudyInstanceUID + if grouping == 'custom': + if custom_grouping is None: + raise RuntimeError("Custom grouping is not defined in heuristic") + if callable(custom_grouping): + return custom_grouping(files, dcmfilter, SeqInfo) + grouping = custom_grouping + study_customgroup = None + + removeidx = [] + for idx, filename in enumerate(files): + mwinfo = validate_dicom(filename, dcmfilter) + if mwinfo is None: + removeidx.append(idx) + continue + mw, series_id, file_studyUID = mwinfo + if per_studyUID: + series_id = series_id + (file_studyUID,) - if not per_studyUID: - # verify that we are working with a single study + if flatten: + if per_studyUID: if studyUID is None: studyUID = file_studyUID - elif not per_accession_number: - assert studyUID == file_studyUID, ( - "Conflicting study identifiers found [{}, {}].".format( - studyUID, file_studyUID - )) - except AttributeError as exc: - lgr.warning('Ignoring %s since not quite a "normal" DICOM: %s', - filename, exc) - series_id = (-1, 'none') - file_studyUID = None - - if not series_id[0] < 0: - if dcmfilter is not None and dcmfilter(mw.dcm_data): - series_id = (-1, mw.dcm_data.ProtocolName) - - # filter out unwanted non-image-data DICOMs by assigning - # a series number < 0 (see test below) - if not series_id[0] < 0 and mw.dcm_data[0x0008, 0x0016].repval in ( - 'Raw Data Storage', - 'GrayscaleSoftcopyPresentationStateStorage'): - series_id = (-1, mw.dcm_data.ProtocolName) - - if per_studyUID: - series_id = series_id + (file_studyUID,) + assert studyUID == file_studyUID, ( + "Conflicting study identifiers found [{}, {}]." + .format(studyUID, file_studyUID) + ) + elif custom_grouping: + file_customgroup = mw.dcm_data.get(grouping) + if study_customgroup is None: + study_customgroup = file_customgroup + assert study_customgroup == file_customgroup, ( + "Conflicting {0} found: [{1}, {2}]" + .format(grouping, study_customgroup, file_customgroup) + ) ingrp = False + # check if same series was already converted for idx in range(len(mwgroup)): - # same = mw.is_same_series(mwgroup[idx]) if mw.is_same_series(mwgroup[idx]): - # the same series should have the same study uuid - assert (mwgroup[idx].dcm_data.get('StudyInstanceUID', None) - == file_studyUID) + if grouping != 'all': + assert ( + mwgroup[idx].dcm_data.get('StudyInstanceUID') == file_studyUID + ), "Same series found for multiple different studies" ingrp = True - if series_id[0] >= 0: - series_id = (mwgroup[idx].dcm_data.SeriesNumber, - mwgroup[idx].dcm_data.ProtocolName) - if per_studyUID: - series_id = series_id + (file_studyUID,) + series_id = ( + mwgroup[idx].dcm_data.SeriesNumber, + mwgroup[idx].dcm_data.ProtocolName + ) + if per_studyUID: + series_id = series_id + (file_studyUID,) groups[0].append(series_id) groups[1].append(idx) @@ -140,135 +248,64 @@ def group_dicoms_into_seqinfos(files, file_filter, dcmfilter, grouping): group_map = dict(zip(groups[0], groups[1])) - total = 0 - seqinfo = OrderedDict() + if removeidx: + # remove non DICOMS from files + for idx in sorted(removeidx, reverse=True): + del files[idx] + seqinfos = OrderedDict() # for the next line to make any sense the series_id needs to # be sortable in a way that preserves the series order for series_id, mwidx in sorted(group_map.items()): - if series_id[0] < 0: - # skip our fake series with unwanted files - continue mw = mwgroup[mwidx] - if mw.image_shape is None: - # this whole thing has now image data (maybe just PSg DICOMs) - # nothing to see here, just move on - continue - dcminfo = mw.dcm_data - series_files = [files[i] for i, s in enumerate(groups[0]) - if s == series_id] - # turn the series_id into a human-readable string -- string is needed - # for JSON storage later on + series_files = [files[i] for i, s in enumerate(groups[0]) if s == series_id] if per_studyUID: studyUID = series_id[2] series_id = series_id[:2] - accession_number = dcminfo.get('AccessionNumber') - series_id = '-'.join(map(str, series_id)) + if mw.image_shape is None: + # this whole thing has no image data (maybe just PSg DICOMs) + # nothing to see here, just move on + continue + seqinfo = create_seqinfo(mw, series_files, series_id) - size = list(mw.image_shape) + [len(series_files)] - total += size[-1] - if len(size) < 4: - size.append(1) - - # MG - refactor into util function - try: - TR = float(dcminfo.RepetitionTime) / 1000. - except (AttributeError, ValueError): - TR = -1 - try: - TE = float(dcminfo.EchoTime) - except (AttributeError, ValueError): - TE = -1 - try: - refphys = str(dcminfo.ReferringPhysicianName) - except AttributeError: - refphys = '' - try: - image_type = tuple(dcminfo.ImageType) - except AttributeError: - image_type = '' - try: - series_desc = dcminfo.SeriesDescription - except AttributeError: - series_desc = '' - - motion_corrected = 'MOCO' in image_type - - if dcminfo.get([0x18,0x24], None): - # GE and Philips scanners - sequence_name = dcminfo[0x18,0x24].value - elif dcminfo.get([0x19, 0x109c], None): - # Siemens scanners - sequence_name = dcminfo[0x19, 0x109c].value - else: - sequence_name = 'Not found' - - info = SeqInfo( - total, - op.split(series_files[0])[1], - series_id, - op.basename(op.dirname(series_files[0])), - '-', '-', - size[0], size[1], size[2], size[3], - TR, TE, - dcminfo.ProtocolName, - motion_corrected, - 'derived' in [x.lower() for x in dcminfo.get('ImageType', [])], - dcminfo.get('PatientID'), - dcminfo.get('StudyDescription'), - refphys, - series_desc, # We try to set this further up. - sequence_name, - image_type, - accession_number, - # For demographics to populate BIDS participants.tsv - dcminfo.get('PatientAge'), - dcminfo.get('PatientSex'), - dcminfo.get('AcquisitionDate'), - dcminfo.get('SeriesInstanceUID') - ) - # candidates - # dcminfo.AccessionNumber - # len(dcminfo.ReferencedImageSequence) - # len(dcminfo.SourceImageSequence) - # FOR demographics if per_studyUID: - key = studyUID.split('.')[-1] - elif per_accession_number: - key = accession_number + key = studyUID + elif grouping == 'accession_number': + key = mw.dcm_data.get("AccessionNumber") + elif grouping == 'all': + key = 'all' + elif custom_grouping: + key = mw.dcm_data.get(custom_grouping) else: key = '' lgr.debug("%30s %30s %27s %27s %5s nref=%-2d nsrc=%-2d %s" % ( key, - info.series_id, - series_desc, - dcminfo.ProtocolName, - info.is_derived, - len(dcminfo.get('ReferencedImageSequence', '')), - len(dcminfo.get('SourceImageSequence', '')), - info.image_type + seqinfo.series_id, + seqinfo.series_description, + mw.dcm_data.ProtocolName, + seqinfo.is_derived, + len(mw.dcm_data.get('ReferencedImageSequence', '')), + len(mw.dcm_data.get('SourceImageSequence', '')), + seqinfo.image_type )) - if per_studyUID: - if studyUID not in seqinfo: - seqinfo[studyUID] = OrderedDict() - seqinfo[studyUID][info] = series_files - elif per_accession_number: - if accession_number not in seqinfo: - seqinfo[accession_number] = OrderedDict() - seqinfo[accession_number][info] = series_files + + if not flatten: + if key not in seqinfos: + seqinfos[key] = OrderedDict() + seqinfos[key][seqinfo] = series_files else: - seqinfo[info] = series_files + seqinfos[seqinfo] = series_files if per_studyUID: lgr.info("Generated sequence info for %d studies with %d entries total", - len(seqinfo), sum(map(len, seqinfo.values()))) - elif per_accession_number: + len(seqinfos), sum(map(len, seqinfos.values()))) + elif grouping == 'accession_number': lgr.info("Generated sequence info for %d accession numbers with %d " - "entries total", len(seqinfo), sum(map(len, seqinfo.values()))) + "entries total", len(seqinfos), sum(map(len, seqinfos.values()))) else: - lgr.info("Generated sequence info with %d entries", len(seqinfo)) - return seqinfo + lgr.info("Generated sequence info with %d entries", len(seqinfos)) + return seqinfos def get_dicom_series_time(dicom_list): @@ -355,14 +392,10 @@ def _assign_dicom_time(ti): return outtar -def embed_nifti(dcmfiles, niftifile, infofile, bids_info, min_meta): - """ - - If `niftifile` doesn't exist, it gets created out of the `dcmfiles` stack, - and json representation of its meta_ext is returned (bug since should return - both niftifile and infofile?) +def embed_dicom_and_nifti_metadata(dcmfiles, niftifile, infofile, bids_info): + """Embed metadata from nifti (affine etc) and dicoms into infofile (json) - if `niftifile` exists, its affine's orientation information is used while + `niftifile` should exist. Its affine's orientation information is used while establishing new `NiftiImage` out of dicom stack and together with `bids_info` (if provided) is dumped into json `infofile` @@ -371,69 +404,52 @@ def embed_nifti(dcmfiles, niftifile, infofile, bids_info, min_meta): dcmfiles niftifile infofile - bids_info - min_meta - - Returns - ------- - niftifile, infofile + bids_info: dict + Additional metadata to be embedded. `infofile` is overwritten if exists, + so here you could pass some metadata which would overload (at the first + level of the dict structure, no recursive fancy updates) what is obtained + from nifti and dicoms """ # imports for nipype import nibabel as nb - import os import os.path as op import json import re + from heudiconv.utils import save_json + + from heudiconv.external.dcmstack import ds + stack = ds.parse_and_stack(dcmfiles, force=True).values() + if len(stack) > 1: + raise ValueError('Found multiple series') + # may be odict now - iter to be safe + stack = next(iter(stack)) + + if not op.exists(niftifile): + raise NotImplementedError( + "%s does not exist. " + "We are not producing new nifti files here any longer. " + "Use dcm2niix directly or .convert.nipype_convert helper ." + % niftifile + ) - if not min_meta: - from heudiconv.external.dcmstack import ds - stack = ds.parse_and_stack(dcmfiles, force=True).values() - if len(stack) > 1: - raise ValueError('Found multiple series') - # may be odict now - iter to be safe - stack = next(iter(stack)) - - #Create the nifti image using the data array - if not op.exists(niftifile): - nifti_image = stack.to_nifti(embed_meta=True) - nifti_image.to_filename(niftifile) - return ds.NiftiWrapper(nifti_image).meta_ext.to_json() - - orig_nii = nb.load(niftifile) - aff = orig_nii.affine - ornt = nb.orientations.io_orientation(aff) - axcodes = nb.orientations.ornt2axcodes(ornt) - new_nii = stack.to_nifti(voxel_order=''.join(axcodes), embed_meta=True) - meta = ds.NiftiWrapper(new_nii).meta_ext.to_json() - - meta_info = None if min_meta else json.loads(meta) + orig_nii = nb.load(niftifile) + aff = orig_nii.affine + ornt = nb.orientations.io_orientation(aff) + axcodes = nb.orientations.ornt2axcodes(ornt) + new_nii = stack.to_nifti(voxel_order=''.join(axcodes), embed_meta=True) + meta_info = ds.NiftiWrapper(new_nii).meta_ext.to_json() + meta_info = json.loads(meta_info) if bids_info: + meta_info.update(bids_info) - if min_meta: - meta_info = bids_info - else: - # make nice with python 3 - same behavior? - meta_info = meta_info.copy() - meta_info.update(bids_info) - # meta_info = dict(meta_info.items() + bids_info.items()) - try: - meta_info['TaskName'] = (re.search('(?<=_task-)\w+', - op.basename(infofile)) - .group(0).split('_')[0]) - except AttributeError: - pass # write to outfile - with open(infofile, 'wt') as fp: - json.dump(meta_info, fp, indent=3, sort_keys=True) - - return niftifile, infofile + save_json(infofile, meta_info) def embed_metadata_from_dicoms(bids_options, item_dicoms, outname, outname_bids, - prov_file, scaninfo, tempdirs, with_prov, - min_meta): + prov_file, scaninfo, tempdirs, with_prov): """ Enhance sidecar information file with more information from DICOMs @@ -447,7 +463,6 @@ def embed_metadata_from_dicoms(bids_options, item_dicoms, outname, outname_bids, scaninfo tempdirs with_prov - min_meta Returns ------- @@ -460,14 +475,13 @@ def embed_metadata_from_dicoms(bids_options, item_dicoms, outname, outname_bids, item_dicoms = list(map(op.abspath, item_dicoms)) embedfunc = Node(Function(input_names=['dcmfiles', 'niftifile', 'infofile', - 'bids_info', 'min_meta'], + 'bids_info',], output_names=['outfile', 'meta'], - function=embed_nifti), + function=embed_dicom_and_nifti_metadata), name='embedder') embedfunc.inputs.dcmfiles = item_dicoms embedfunc.inputs.niftifile = op.abspath(outname) embedfunc.inputs.infofile = op.abspath(scaninfo) - embedfunc.inputs.min_meta = min_meta embedfunc.inputs.bids_info = load_json(op.abspath(outname_bids)) if (bids_options is not None) else None embedfunc.base_dir = tmpdir cwd = os.getcwd() @@ -522,5 +536,5 @@ def parse_private_csa_header(dcm_data, public_attr, private_attr, default=None): val = parsedhdr[private_attr].replace(' ', '') except Exception as e: lgr.debug("Failed to parse CSA header: %s", str(e)) - val = default if default else '' + val = default or "" return val diff --git a/heudiconv/external/dlad.py b/heudiconv/external/dlad.py index 53317338..68a0e758 100644 --- a/heudiconv/external/dlad.py +++ b/heudiconv/external/dlad.py @@ -10,7 +10,7 @@ lgr = logging.getLogger(__name__) -MIN_VERSION = '0.12.2' +MIN_VERSION = '0.12.4' def prepare_datalad(studydir, outdir, sid, session, seqinfo, dicoms, bids): @@ -177,4 +177,4 @@ def mark_sensitive(ds, path_glob): init=dict([('distribution-restrictions', 'sensitive')]), recursive=True) if inspect.isgenerator(res): - res = list(res) \ No newline at end of file + res = list(res) diff --git a/heudiconv/external/tests/test_dlad.py b/heudiconv/external/tests/test_dlad.py index e1f91d2a..c342b2da 100644 --- a/heudiconv/external/tests/test_dlad.py +++ b/heudiconv/external/tests/test_dlad.py @@ -1,10 +1,13 @@ from ..dlad import mark_sensitive -from datalad.api import Dataset from ...utils import create_tree +import pytest + +dl = pytest.importorskip('datalad.api') + def test_mark_sensitive(tmpdir): - ds = Dataset(str(tmpdir)).create(force=True) + ds = dl.Dataset(str(tmpdir)).create(force=True) create_tree( str(tmpdir), { diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index d353e3e6..c5a76914 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -126,6 +126,10 @@ import logging lgr = logging.getLogger('heudiconv') +# pythons before 3.7 didn't have re.Pattern, it was some protected +# _sre.SRE_Pattern, so let's just sample a class of the compiled regex +re_Pattern = re.compile('.').__class__ + # Terminology to harmonise and use to name variables etc # experiment # subject @@ -372,14 +376,14 @@ def get_study_hash(seqinfo): return md5sum(get_study_description(seqinfo)) -def fix_canceled_runs(seqinfo, accession2run=fix_accession2run): +def fix_canceled_runs(seqinfo): """Function that adds cancelme_ to known bad runs which were forgotten """ accession_number = get_unique(seqinfo, 'accession_number') - if accession_number in accession2run: + if accession_number in fix_accession2run: lgr.info("Considering some runs possibly marked to be " "canceled for accession %s", accession_number) - badruns = accession2run[accession_number] + badruns = fix_accession2run[accession_number] badruns_pattern = '|'.join(badruns) for i, s in enumerate(seqinfo): if re.match(badruns_pattern, s.series_id): @@ -391,39 +395,65 @@ def fix_canceled_runs(seqinfo, accession2run=fix_accession2run): return seqinfo -def fix_dbic_protocol(seqinfo, keys=series_spec_fields, subsdict=protocols2fix): - """Ad-hoc fixup for existing protocols +def fix_dbic_protocol(seqinfo): + """Ad-hoc fixup for existing protocols. + + It will operate in 3 stages on `protocols2fix` records. + 1. consider a record which has md5sum of study_description + 2. apply all substitutions, where key is a regular expression which + successfully searches (not necessarily matches, so anchor appropriately) + study_description + 3. apply "catch all" substitutions in the key containing an empty string + + 3. is somewhat redundant since `re.compile('.*')` could match any, but is + kept for simplicity of its specification. """ + study_hash = get_study_hash(seqinfo) + study_description = get_study_description(seqinfo) - if study_hash not in subsdict: - raise ValueError("I don't know how to fix {0}".format(study_hash)) + # We will consider first study specific (based on hash) + if study_hash in protocols2fix: + _apply_substitutions(seqinfo, + protocols2fix[study_hash], + 'study (%s) specific' % study_hash) + # Then go through all regexps returning regex "search" result + # on study_description + for sub, substitutions in protocols2fix.items(): + if isinstance(sub, re_Pattern) and sub.search(study_description): + _apply_substitutions(seqinfo, + substitutions, + '%r regex matching' % sub.pattern) + # and at the end - global + if '' in protocols2fix: + _apply_substitutions(seqinfo, protocols2fix[''], 'global') - # need to replace both protocol_name series_description - substitutions = subsdict[study_hash] + return seqinfo + + +def _apply_substitutions(seqinfo, substitutions, subs_scope): + lgr.info("Considering %s substitutions", subs_scope) for i, s in enumerate(seqinfo): fixed_kwargs = dict() - for key in keys: - value = getattr(s, key) + # need to replace both protocol_name series_description + for key in series_spec_fields: + oldvalue = value = getattr(s, key) # replace all I need to replace for substring, replacement in substitutions: value = re.sub(substring, replacement, value) + if oldvalue != value: + lgr.info(" %s: %r -> %r", key, oldvalue, value) fixed_kwargs[key] = value # namedtuples are immutable seqinfo[i] = s._replace(**fixed_kwargs) - return seqinfo - def fix_seqinfo(seqinfo): """Just a helper on top of both fixers """ # add cancelme to known bad runs seqinfo = fix_canceled_runs(seqinfo) - study_hash = get_study_hash(seqinfo) - if study_hash in protocols2fix: - lgr.info("Fixing up protocol for {0}".format(study_hash)) - seqinfo = fix_dbic_protocol(seqinfo) + seqinfo = fix_dbic_protocol(seqinfo) return seqinfo @@ -484,10 +514,10 @@ def infotodict(seqinfo): # 3 - Image IOD specific specialization (optional) dcm_image_iod_spec = s.image_type[2] image_type_seqtype = { - 'P': 'fmap', # phase + # Note: P and M are too generic to make a decision here, could be + # for different seqtypes (bold, fmap, etc) 'FMRI': 'func', 'MPR': 'anat', - # 'M': 'func', "magnitude" -- can be for scout, anat, bold, fmap 'DIFFUSION': 'dwi', 'MIP_SAG': 'anat', # angiography 'MIP_COR': 'anat', # angiography @@ -540,29 +570,55 @@ def infotodict(seqinfo): # prefix = '' prefix = '' + # + # Figure out the seqtype_label (BIDS _suffix) + # + # If none was provided -- let's deduce it from the information we find: # analyze s.protocol_name (series_id is based on it) for full name mapping etc - if seqtype == 'func' and not seqtype_label: - if '_pace_' in series_spec: - seqtype_label = 'pace' # or should it be part of seq- - else: - # assume bold by default - seqtype_label = 'bold' - - if seqtype == 'fmap' and not seqtype_label: - if not dcm_image_iod_spec: - raise ValueError("Do not know image data type yet to make decision") - seqtype_label = { - # might want explicit {file_index} ? - # _epi for pepolar fieldmaps, see - # https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/01-magnetic-resonance-imaging-data.html#case-4-multiple-phase-encoded-directions-pepolar - 'M': 'epi' if 'dir' in series_info else 'magnitude', - 'P': 'phasediff', - 'DIFFUSION': 'epi', # according to KODI those DWI are the EPIs we need - }[dcm_image_iod_spec] - - # label for dwi as well - if seqtype == 'dwi' and not seqtype_label: - seqtype_label = 'dwi' + if not seqtype_label: + if seqtype == 'func': + if '_pace_' in series_spec: + seqtype_label = 'pace' # or should it be part of seq- + elif 'P' in s.image_type: + seqtype_label = 'phase' + elif 'M' in s.image_type: + seqtype_label = 'bold' + else: + # assume bold by default + seqtype_label = 'bold' + elif seqtype == 'fmap': + # TODO: support phase1 phase2 like in "Case 2: Two phase images ..." + if not dcm_image_iod_spec: + raise ValueError("Do not know image data type yet to make decision") + seqtype_label = { + # might want explicit {file_index} ? + # _epi for pepolar fieldmaps, see + # https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/01-magnetic-resonance-imaging-data.html#case-4-multiple-phase-encoded-directions-pepolar + 'M': 'epi' if 'dir' in series_info else 'magnitude', + 'P': 'phasediff', + 'DIFFUSION': 'epi', # according to KODI those DWI are the EPIs we need + }[dcm_image_iod_spec] + elif seqtype == 'dwi': + # label for dwi as well + seqtype_label = 'dwi' + + # + # Even if seqtype_label was provided, for some data we might need to override, + # since they are complementary files produced along-side with original + # ones. + # + if s.series_description.endswith('_SBRef'): + seqtype_label = 'sbref' + + if not seqtype_label: + # Might be provided by the bids ending within series_spec, we would + # just want to check if that the last element is not _key-value pair + bids_ending = series_info.get('bids', None) + if not bids_ending \ + or "-" in bids_ending.split('_')[-1]: + lgr.warning( + "We ended up with an empty label/suffix for %r", + series_spec) run = series_info.get('run') if run is not None: diff --git a/heudiconv/heuristics/reproin_validator.cfg b/heudiconv/heuristics/reproin_validator.cfg index 94f29d79..4e7bcf39 100644 --- a/heudiconv/heuristics/reproin_validator.cfg +++ b/heudiconv/heuristics/reproin_validator.cfg @@ -1,6 +1,7 @@ { "ignore": [ - "TOTAL_READOUT_TIME_NOT_DEFINED" + "TOTAL_READOUT_TIME_NOT_DEFINED", + "CUSTOM_COLUMN_WITHOUT_DESCRIPTION" ], "warn": [], "error": [], diff --git a/heudiconv/heuristics/test_reproin.py b/heudiconv/heuristics/test_reproin.py index a995e390..d8777f21 100644 --- a/heudiconv/heuristics/test_reproin.py +++ b/heudiconv/heuristics/test_reproin.py @@ -2,6 +2,10 @@ # Tests for reproin.py # from collections import OrderedDict +from mock import patch +import re + +from . import reproin from .reproin import ( filter_files, fix_canceled_runs, @@ -78,7 +82,8 @@ def test_fix_canceled_runs(): 'accession1': ['^01-', '^03-'] } - seqinfo_ = fix_canceled_runs(seqinfo, fake_accession2run) + with patch.object(reproin, 'fix_accession2run', fake_accession2run): + seqinfo_ = fix_canceled_runs(seqinfo) for i, s in enumerate(seqinfo_, 1): output = runname @@ -106,16 +111,20 @@ def test_fix_dbic_protocol(): 'nochangeplease', 'nochangeeither') - seqinfos = [seq1, seq2] - keys = ['field1'] - subsdict = { + protocols2fix = { md5sum('mystudy'): - [('scout_run\+', 'scout'), + [('scout_run\+', 'THESCOUT-runX'), ('run-life[0-9]', 'run+_task-life')], + re.compile('^my.*'): + [('THESCOUT-runX', 'THESCOUT')], + # rely on 'catch-all' to fix up above scout + '': [('THESCOUT', 'scout')] } - seqinfos_ = fix_dbic_protocol(seqinfos, keys=keys, subsdict=subsdict) + with patch.object(reproin, 'protocols2fix', protocols2fix), \ + patch.object(reproin, 'series_spec_fields', ['field1']): + seqinfos_ = fix_dbic_protocol(seqinfos) assert(seqinfos[1] == seqinfos_[1]) # field2 shouldn't have changed since I didn't pass it assert(seqinfos_[0] == FakeSeqInfo(accession_number, @@ -124,8 +133,9 @@ def test_fix_dbic_protocol(): seq1.field2)) # change also field2 please - keys = ['field1', 'field2'] - seqinfos_ = fix_dbic_protocol(seqinfos, keys=keys, subsdict=subsdict) + with patch.object(reproin, 'protocols2fix', protocols2fix), \ + patch.object(reproin, 'series_spec_fields', ['field1', 'field2']): + seqinfos_ = fix_dbic_protocol(seqinfos) assert(seqinfos[1] == seqinfos_[1]) # now everything should have changed assert(seqinfos_[0] == FakeSeqInfo(accession_number, diff --git a/heudiconv/info.py b/heudiconv/info.py index a81231f0..fe6a1ab5 100644 --- a/heudiconv/info.py +++ b/heudiconv/info.py @@ -1,4 +1,4 @@ -__version__ = "0.6.0" +__version__ = "0.8.0" __author__ = "HeuDiConv team and contributors" __url__ = "https://github.com/nipy/heudiconv" __packagename__ = 'heudiconv' @@ -23,9 +23,7 @@ REQUIRES = [ 'nibabel', 'pydicom', - 'nipype >=1.0.0; python_version > "3.0"', - 'nipype >=1.0.0,!=1.2.1,!=1.2.2; python_version == "2.7"', - 'pathlib', + 'nipype >=1.0.0', 'dcmstack>=0.8', 'etelemetry', 'filelock>=3.0.12', @@ -42,7 +40,7 @@ EXTRA_REQUIRES = { 'tests': TESTS_REQUIRES, 'extras': [], # Requires patched version ATM ['dcmstack'], - 'datalad': ['datalad >=0.12.2'] + 'datalad': ['datalad >=0.12.3'] } # Flatten the lists diff --git a/heudiconv/parser.py b/heudiconv/parser.py index 8ceba641..0c590319 100644 --- a/heudiconv/parser.py +++ b/heudiconv/parser.py @@ -161,14 +161,16 @@ def get_study_sessions(dicom_dir_template, files_opt, heuristic, outdir, files_ += files_ex # sort all DICOMS using heuristic - # TODO: this one is not grouping by StudyUID but may be we should! - seqinfo_dict = group_dicoms_into_seqinfos(files_, + seqinfo_dict = group_dicoms_into_seqinfos( + files_, + grouping, file_filter=getattr(heuristic, 'filter_files', None), dcmfilter=getattr(heuristic, 'filter_dicom', None), - grouping=grouping) + custom_grouping=getattr(heuristic, 'grouping', None) + ) if sids: - if not (len(sids) == 1 and len(seqinfo_dict) == 1): + if len(sids) != 1: raise RuntimeError( "We were provided some subjects (%s) but " "we can deal only " @@ -208,17 +210,21 @@ def infotoids(seqinfos, outdir): # TODO: probably infotoids is doomed to do more and possibly # split into multiple sessions!!!! but then it should be provided # full seqinfo with files which it would place into multiple groups - lgr.info("Study session for %s" % str(ids)) study_session_info = StudySessionInfo( ids.get('locator'), ids.get('session', session) or session, sid or ids.get('subject', None) ) + lgr.info("Study session for %r", study_session_info) + if study_session_info in study_sessions: - #raise ValueError( - lgr.warning( - "We already have a study session with the same value %s" - % repr(study_session_info)) - continue # skip for now + if grouping != 'all': + # MG - should this blow up to mimic -d invocation? + lgr.warning( + "Existing study session with the same values (%r)." + " Skipping DICOMS %s", + study_session_info, *seqinfo.values() + ) + continue study_sessions[study_session_info] = seqinfo return study_sessions diff --git a/heudiconv/tests/data/phantom.dcm b/heudiconv/tests/data/phantom.dcm new file mode 100644 index 0000000000000000000000000000000000000000..b598c5a5b8217dd6b88de2f04e84b9f2a80e00ea GIT binary patch literal 162304 zcmeFaYmj5fbtZTe*AZU-*jPKl`$rKD+u=tz?67~? zf4u(VAGX7>^7rMr=Vsm)3MimCltokl_ukBtC(k)~PUgwS&CFyz+|S1L`fe5^+}(LT z^Y?JgX4W$Mw|T{9EsN{j?aybtd@)}y7xG@AP^jlCc`ska7r%&q4iIHPl=5B?V4hc~ z0iu{Mm-5xB$FPD|E!2x;&nwj{wR)-K;i+CORkE2INc-sscIM>)@5_jLIgS@GpUSlE zT+8ep9^c#ayxZ3@AIYq@u4lGe`^VddIgofQa}6P%&+H#=zqP&B+Y_aMbA zkr;~>W`3srW(NAV}+fN7WQ72b=?M{0z zeAFGiws+^XyRFv>rCOx|sK1ms*dyi}H&}wr`v*otu4M{0cJ{xRdDv}texcoY)E=C4 zJGu77MZb5_p7e%;ajxHcFlvu_-SM@|n}B^O(`?=W^l|G1`CrR?6JemIdqIl21B4vh zIM{n3^KGPg5BXl0r=#v@^psQp`i0EFs5d@u-+&Gt>|M(|ZI6cIe*4i4lnQZm9`sIb zO}gWWcOz2>$^Ay=n?^4CIi$VOY(1YT6ub?jyRiW+^zzW{dZARUA{|0r%2ez5LaAP_ zmc2^lmCTPL?em#Co6U_^kQ3$m*5NCeFPOX5!7G`YCjMaSAooh@^VnD+qr+1FJ-?f5F_tMr*HRehv58 zj0cE+5qF@tSjrbGFi3>_I`To>omVn{5%wfd|B`_(A@(;A_B?E)j{AQli2ohI4Sjkg z^SjRd@0t5bnc*2DaZ}iM%9#=zKOLM6bGwH*81XAc1Hz^HO6KjCTX&w%+#YtGzLL4| zsk^OfsFHpkf0@iHnU_CBIUerc2fj~#iX4%uA4h05L#ccQVJLxop1|UITdmyZpaVPmF9-S!npi-ahlsg$V;yCP>f^1=+}_>YTp3|6f_}&$U%>sx zP{uD%hUE&Lehhm3LS}C`I&b$sm-!@~K+W)TnU`<0c3uMA^7&u>pFjDN&t*Q95ccUC z+rUKkT2Le)}Jpw7=4p>o>pm&Cg{%Yw|AStMyvtbD39=_W4W& z)j%%uanwrnc-?91BVxf!8*N|hsYrq zNYrRjE|>YFuPHgyxuR1!)V6w8gqD0hbIW6`%j`i;sZo9RmRByk>y-*fUjxKTkcmgB z;O~}~%N$-->!J~{txl~{t~aVm8S9D+Y_AFF-H7G}~0{)*9N+Y$oz4YvRP-K|Wfb~<@Yc>z~`iStGOtM_#jIX?&vK7SnC$<0-tN&ot=KRq8<4oK7GA409+@5876041Ek8T?-yL^5>kr$5LASr#9>13&cBNUa`b1#f-`x?N;WPEX z!P%OOdV@0~4VINn8NO#q!KK%`N14n|;6Zir$8paub&}V25wK`efj2&WjHj~i*Q7nQ%4fUg=b*rt?v8NF8Z9dQBoRRiiK9O0HIv1!N|pLmsHoUgKmDw1Fv4GT=Z(SVjcJT zVVSs}t3EeR-k40R?$_s_$q#w zPCr+rH=5p8pU&2BiZ1?$E+*O#jzMu0%F3EyLVxtv*0X8!Kg+sit3Q0aIXD@1;C^iM zM%@$R9IK>xRk^!iE)SM|@<_Q7=RGJTpIu17#_&ea!jUn#-#6M4F-fQ}{<`-SPHdcQ|hMViHnn!&iAiort=I z_Q((E+UT90PRHoXyftcfdaS^L0)#vihm}mY7FaX&R#eW0zPCoh=>>DKIP^}nL1}Vv zY8gi9ly=w*EZl>F4e1wZ4eIKb;#NFzH`Hh#6(CH6~wAyFdW@&2CZ2mrH za4?)0`TM-NUV?L-<}UfGm5>(F#9#gqk32JM4myuUZJ|@(6^~%c`uOBY!%}&d0Z*TE zONC9XbgFD!UQ&1$oIdBuYnWCEWiCR!@GUx=Snh(;;rw|cD@iA~l|o`6l_F=BRSKQk zgM)s1&^;VZ3>7R>#l*BM7s1b~Xr?m0ir zi@@_yr2RNN_#60sBJ&ddt_Oeg?4meL8@Dr*IUl5c2_wdz#owoJ_X237#7M2kRCxXf zFy4UYiqUYyrr>nyrJuqK%NOzYDy}y(U&#Da=FcJJ4dh17iRm-=y^iM>@%utD&I`1_ zqv<)_?8ELEoQIwEgKj_b=YjDvNckM-{VXW@B&0`KUIWEi@@ntVwqt#GesRp<-BEAy z@H~_GN#wo;Je)BSYB~`i>RkQPp)A0nTMK{%6G_8xtZDCUAY)&UbjC!@fyV9qa+$(vDen{sI#_&GvN3bmx zdHNLkcomPOCW^^Er{es%;e&1X)ryO7{3d_2UJ@%y!nw73xZRrc&ZqtD!Rasw3-#Up z5tIChwG(G-j47!<;*X9}b68%AMV{oDam4Z8lF&*~>wvL0cZNOi521toh{k zBhXvH8MXKIaWApDV>+RWIG#g#`FkF^`+~vCuon;$w|B*vP~9N~Dd8_u^DBQJ^{>o} z{h9)r&lD(Jxh3vPpceCHgwzIu#l71>qQ}{m0=YZ|kGB2+E*4qEE zR?W7u_p=AtJ>WXYzL&koPO^`(Q(S)$&)-eN30%zwS2zMYPV4CkY`ruA*zF@)&&$gH?qUddy>7^s5} z>nQh=_PjSV{zZFagq48|hT!5MxOfM=y_IcdU(bHS{JjB=zKQP}2>Ckr`bPFma7p=V zUveJ4l%_O#7LDJgS3A&`)_6ctL&d-|L^R-m;K4?_tyRxUUqiFpepomjORSt>&W{6zU#903^WknZLge$udUMJ1;cb=% z+~2`C^VhQfB>VqnKl0o^hPM3b+J6SAd>TA`4EcO2;E65FN5C0dcJ(czV;zE%-tYgA z`5GkhDlohRT&!)xW6_p?7{aY4b<>p_$tN$n)BdCf&*EYmp34(yYgwZT9r{aL()J2g zfzw}Y6SXX6*(>wqXI?o9bUHjt^$?xmP+cU{3cZ?MrJ>JyrA!u~e->}E;xxZ``37I> z!(QK+XB7--d~ozp+i&S(#OL=oKFRN{f4#GLAV~2$FI}1-4gPSeb$?@{J!v03y-1!< zVaqMFiZ3hA)O`9i)HC&m()8(Era(GM3$JPPD9pU^Z^g(bed@)~QxacBY*alD`eM@V zf7tX(x4ZXCm|z+7E~fo9I}M;sd{FU`=&1<3^icO$$|*3=vwR<(=@cO&z zcy&K({@%>4!>@ZQ`z`$K!XHbL$>x*x`3`jVyghnvoOuk4kAeFHeTSbz+I9SG!l%1~ zr)_xb`^GbTKl_)m|1LbS-^dn>OrrgM;v}VqLrfPKuW%db`L_cMN9YXZ{||GWgP{fyN07)$sp2v#}22gT!;^#ZT2YX-S{-4W-G-nhAOh?S2w z-@KlCeI2zMM{&AiJOz(WbDU^+<7Rg-JzkiH5JdB)EfD$s{Y+Q z3}(GDN3i7c?7y7-gY4hL@6Tob$F+mCe~{rinxDj|+ecvapEjd+>IblPPO5{r*5v+h z)bHHma-m%;T<4(ZPXhYm!0>VC6pFW+p!;9n+js*5v0_G$BmuYz4TwgZ};kfREUxxD<_>1i3fBj(8ZJo3S2xpzlFX4FIKIa=Ey@}JJ zew}{0e*Rc*{Hs3W&7mg8dff75c}O|QRs1sj70;3KVwxm7*f8Qf8IF#(2b1mqt8t!U z?n>8!h$ID@=0aK95aNvL7uL3pHoB+q89IhVTC38jDOrMgFXbuq46!HCNl|?st1<4=0(AW&dRDe_#7A*Zyej z*Vhi$KE3weW_}NEqi{6v)0qnD(SHuL){AIYS#Mr5)LiDlRO@I0@d1k9qot++SAEu5 zZ|Kt)`ZUh0HP`;H%pYYspfdwsv5fb8UO`PwpOL)?&i}|br&u7Rc^UogDB!a6M$?;d zoAcfTYb(~VP6O{l$r}8RqJ^Pv&lxNF_0&(HSge`%V()guSUsDMBNKji{i|20@ddt8 z4ew%T{=8mQ=88kpn^sL%+B9jtlW7Ni6d4^5?G~z)X07Y=8t36-$b?_=z-!VNndXmZ z^SSD?zi=Rd1EDvKYNeh!q@(s)20uvpTiw2_PD|iGX%q>CpCn8?fDe;?C3~aWZ$CB5 zVw6t3Dy7k@*7b4uzXVWzsXx4y@TQReG0&m6%8a-K#hX{p4o~m5Cnpa(!!x@E6{MIt zlUMs;T`H}Kv{TYDeb{x=;pxHfad)&i8VyJAT=?3iS)oSHjPzQX^`2L0qv&h6c0pb# zw^n;}A5K-5@BgueP_CtZupSCap1H_)66GOz(z8@u4RE5f{nX@ZTY1q8Y1>7*%u7bp ztc*F*!PdayRQqLn$1nAVSN}CSt0)MI+Bn{DMkR|E61~}RNNPUw>XD9taN+p=;{o0t zv3y$IM0RX1WCTY>h&XP>a)^#}UPBna4yj!t1K&-$$>)G|7g65B<;5zv$fVOfWc!`4$~_B+Wv_V`K6;r$rszn%G8na{8N<+U|4nxyr`he5xQ|HO19 zK_g`?zCCByuVXCY^JYx5nfbe!Kfm?|Yah-2JVwI)ZuURU{{8IV%l;9@#N0acL!-an zgNMo$)<15%|K~FMnSY$|K<|s$W6=Azv;R@{Kg|C7n3erQjKTfe*?%MZ*Ry{OGrhl+ z{q+x>{?`5D_t0ZIK5f6p#tS3Fzm@%4knVo~s{c0b{zmrqLG549{#DTX+u6UE{S`A> zd6xYiMv;CVzw^i+?~JQ!5mZ5|s(+;ZY(F_3nqlV5A7uZ{?7xP;zmol3(EA&h$9{s* zRnq(O7zI7X3W6TSZoiM1zXG0q&8&d9O#f^78aD5!;cap8m0aJ*^YXPqp@QbG6!V+B zQohz`REzaubz!~(zOYjHcz_k~${@IS7qUOWs*ov4<2SQ^Ir}@{v6B7o*WO+GFEZ<> zrG6%JYtDM9S}v6;MXwT&x3YT4&=0%FK<>W)xu0Up@iFB6+bEac#43fK$o|8%H`e|> zdY>NywO@toUWWCEf5a7kNj?YV?#jMgr0lDWdc6{pJ+^mf)Jk5XQ7^~h;Omy_1w=I( z0ps)Pqv6XpmVX}O^Z#t^udaP|?RT-B@@1?9_>!r!J_Y(Yt3y6GdLdk~N6Ma3+?R^B z7thjOIF-FM9BG#V-q5GfDsZHNEwy@;+A~e+*~NXO=m^)NVegsD2uEm#HHtq(sztWi zAL3O$_9|gfKQBsqvsLF%c<`B5FEl|KJZV3g4lqoJ88*K=>W)fUifyRkfo(w1w+@DmS_aQ$UFON@4GN9`)Qa-xVGY;_ny2g*zMe;QN^X^FZK=5bz6@lrOu^y9AC+Xv>XG33y zEsE50T6bxGEXqp-`{SWQ;H)#6pFSlgjXzGJa~#ON;Yj~W`tUO5C9{*3a`rdLldKeC zt*o^l?`~)CZj5}0Q{O+v zn%iLVcx)(E~PH{O;~7wyE|WV z*EyuwB9fTTyzyo=dUG^tV-k6Q!SwFHj4a~4T1s5>UTj(WCoCI|cYOjg<1ZWfFVm}M zxmwK7k7apSbYUhbMed>>)Oqsp9H{*mWX=`mT&YLe{~YF~zl@PC%3pnam48%!WZyfg zkU?mE=9RCeNP~~PN&N-M=$N9q}Eqf}3x2d(_>`qyW%qTF42{1V2IZ=mcs z`%W6E7qm)hxU{gOnEE6A8|ntXQMzdt{E}{7pT+89tlv27_6E3M6#PWKOS*aWzHw)(^$qTNTK)#d`@d28Mgf1fyh@{9DO9Rur^2LO zxu2^(tEa|QNM~JL>ZIH#?YzpLaOUy{zoaGgyXsNYS6ik%RUXs$+a3>3;8k@;n4eS% zEp?ij(&vW|`jI>Sewg&bSAIC`4th_tAbj#jm!;Nnrqt`z(7z{7D*u2t>)r08tq?bJjNjgPGbgjLMf1${RM_%@D|X+^!Tn$$;mpVc zmTl&5-URAgj>t`n#@!wc`}v#O*m8yoxcO@F5R0J%Siu2%51TMyJvT_=;x|JB5xu^{ zNs&Wp=9{@+&fPq1_s59!fxd4tlV5Wymu_eR?!8x->kBuxNatAB1b772? zP@1jj!0hj2lT|}GK7fYUxEkVQmH#;RbU4jThPiQflAB)S+PUC0w}3Awpp;CxP&Y`; zN{a3{H+k62VdXfsgUk(2&C|&giwg%rs)i*Xe0RX*Zi3)vau24H$#6g-+8u1LZTQT! zCk7#Q2~a$JtYyr*DQ`I`;Z%1g-53oH=?z2rqv3lXJX#VK%wy4RcfdpM6l-yF?S9|T zLs~)7;~t(LbaN-Zu{jA@=CUkO`0BIB{P(Z81zpbQ3gp!GQ$quCr+%?Hc&I6@_M@)J z+DSiFImG3e>kS~h;h>9BALg)FbT|k! zz@yr!>X`UL#=DGKSgIi3Fv%IGFW15XbgP7(tAx~~C}fT+F2Gvut=2*A@x$&Qcj52; znCn3p7hOwLkwT-_mXoDqIrg#wI*Js+_Nh^E5OWhBh@_2|b9npOyV%2N_@Lcq`~;F2 z=T1k%bE8VwfN$CbZ-Z%2@cS6j>BeRJnbJdzMKYfb{rwTVzQDJZm5=;(og~$(;^IEdG(H z2e?62kPus4qw*QHJGq|O-HZ@tqxJ=MRD!t9sMli)OcS`>1vbho*n82&vy{v(T*=Wa zCJ#pABev-%fFi^eiy-4AZ0sM4Fq(%#iP4Dt&_TlhyQ2i?%LESVW|_L#?6fbqnS=sX zicH(u_}QQRSsP^1ZUoOIR*=C$aT`og-x;0{&)|*xlC3^+9gK#VijPxrx#{3>8}3#o zhu0yXkKFZN||E6-)+AaWLKu3ce|b5^xTF~x@gzXfZH%K z6ufw1gUOVmo{p|njATM_R?V=VIT8l8Ammj&V#7LgzEhTKvo^qcMT7CFs2heVG~cSJ z5n8Z704C^96+8UjyvZFuXdj6XMasrl0;>nq9dwLxo!N(G}7 zBf9?xuM??Dk;_$0&0;X|Ee2pRMq)Ff;=a|zf;UjC*|6>B=tG<=qnc1!zNY2xi4+)D z)tGREn(?H4ei760YLVLOH*x1fkZcK}1o7sR4WMG;jesD)7(CW=O8}iVx3GYyp5YX62i5mb;i8I-Tpcejf-qFGP)Wk?>R_rW`5Pp>|cu95%&G&e|X)6Njck3_1XO;PBea}!8b z z>0J;~z03tVCQd?Y z9_E{VgW)#`jzYNA2rGCM)e~)hm<1wc6rt8SKo!(Pflg{`<7il=Vl#H}+1K{w$h7pWo z?pNq^Un<)`A^kxplOQYsk7IFxCe2nLj>61g__4v7D61d~e(5s91ebCiX7c8O#k~G- zi1QUt(*#RRCUCqUr?_1ioal(y>_Lf3nGT8@?!DW^v&C*kI)g~m5exKE@iq1RB|5*L)y`I{*z2QFDatLZK-A~UB`8I_ns1>H=icj{Tok`A1g_$VpkfzeS>^<`&| znEL5vct49Y;MQYLEJG1d3sX5Ahe^w#FHeO)Vun&N7=5ZKLN(A2 z6;s^9YA$7p3=f9~tA7}a3#YN1y8}jPRuW+>dE9$023KfAqH%api?d;K7*d3Zv}v@t z*tkl=xtP7>>@a)a3222GYpQ~7kS~pvObtr#9HQ})=~RY0*aE8OHb%d)i6mb}R?=Uz zc%u`P(Rs<}bR_zf;aNy4LVFj;H{^p#-)C3mZ!UFY{GL*1x@Rs;TA@L(rNDLr1EB>i zo6NdhO1o)cW=z<97<{aAX7Eak%$igDL31Jups!_S9Oo~QdFL#DVEhCXOd?0vxnCzt z(<>Xt6QNBeO|tsz7?bcvisFzufvWSSH*d;xskLXl(`$l%8QRx9?66$sXVO~CdK!Q1 zYCiKySb^hno6t*LZZ*!cQi>Vke4FcZoT5*aVW3A`!XRsfBEt}AE{Iw4kg&QaF2(TD zprGye;a_PqSHU@E6tEU@ogM430RDOlSBN^KXlnmZ1Ul|;K1`6TkCL@j(J%qC%S}{2 z$VO^@L0fB~gP>6Ji=9*7gJEKCa7VAwl*61V=&ne?ap zM*BkYn%lcsuKz@t$P!zmn2%H~qCWn4D9qB-C< zvS5THq;ekA5M%?q8?NVeWMhgDEZ~`oZ^yXO*FpK196L*dOOHq-~y}tniTw*C3~Va3;qnKWm`JzL$7`uh%Cs_xr3lGmh2H=6 z8Fsth9PVu5Y=UP+7>meXQZkRs!r;piB^oNfMoUA}E!`1T z^PY1R>UtZi)dGD}Jq_u#KbvGHPmZyn?J-tW2gzJXTWh=JjQ$6=mM-4mU+7wIk2*VT zED4(AowY=M213>Ad>xz%h<2La-oJZv>=oWA9^c>I*gRg}-#a?o-)X6(zV5#z@+Ro= z@drzkIM1K=7g3GnRgTxuk-}63UJ)qef=upiAFtopT>sYX{da&?sJ;|$uz(1;5@)TJ z!{XLC6GvY7L9UiCtiWB4vS0$=ou3Yftd`~_GQi`k0|?O6=H@j(qwmB7I2v=4vORW8 z*oq1WmJ^jz4*u-4MUL&rF2T;@|MS zZ~y43q7MkWl$}*l6BK}7NxG77iv0rPt*kFrD2E_tA9qU5T6xcx^X_Q3b-Z!sICu+h z0o{+YJDVgoHra0GFH#p*km@qMPt)MU#pS3|`m)4(6jTI3ErYib=PzBAacEr_)9#RuN@4mH51JnOh$&tvKAW2(YGUQ2=1j>Z0XmC>pW! zbkS%{uxdRHGa9QfqR}CZwsyaAUA4YPOe8icL`=9p z1%i1qDUkPv@j^qGSLP`Jzm7J^;1n=B6mV#pv;KKmDiADAnF*zv1Ruuky^hpcHgu2*ZiD!xO6~SgK=m?m6ii zr#aA!04_X3yLy5`OKwBv5J@>JP$*+-HHqEjRkd*5 zv6+8$M=`=fAE81H8)n>eBBp5o=IqgGpQpKw;c4TKIjSi3!x?dR&BE#{w(Y~C1iKOa z7cZ54^pHq?P^2Ag@2<~7+!FQ4_3vN59^YGv+jUjU`v>7YqssXTrzL{nR(YR-;zN)> zvU9`_|$9-2>(iD6Fn%_fL4_xq162n#RYz(L8E~ zv4>InVE3Cxn}^tR^jkjiFi?>Hq$7LoKnl=oGbdyy`BJOLEl$RpkJ^}jWz?6xCK~5o z^4&-9prSOIp^M>&vp_>~!DB50>00@^61IRF*s}~~#mG=YzGjGdn%as&i6k$X7g zmOwPZ=|o5l@J+G(Q?+~yk)mKvs82bt)g0%3+6jR=O-8RvES6XFIB(s>G?@!yj`QR& z7{;tkgv-|EBdoCl2?mIr5Hsp)itq23Yq9lEY1g?Cb z;pu4_CjsBLu~YE4iA|dnT?)^%+GDIJHCv__lkgX7xzhiU5z~&Ir4dg$8{J1+D15W0 z+H5m+VSfk>Sv4s|;m~}gn1F92sLd_`=*eO#XlqIyb1Aj^*6euF>2x2(v;C3cKV!w; z-9DI4>n-4qR9?Jlf31XXHKTS1i~iyo>^7W(;jw@n^Zhz)LK51UU?pk$NpGB#MqYnJ zUk_f(&aFe^xHNCUN5}J>TU)q(KZse%NPlF2&)NVF4(BvL2T;AtA9M%fnKvzSH5@*z~ggR>{0JJ??2Ys7FI1m!1BCOKq( zecJbTKzuRxM(#z$czvP=wCQ0%Ci>=09WFrhm)>n+#gZ(&zRuqJWOy<(XRG+I;nmsK zBO?$IgwFWGyE6XZUu68@dcTF$tcP9f96P{Z0Y+#|dK@KieR>j#!eXjkMrWfgFj<nJpr zz2MNoBMc{5C=3f@yn`hi08>A<`n?NmgVnd8(2b-0T|Z%;2D80ko^E-KDn#JF zHHWMkx~3ZDFJK#MWc{)*;5k4>s5HG_0V~w80Nr4}0h9a$13{b-n8r zVtQh#*28W`wvRHLxB%_(TL(=xhxy}UoLn>N;T(hR@o|2Pmxj`TzgzhE{DaL-2S;d~ zV+HQm)CI{D1R0n#w4cRNtdEA{@lkhlzF@*WUd_Z?8}Aq<%3ihPNF_v*5BO~Zc&k<` zT?Xklezi&g+7%R02Tc|8m)9?wDkXWkBh7$5hP}UUMOpbGqc8a~Aq8;w$}Tq&WJPcHY%uoW$~Lakb;7R%)tep0RBGJGY$cdL=FmWp+pk`BG8c5mSr z`>Rq|b|{1{G;r>Esotn8h>sOI+rSA%}geF^gXQie*Gsmirtu>i$h0-;;T^-=@=XbH+w zORXAXYWVFH+i)147B6kM3WYAi$0!!lYqLwp`>tQSw@ejRsuhaWTD^cqD7AXL3}H~d zsAkKR2CB>&8o%^L<1&0|#zLTn3K+*{z?$bS;)wj*@-2FeGMWRZV0l4&u8Ayf#D+u{ zr7dOXUJ){^!-$t4QHvSpYE~-sdc&)f%JpUN1*MGgfXUU1u#ov{g*d;~j4S?bm1s+r zA;#R<{3L#7#a0}~r{ybVhk6i3S1uJRh52i)2twD6p)NQurcp&tY5vwY zg3m8tA+T6SEm(omRc_4R97hn^5-y-GRq`rctx{^B$2@P-6v5{gurI-SwNZePG`zz6 zN@EbZR=2z%$53iPd-+PChCR|zRWDFi$I%6?aT)b$xlu;r=`AQimt)%+7wVOYS8Bk$ zEqe==aR{MP%yK*P@U0qT!ziIY(>^naG$Hy^g1PsE$nJ6E445-6&aKbM`NP5 zg+`;&m{-{%xLjo`m#TOHx>A|ZogU9(Q|l1c#OP$z;8RslnWWb13Ak3NYjNMg@AY~R zToD6vm1=256?_SGjw1!lHOgD4rTfa4VGERvV-lz+;YxXMztg>dFv4(?xU9Z~u~~hC zmyCu6%|oT0I^Yt~kO->4LJLr}dZh&C1pe5Z8WO?fh^$(#)#226wc3mxUxLI|scwck z7on9~wSXo86gC!PHH0jXoY#PQV2-5j)#tVOIB8aPz=WxmG8+3Ct$pIq=R4u5WO1dN z2dfn#45L&dXwRu0U5xu8Y`GjGkJxT&=;77Zz<;)mKGp7^@`|qD4iG z@n(#h6lOT~$)b&E*GkDv5iRNYrC5lD{SF_Xs+j-Fs?ProaR`^I)&jO$<&2y^KBr&tekFzt_nbtXaN z?e)zkC%uW8<_%_|oy51-vD_d5#1e7Z{}#3m;hGjRwT72>{HZWhc0Qd^5PyOmLpNz4 z>sTDJGkk0?p?fh4>`w3OVG5ktic*&YV1yPvUde;wSn=TQglIF6%+#kHLqM2tgLjTj zZvgs$j3x^iD^a$LfP;k%R)$!~L=gwlwRvM@7M2(EF<;dkPRCZt!cLGcmb64lDVd1= zZI8+xWMoNbkN2i0{qA&Z)`g_32e5R7z+nRUiJ#mMhC>GKaK(l%z!)%=KIn^}I7hTA zi!p2w#^JX{va;u?@QqRVO0iy^k>Wv+B3TfzEU@6Civw@4GNv2M0ZV3;S{^$lmg*R3 zPv_`CJoVfeb8!O~=g7(;AR+;ao(MMNWIR+jpx(5lV=FBldG*V5wDV1a_+c3Y9G>@tT%Rji1>gY*Oj2NEGV^pSr zksi<1it{1FF5WoU4y&_F-C^qTP~ps_or$Cdm{(3SUrwE39R)WZ$Eu6dqubr2C*)IW1P|ZH&~No_30*S@lq2 zj*#&L>wuRhopK#mRY-F!X4EQ)^hMAH{zuu%m&;gOQiCIsW~6f=^xc(WxtK@$hV_1h zB1YL3LK)JnXCQ+#{L2d?74r(D((b&~?!i46^^7yMDusxkRT;o3@~Tz(=Z!^GEQ-N8q`L?w&X33;B6C@WrPcop-bl9jScqhf~r?vq)IA{VW}W1KsA&~w6C zY}44IPsMRYjQC=|+Y>sTDssIl*~I++zC?9_`KwXIdK)28pDgKcWMT1t%G3^j69aPtAO+@BJ~W`26NUMZ!uIBwg1_9IBZlJ zn7%gE%<5Fisb4URf`x*R5?UBelCP*Nm_xt>mQ|oT#l@1<8O}{RH2P(ZEK4)4wAut( zmN;5*E6rArAhG(1xb10lq;ZpmHqBpM75<1TxGJ=>Jc+bCW)xAP#FuME(gdDrWmh2k zYWj(B#Jkw%J}p1Y@>`qpVOir(xWTLO0dWA};g%UobUVimsPi6#Nl+~1i>Q?_c3yKG z)8rI;J9DP7<2ES+1R?89W!F)QP=atP2d@|JTQq8A&-Ho22~gy=hM6aUQ+K;V(BWeJ zu2VJ6fGS%DZ3a|jE~x5UP_?T^LgQlJiw-hq~IULmdgZ1LUp?5LmFucC`#m~s9= zvUa3FH?b4bBhmB1lS`4GqH@=FI%SRo^|aAnw-8eDI_)2H+wUEA$JijxpJ6dc(JQ$= zKJ+JGq|2`_RHN1KBe0*dyur)9=3#r{knBT*1_;x{hxRoITnt`D^bJiqB%cPH*vMHn zLbPBZR(hkT8T2{MvEIJ8H^l4(USCdQ04_+7WaDwS-{*{L8nCI`;^5%cFj=V=9FqG1 zI$b#?@p#wV^zWxV++m);oec@aMW;$ak@wq^*7PDZ?co--Q%sHB-EiOf9)}Q{SLTm~ zeZ2LJ$q3QK>tPV*#b^5LnjB?_VOPFO51}8)EAdXW;rDi-dGdZLUAMiISmptwQxr|- z2&aX=H|UP#&2Owtr@AW*E})q*+uFZpcqmp&X>fc+8!zE2WID#NY{U-|{LKq!r0xFE z;VgY&yebHmzMBNwmn7J@Bz|V&iwrk4Lrmo(L%hj8d`ur@CfWchNbICk753aPe@$5^ z%%3I>h57Tsp|FaWOen0H5LQbFt0#mt62d%P{^=6!>1g8hvGK^8qfz^5zR&SMRrXML z`z$FKZ}Iut9E8v~FH1x79X!~y0WaO-U2)8Yn_$-l9Jsb5npIFPS|1!G7PSj3Jjufm zi=#HM5z{axlmZ&D4)l6xN6Elr-N7K=LAz$;cFUUuuLjjpDMxXOuuv=PkFa^c++rCt z$0zdxgX(#K$49O9Br0pEf*g?u)D(*&+QB{(V{=GHg0?x~c#)esIL;&{4$cxCxcZw6 z(!s_g6FM2DTnc$K-7eq&M+HBaA!74ZH>ndA+CIEi4D66P=*9=Ba*9Dkt}|w;G)>ri(I^2N5|_%+G_AU&jYaoet>x3=gBm&&z{L>oNB2D6zdh=< zClBvnGnI|0oD6+D^|yoZ-z2;~PEF$MgA)aUM}5K8(y#{qFpQ(k*8hY@TZ_Y2zIk_)7Kx2?(5=LVIjUZvq)5fRoB-mlHHx4j|%93Y0_9kf_J#@x}L zxfsE(SJ|LNae(@=0xsw-2h4=QrYGBna*RWQAo`TU^a4jlCq%-qjc~OxNwd&!e$iA~ zC6HQ6p&j)Kx-i@;Nph!f6~H(T5daKi*ZKaW!eB3|-Vh7?*0G)PsEq|< z+)Ax!a1}$jDPXJ3?L-D=8)h$+r?&;vY$l{Qv3AnX26N+a-CM&cwh9|zH!9tp%Zx+# z(0y*O6d@8jYXlTZ5p51mh8^U(gH=d=bsU0WP85Q~@#Q~7FZ-n7z@Zk-#S|HFFj1oq z&E}#!-pNNBkHlVV-BG`b-K57rWAGU~4i-Ze^LRM)t=BRz#r||dI+`F~@(*&yW@!(( z3@Rii0|V>#Da2|?Y)*C=jBKEW99#_!FvzhTM3Wf3fzb(UjALt$AR$^MZaj#cn`4wy zWN<|jxoVD|o}YKIOzMQ2+a>0Cu)p4l;S8t4#&W~QUAt+Ym5AScNRS&V-!1I2$5Mhc z^8L+&ZKsy-BC$u)(SzZPcq95tfx2gqU$C)XDF10+HUvD}YF%J`sH%WjsqE6}YXa2# zaMOszjPr#dT9Qv}(XtDL-oY@Ll0J(9<=Go(U1ajXOB$#VA3jC#2Qq8*&anHKzja$I zR`v)@i=1v}9Z4IkDj7za04@%Y0Y`0`4i2yR;nT^(^B&I8@K?iLo(IoU z@3l|J?f&5CA=-oS;jEw{)Vbi|Ixr>k>mt*Kc+-_{;sZhEH+Gp z-vi8}uDe5GtQz$b29O+y#YS;%JKMVGo|xW_@8Lc@VcX-rrV)Wq2P+xGpIS9d&+Yc0 z^SIZU_^+Q}*iQ4}#-!)yX6&;w;=74PHm11?!20a;W5G9Ool|Ib0QQ0*f1BP;v?o)$ zx4)SjVc#`Lj%OC!3|4DsnFM7Kyz;R^5sAi!hxQl&MGMyBNO(j0b(^#kN! zumGIn5g4WJbG!*%-jm6JII672Lg6LZy_jS&9EL8vBLU~w4Z~3q&7%N0D^C*`trmia zBgFfviGIP>2jA@yjF#~to#yF9=L zbI@!KIvBX~w-SvZ@naKE7%9Z`m^5_obJ!l7b*%eDm{`Dme(b_-4cRXY z^`s=$H&pW;-s9fn-Q(^mi2b)l4DHgENY;NNPR}A9F z2Eh3p)aj_~uz{{f5Hr<>-HWb2_+obAYNy)Y2W5F2cdT8)3p12)AuwQ z?Ti}0Nbr&a)&b)^5)Ye2&5=5~LPrEn`-o&L&lpO<)WC4W?X{hOUYtm?iIZt!i8^$k zEFxN?v2JUM7%q*aY9BtaX-s>NBFDIRVoA-kne5dq%}Jnr#uv2t80jV^Pkj4!o`gLt zL+bcPzZkiqiJB+f_Ss-K!dAMyzL`zb<|SSxiR9j1$MjBUyMI($^JM!#eac-p)|_a2 z3I)Buxg9uOL5z$`zUIMXNtikBHYc_=TD5WNj>7C8%|nx5p*r+8`#e$ ztVd1x7tIFYi{UZZ`U2nH$m5WNUO#xvl&6dymx>7$N1|q1Mk83w`nvVuMKCzL9F0D? zhG*1l<)!H}T-);^fRl_LK3N*^w10Qddw<&fR##Z(p1&n*xW?ldt{?+z?|D=G1f5f~ z)C-{F3dZGaj9AiaY%yykrW70FOZN+MmS$sob8RtHE3q*!XR=+bjm5DpV`Fh_lZcH~ z7Bq!8#;7UaEty&ss~tJVxLC0m@D-T?7Tm1B6o_Rtra(-qF$H2uHHF1w4pb{K1>kuG zrVz)vj48yiMNOf)pee*LMopoJP0R}oY=XRyB`Dq%SppW8t-un9CCw7%><#!{1Tm#q z!eZ)6Olg+Db%hJmK0vh+O8}l{UH>##z0JI#=xCU7BB{&T8S|L&oeNFIM!v1A&xC(3}&;P#l#n3 zJ8BH|Mxoxo=?sgRf|9x-Q@~>36_^6Cq?y8EEYu zDWVgr=#a;;E@KLDY*AD279BB&ql}uuEf3qAplMjt7!>o0i~;N7S6~dpvKnI`rqvh& zF{K*AVipQiD=`M(c?QN1$GVI$#IePUVbOLWjxufx#X5S=>@qA=ODX0R83SGcS%EPS zOPVn(RtXbRiZOVLwG70RW(?fzcY&4xs8(VO!1D}@A&zw!V~Asm8bh(RVCOlGGHwhQ zQpK?jI09tRf-wSJkuhMWfE5@6v8=`zh-o#(KuoE|uvi5IR4XwC;CTkd5XZWVF~qUO zjG?}uF~m_ujRA+^6)HG2g~z!qP)RA`71;vz!&reW5KEdZEH(f{OewaoSdW*8(oDfy zY+ezlR$>ak^9)QOj&&JRh+~VIg16}WLL6nx6tG<<&K|>Vk_(xFB3_Xx;1$zV*n&Z` z8e=f1R$>hXQ>r;ECU)Xli9Ha}Gcbq*w98mT0=k$`u;5fLQKgtfaj~YxP?csA*nVY^R++d~V-rC1ENmi<*>EV!=ABdRpBC@m;>;98AY08vDLS7H`%v`d*q99`5biVL?Eahy@J z@Nj4$4uq`mK=(zgLNTw(Dlo~n3acQR68p%n9~tOB!rtFQ{9NwbOtD`lcev5Ezkk9$OwW)(b@ zev#@3xK?8oK=drEB93+`tB9kET7|c8TM@?@wTfFj%LE5-a+lHt3&+5($|^AJw+gEu znl!6e(1Rzc6szzS6c$mXSw(TN6^9;ht;Q;V=vi1r9PLt85l0ubic(?Gwjz!*W)&W+ z0z1*dk6O$u6!ogi0#l2tFbkqtky#Mcip+wjQq5vP69cZ*m<13$3$uu$UCJ!t=%Qv( zT)4j-#~C#XuUy2Yx#jxeWwX+@0yBZDFbbkcGYW6PaW>*gv5Ey(MtVe*W)rNwSB2+($vT83dIHN_ zJvoa36HrmEutW#@*cWR&ylZi6QLg5Gtu;i`l3Za;$;OaOL7GlA<9+7E0v3=2DzW#f^{IUMY)=Loun-*U|EtY zu3?I>#d;lQjpFRr#S4q*Vtg&$?h(zhj3xLNb-qE1!|ShLTao2U#%-E=)sihRqFI)) z1Q(-L!HR?;4xnAUG3JVgXo=13Jv)po?__Y!Kcb3da8+rP3YEqc7)x?97B^H;XfyYOphHoD+fkl(94m}w;Sxj3 zx5V~l zl5~2Cy}qZTu58teW7d1z#dEx1DZ$p~ooSg-Cd7j1q+L4~;}?ilZ6|S*%h*XATg*-p zXRBumFT!)wP;OO=)q1s3P3p^Ef;eU=+vlc{!#=}{S_)YYp>r(dRv{1Xd?7ndL>Ew3 zt(6m8$4i)vSW0-$&Z{ZniVY7;FWfpAuKLrnasB|mY3*@>-xf~DmTbGjqq)*h#9i7n z48UcysF*NLA4vm_m&`T5pr_9!b}J&;V&GuOYy%8Vw(+(gk}bOROJ(wQqVQqZX0zq0Uk;LwHT9zv*kWltZ76@=HTOdTSc!5M+pt(&envkY7 zn}u*snMD)Baj6($IKowD8uejs23d!bnQCEvUnZU&bT3xjM)D7wgER)B_83XOn4b`* ze=#p1o_;YOA)bCQ4E9O2v(Vh{UF>$xaoG6DdzTywg|m;w zA?TUsLvfxr42|>2(%Hdz=leSw6hWtVb}pyQrJdhX0cu}ZIHP&DeQ|-~g)p?@ZH_0s za~$y`1JrU7E70ngnpula1lz-Vle7wYn1? z=sD)GP@|qdqHm5i3IiXsr{j5m+ZZfzNtku~5*FGAnY23#rNa?vJQk2XdE$Y9{?OX4 zcrDWteeOL0v%wKg9O95MxF{fypWy6b7xLif9m5Qc-a2~cs69ICPM{=75Mu!B9vqr8 zx3vJxL#n25>5Ae{NZl7yC-3j``1Pw|88pWyS5Av_`mUIEeAF_B8sd!gi(tYi!mK%` zN=0QJ7Rk&_JwO%qV1|si#4$bd@xcsQ+e~;52P>YZnmmwi4{$(j*BmxH+{D4tc^s(8 z5Zt@E5(nEG?w{hwJA3wN41`D2o#0s7(NjJ7JXJ~wbi`9@mmOx1ke43RdbPZwsY7}3 z=)DvFbYPt1e)1lSQbt*MLYO(ZFo-v*A3iG4(Cgv|!-U0b;c&dSWOW;tv(a!m=$I)< zlQY~1H{Spj#}SWlZr~$5C@_+D02iDD=(jjwbZlu*f2Q%cI~=!rCuy-f;7?9i_Rlo8 zfE&Fh!-3RFC+2=e6qRvmPoynS)NcB%_IQ$tW4t>uar+lM(ThiBn#Zm7Bq_%f==(f# zaRK=4fjyNvLT*Y9`xh5OoSC<9vQh60hkW)oPaY0~v+7XNRy>-Y6=FmZh%oHoKv!t? zP|vsx$#@MqU@wmN}hjh_y1=5Vv~i_VNfKjzLDC|G6ry(eyl!8OJqVpq4VGt?}{eTIM?{5Z=p&Vstk6pu^Bq^-{zv$6 zdBf?ozBd2frh3^)2zDk8$kN2r$6q)ewuJ-O)W%s>9J6E@`hhdA=4x)OrYirun2d_ER~W zzvl46L|7jm5V|uQoH?n&GD1Sjyh-Ws6lA#3JAb3_fg+NpJ7!o>?r--n{xE)T_hR%$ z?E^A}m#KR!#+bHx?r*QOA09 z9Ighv!SWb$6fKWh`@pN`YxPQ{R)!8m{NZmut<7o~i$R<6!fQ`F0_^O6)^m1a_JU|Bc|zgQ{aJv*2^Oud+| z*0^R12M{v~1y|tP)kSIps+%gRi(-v6N1=$-*To`UBKn}7=esFYR2vPc9I92Q7HT$m zLQPjLp~8{+2K!whDWbdv%DRr1fveEKIcf@rve%p(Ukf1$%F0y@pL13fmLDFNL0l~h=UJ-8!Q$dPoAiPSgRR>g=Y&N{8wYu+5YXJ3)Ral|{V(51?3bDvL5ga7vR~S%jLBvZ}}a=s|DN z?jMiaXFJQOM=1}pn?pSy!a}KB!du|9fp6 zA9$#fv43O@=hx3uUAT0iyljTAxrl<&3bezrj8v8zQ_7aicx~Ye2Z=1#Ok}wU^15n0 zQAtpQ!{roVUPrTrqOKI`c<~E`Ql<0AciUL{c)u-C6~HF}UqIY6DXt{or-Ma?O&o?4XbxIwQK^%7-cx@NHDcGPJZhRHN$f zgLE?CIn?0}u#gIP;kL>$gC`KthcdQ7E|&`UVGMRDpi_VyK=6dpY@mx$GCdMUA8w&; z!ckpbsSb3+UPXrqK7wh6pb;fBF6_$)ilA~pOTfwoE&Y6zgQe^>H^INeZNYjfSSmN73x6^sTz!>8n(^BZeYfDZ4|4>0J>EE0Hq4Y z2K5v?+<9D0*mpo1^&`lW`Vn9XO3T#`VV&w>88YgDuDMd0pdRRM`C_7a1l%*F!&H!C zTt6_fKo2clKm5@eURE{MxOG12nNgu&sm$1nxafF~j1b9_9f`5ejE{~V2RO}0Q#h3! zuMr%=lD*fvB%GU1FzA5UC5Kd#T++_PHc#H4_GFZE zqdPts^~_vftNZ@6J2=72h~}dBd6kdk8Rh-9H##2!z`|P$dKbo^*yPIbxafHB#}URa z$HRE_3Q%`vWH&lq%yEPgUhpp!0qlSNSZ@}5HvC}5xI20TF%1X8S@@SItz6uUL`Lfv z-^MUrFuu*PVSdREuY3FW@jq9kGn)RW-S3VMyQkd|%kg0NxI6O8L?LHi$CW5LC?Djv*?u9@cNLZxy0`~qt#&fJ{R?_~@(letXx#SM#NRdl!7r`^d@%*PHo zSWaZ|$L~18Ox=;Ob{R94G3O}V&jXHM>I|>jo9p|B0-CTKq zPWDmUp*6 zl$EQ>*;*|_1VZ5?B&zo^`zQ9w`@704uB+0U<_L86+Wi(5?(Vc7bXlvrHp?tYt$gIp zQZI2zH(^K@7n1hQuq?D2Od0wd6JASFFou6sGI9Ut?zlVJ=$>L7U#A6vyRNq>sRWaI zuecdTnxp7eKWI>-DqCR=rrOtGn?6?-A}+5E2GEF@lQBpfNEu1 zcWz{M4;4=0znMA8?(Bat^RV0Q{6f3)s12W_lf%kwEL%40k8}OrgHfA8{$}P^vMtPr zz73ygXaD6)YkPBdbFY=#J(2E|>mYaN z6})^EA)hc9TkSL4|CqVIvw48~PvSnCxhTDzc^PS*&m0}(fb&!4{%3X$?!2A(jKP0V z%I$t_Z{urw`-i)4XFiMgYgiMI1M(iQd=4-#WSZXB4z>>d=YQcwX8%4!{!b|PAAOdS zfYudJO;=ryhbQMpJbyMnl#p^SSNm&YC>s}2WhD?{IE_oXYEDUUHY_P88~ej_C+$s~ ztFT1k@r&}XJO-QNDsfq?SHHU&J>gX0r>oQQr!l3~HNE>mN<;Jf(=+(v~ z-??d%A+8pxc=dWUEQ_SNdQOy8TwZBm+J?-|Z+7gwe60q{yt!Q+E=g9DllJG5<0Zdz z9@AseDZNUXbZj41jA*0qBKJy3_(?;ra!tmx3i+)9$08gh$w{mSzP`)UgNUIZi`3B>ZG~Ps->2uVErL`Bfe%m#}Os zl)~s$WzFYs8ikM;7tW3Ou5QRbn>Xv%XWh z^ve9w_;q2D=}JqP3_UBa^c1sV7SDecwt3A>j|dg77CTlxNH=E_!`F-m0=pDc*aavtZR2xe5jIU<#+x)L{;iV3rTFbjzJmt9je2 zy^H3mF(QQ|>J_WTMKSPMuvmJMU~HJpGo02!*t>AYYokdmqL}YBg3V7plTB zHqK42DcvxYmkrajHbm|uAGd6x5QZa(y*lKqIi>VSSc`9`O_6hZfl}xl{SUda9&+{+O!ja&VlNXh1DB}2;Hr|>-~kyMsLy%R`FtH8;#d#^HA zc!_f%!ZC_b^Wn35pOixqgyLDnRkY33HQq;g&@hS7SbY*nMnjb|ff5*{SYh>9t`e&0 z7-sKzr>ShHrjx5a#p9zPF8p%$+~Y(o)}@hHg>%VU?lmtyS(z$l%AY{Vlcn6=x$%lI z={hS#IIno>Xxe%6jANP|LOLi5(LlLl3JF)q*sHx03YAWQ*C$PHpM?tdnJFwbiRZI^ zq zgTPxHHYdG{LJ(HN(y&;3SGiL=RQ(eTRCzK_mm|5izsylFSRRz-aEOKqm4aE)3!H@8 zU*X&4F3-f`el*k#(|b*;bZABsC~L?5_itvDvtJ?}TP*oU|mobM2UT1jc>0!D^k}37q8W zQXJ-9;sw@)(sW52Y3>$Au_%UkDveQ{(-f<|D)-D!>p`JL<;WbPx)^;HDkYuEt=!uX z;YxWEcw0KkTNGX>2tx~ui^;uLOoCf~ZK&R98hu{;Dm9(pk$bMBmybRxjL@v@hs%N9 z+pAy@yq3!FGfTncjj2_d?mcnIGw<}Hux^~@uP|~Ar%j_KoMzr9;-*9dDq8AE{gR6C!c~-~Ff}ebV>FH4xz9qE;1#Tzi=?!$HV=i?5T#0RX{dWmkFik6Frr&R zxzH!I9K-pnvei4$PQm3$O}Nrui_hLkoY13iylSp;RValZyxuDZl1kI*Qy8;6g;QHd zduQRUc1KLANtB1kQ+{nO3L`t+X-fy}H<XSgZVK#<$`q4WLldHg{{YqMmRS8K-eNz0qDr7h>K4}^k zI{n_tPqamCRd)1LiSWvg;AWhBtiCeNO4w2tzq7OpyyCUA3wB9qq1|VjM)3#YI^CSPC?RY2~ZGHYWW}-gqYOA~B^vq{>}*87@5&?VqT` z86)ugmGmklg;6>!obnS+qi}*n_|z~$%15OqF_M?YDzv>ajXt^e8pf-9Y=|E)_eEO; zCJ^d`KjmLX6v!9D^h-UJdXT^NDa^UjrPwXU8pEsR6}?v&NgaMt$%Nw+My`yra17&< zhUryd^=bfoD0`oFN9yIz%1 zOd+8fuIcPEAw>=nr@uB_F-Q&uVSrVj6syGP6R&BX^cazy@Wo%L|K!ffhE^;e)jHKn zD`@H(W7BQxYcpk!Qskj$wxyJvtDhu4`}e|yTYx@zFme4J-rNo3zVzhUHLDR{TNoPChn;gjigHm%qxD!n)7IqTz}`Q%H>DqrU>9 zPwrK6R;q<=!K}WsJQ0F&(_h|^O3GAzRmU`p>6A)IDJd-7E(GrdrBTv6@zL z5h)5ka?fz#-Ibp5DHv>wN>;E5lwgp1^6$oJD5=m~6t`X_jx>cojMtCe>s7;fB~APg zzvdz|C?&j7CWLUMLs_$46d!`}&}We#vGGsKgLf{@aEy($@PbEEMXwqwxg`B6CgCp$ zLS`}QrVOe$6q(by_Xo`wxMz-A@-Lr(I@wrsggcgib^}v@Lpsf8fPt+ z(3Fkv%2b*c&NX^;CyZ8y`;^mX#BNhA;ew9;;nIX2+VA^jG7PlD*Ds78~SRRR=I4ur&c5!L#Ari8`nvaAC2Br)D2?zR{ zCO?&rm6YB|jEyIz@W(z|E>x2wPN=f6;+=>rrIyoQ<;luLLNsldH{nLYBo9sJ#;67o zPUVw~kFgem-1Axg1cN^DDj{L%FkGdkl9DuhveMExdlgQ!?v*Et&C;ye#Qa4&v>g_^ zr8Fg+X(&OLZ@m}VHBWg8$H|laDi3Z7y_YyQwZNBsFf6b5dNlwav@UC)DJC|5$o$4KN$-TrW-Evj=**Kez#7SFh zsHl4Pc7zfKnrnOVgm*_1R{))@%k-~c!o^|<`$ z7_$0ZLe){EoJ%Mwj!|Kpwwi&a_0zVn(^`zMs=-bU+ zcfZ<%sj}y?O4G5kC$!yv{hGsRkpYWW8M4an-6z0(Dw6AIRy$6f|1VUm&b_sU*^b=T z-$b!|Z7t~`t%`5o*6V7=w(G|2Z$g=f`_z5!kms~p;oe~>wx7#X>TFV{-<_iyUYJu> zrBF=$(yn#aAg}K0kcnk9GJ7?%uWdUqek`wc?Z3JItk(Rja_Nf~-HhlDPfsd=26#PV z;GNBB=01Iav|jUeR`XJwm}hU4(^5;T_&lcyT{VeQSJzrh?rdX;(|f&(;{J4b4>!?<~7GTNIgJrioiCr=Umu%gtwdpnVM^u~~KYH0nm0kd_@ zpI)h|e|D(mNxYstB7G`#9%SR0?&{Q56MHA7hn~zmHB1p%>>XSFy*GcKBG00_svf^F zt-Sk4RccMW&H$CH$edQR`gxzOwq}DNn+wnV;#*aQc{;lmF;7_?vhw$?Qx-KRO^Kq_ z&QT}%)CM&Bc?L#(5@I-afBgBM>cQI!58ubd%5ol3OwQAkZyE`29DYTd2$LY1iuw=a7F<@ENB?OAYMyw2i0*pq22e!Dyo@P(?9 zd}yOWR=z8#hbLvlbaDo}PO+J5D|Xeb*;&7L*%kNH!|D}tsF`|asHS>y_dk1uEtV>% z77S{oc0CF4Vbm{9HAXQznpKsDF=kii!Pe>-wo~6aHQL?SD;-tDx@qX=)}DN3#jY!W!zw9ksZ!i+nkc zm?-_??RNO<9}Z`tzd_lp=LN4*Fp-~w8|$3vBSW33nzMxh(Ni?;XB$taGiY@gRo1f? zLVlRATCtqPnc3F~tNLZlUbWn-u*zjO1>B*k1=6ZM*RZU5y0iM-5hg|K*`2faTW~n2 zYjxscWFc#`nQz*jS5FI#{f?F=-C&k)JpD@@p8XpA^tw;0)N;S?^w76z)%#T}lU-mF z=X79UQ^QH;MB>kv8f570Imj_Pk=|tx-g@L&a7^QdQml}+;{cty+kHRz7auj-cSlIl&wm;n6B(S2~_fItJmzw zuxj5jvDdxaDbUlu>efDW+_#%m{Z19D;LOcrXYx>WPKs*i%iBAW$#``>Q_Fmx{+`fz zL~);3JNqEt_pw%=6rehfo<5!u-Kc&2tKUp|`rUa)tM=(OdW}`gTJ_yotox^xE>ww* zL2yUOQ4t^B$+vga@cgM}evwYC&naYrO?}290-ur89fnkx6<&8}4$&MG*k%)RwtIQ658UT|wWGq&}- zvm1@@b)7btuCekk)AT)HWKT*)IooPJ)t6)JL5@;bXC-*$Z!Xbt!=bg+!?uod*sFQ5OeEEaq(*A7`cj$v z?7UL3J%4faeQ6E)ORuR86FoA4Q)H#2O6Vo;JP-3EMzM6XLDuz&UgI#fygH zs+SS&sWz4>8q=FPs`V5J;lH2u$#30yq3_Ydj@V=Q*~X_))$7^}rygE)p_7!XD2KG? zLKWt;W<58Dvv3cCnxOKzODA?swK3SfnoSPXnTYj3`^!`-FI|vtO?Ovb`OSzOU$_)W zuX%!ZL}C$nb;Ue=i6!Ukg`L&L@AGGpeI`5|qMG(POLfmGddcjm?mn8Z`t1ipUUesm z_tXhBjJtNnf7*Ip-#b56W%B93IjW=|r8ouPSp5dltgw zzb+LmkG(UVPsp66SB>szWA0X?uJ&qNrg>7T((?nqJ=uFzKRs%Ny{~h(Bid=y9MhAE z5$E%5{AT!l=*a__hTzcAcetI!BnNl=@?aQMmaj-@U4#H{EY#QBnWeCH|C zk1nIySn9R!;K9<17dGCyn{~WIJ89T{Dmq6>icSf3zKGkAnVi%8svGM1GOBHM5%mkj z3g@uhJrmp3o*erg6LG&&kBT#e`;aop$ld;(+*9c(7o0@4T~@g7ujeO66F4z5b5X}g>fs}kU(ow|gLoWfx_2h-=ojKT zrB(9|Q+Vf%p}4)2CFb1Wa|(LW61VThCffe}3WI%vOHF%JwcWivgKvs3s^Zzg=D$vB ztp0wC&vp837C6G_Yo{vg#wzP+b@GX@Z?~;04$P+)o$z1(lA|uMJ2}2iqTToXU>tOQ zl}+85m7bj1TI##KO%|8vd$#85p(P^;C?LHJ^B+kY^73`I76ZNz;w;=bXfy%UO(uwg-IGPXlYmg$CF~v)UII z9~Q3{`0MrL3qv=fTfL1Yb{wKzQ|%SZc~)muW=w56uL2X;L;k(mrP%Gyp1x9oNLpz% zY29q3Xb|r-dLOHoy*4kC!R>^8s;Sa>CfRuo&U>8ZbMFcBuwj<`&&HZ)U(6q z>~tB-Sk>C>+CO%;Bi=jC)U3Nxs&;bGULdKkHPv&fBBCsn z7@YpiVfNt7kcIp7;7-rjpIDwPnVe9T{6^|3<@~Jl)cJQ@U8=?0J+nm5N!?j1J5*dg zjhRyabfg}5>v=rGakpY@HUGN%BK__yc27!k-%nN%W}=b^cBrUI>JKC6{i+HP<79bF zOv^dQ)|$C?ycjDNv5?cp841NIyK(MZF}eHfPF;F+GC$kcS)D>n8{El0 zs>2r!50|Uh(djGn6%)T_J-MtJPtPc(@>l`mEeK-ud=)z56|h;JfxRV}-k`u8p@Nd2vG$al}QuX2ii(R|%b z=JCI`$nxZLel`y==I-}1RqId1RU>~))1?tlI-OW8t;;h)H0#Uw+FQ^4R2MhB$aJF4 zE@GVdtTz6#KV4;-{_18J(L?=+ZM#YJ+db&?j7z=hRIiwGIvK8aIw7ZZtWmO@a3}UF z^>UMCJ9qB(zqjj#US@VRAQP_>clSML@7I`^zF(g!i;;V5vF%x@)0Fy)T6^EYg~8%t zFGDO}?sWq(S9a??GM=$OtJY|Ng^$awjO9<~demS4TYG)6?47>z&TeXcncr@4&umXt z8E1^#F0I+;f4{!NrwI(5wW}U>z-JWD{@Ei6d#_?Epp@YAd2%|2^|yipf73Y%87^_Fj9>H+EL*HP6~aMeO@v zf5oaox;ZzDr_~ly^GP+#NYK8>IpM??dU6wHs&+?XPK^34iAx23p(+Xp$L}~H+Sn-=qom! zkjBhBYk}}3bI4(P@(;=iv;TE#`g_e)$7+>c>QhHgJm#e${Gso)KwYEe4>rED*4|ZX z`zC*^@5KkH^`rPRzjr8qGhF$anRVXfXpzp%6lWRjH{%&Qcj5OeA+|5v>-CC0_i>hg zM)Us23gc$={i(0_DtzJ2HGdV0w%;qg5-+FTTQ?T_?$>kmv-iKO_D?L6Z7*Pc`uHO()x3O&>FrlZiFn ztm6w^oHdm8+*1e7TpOMDWX;QZ?fJUj*uTOKFddJTDIQ7z~sobC4pHmFt^xwO^DqRhb6C1ueL!Lozj)`XduC@Bt zUfH85SFx}t>~;T)%+ER~takE=i#z7=W}WNi(T?@Ho_W7-SLbO&rR{US>reC=E238J zxceH8**A4DBKPabcSn8Z`761o*mS;Y>tU)C4=chr_l=zFGm`)3)~Qhb-BaXTU*ld! zPKNLAd@uT5_21q5{8L;1UDSH#x$&$_C+?{@zI**a89Vp0gQKb--*L+H*W7=!P93xQ zJ7$088I6-&Gi9lA_j3mEWgDBk_N;>$>*UPeTK09X|6h53n$q5PZ=vvw2D97Py4N?6 zu|jOF@jY4OT6eS9)f4``Z*#myen0AWXqwdH?C%Ws>Un0xYWKttf3at5-qtiM>)*dJ zhqGMo+dut(zhil>xMzlEZ@G0ad-aJcdT67kLP;rio~7}*PQ$Q%5q11;<3N;Jv<%N8F9Ss^U9z1X*V65sPS*>6U^X2 zKWA+3wkKEXI>$Bs$v#D9eX8g;GZ_C9J&L}sRNje_Bc?O$rMNH8U5alte?w#&5VA0 zx~bz8H>bq9-LGqTXLn?CwT61Np4m^R5Uo#`C*L#v+543L-gi}LCZ@tvNr&w@%UuPKk5$yN4eqY=!uf3{cs(w=a z%J50fi8&d1lFyju=!yN^yR+Y_o!(QyGiFA36Jp+n#L~|B>}q^WW8a z&;367j`tcHCakM|Cz`0Q)BnkC>WIGXuN?2Y1Nu%_Tj%$k6U~> z<7@xCTKzfppX^l;XWV@@&#A`NPfd5ukJ$119L_3Np21!tzqbEX-cci__j63QpW-It zwKZM5#(Wxaj;8rE>*sM>hdsykKe6xlzpmS-PEOyC>i6@#|NS3-|HuFQ_uu~Zzh?vg H?|1(PyWkW| literal 0 HcmV?d00001 diff --git a/heudiconv/tests/test_dicoms.py b/heudiconv/tests/test_dicoms.py index 4786cae4..6f833474 100644 --- a/heudiconv/tests/test_dicoms.py +++ b/heudiconv/tests/test_dicoms.py @@ -5,8 +5,12 @@ from heudiconv.external.pydicom import dcm from heudiconv.cli.run import main as runner -from heudiconv.dicoms import parse_private_csa_header, embed_nifti -from .utils import TESTS_DATA_PATH +from heudiconv.convert import nipype_convert +from heudiconv.dicoms import parse_private_csa_header, embed_dicom_and_nifti_metadata +from .utils import ( + assert_cwd_unchanged, + TESTS_DATA_PATH, +) # Public: Private DICOM tags DICOM_FIELDS_TO_TEST = { @@ -26,35 +30,37 @@ def test_private_csa_header(tmpdir): runner(['--files', dcm_file, '-c' 'none', '-f', 'reproin']) -def test_nifti_embed(tmpdir): +@assert_cwd_unchanged(ok_to_chdir=True) # so we cd back after tmpdir.chdir +def test_embed_dicom_and_nifti_metadata(tmpdir): """Test dcmstack's additional fields""" tmpdir.chdir() # set up testing files dcmfiles = [op.join(TESTS_DATA_PATH, 'axasc35.dcm')] infofile = 'infofile.json' - # 1) nifti does not exist - out = embed_nifti(dcmfiles, 'nifti.nii', 'infofile.json', None, False) - # string -> json - out = json.loads(out) - # should have created nifti file - assert op.exists('nifti.nii') + out_prefix = str(tmpdir / "nifti") + # 1) nifti does not exist -- no longer supported + with pytest.raises(NotImplementedError): + embed_dicom_and_nifti_metadata(dcmfiles, out_prefix + '.nii.gz', infofile, None) + # we should produce nifti using our "standard" ways + nipype_out, prov_file = nipype_convert( + dcmfiles, prefix=out_prefix, with_prov=False, + bids_options=None, tmpdir=str(tmpdir)) + niftifile = nipype_out.outputs.converted_files + + assert op.exists(niftifile) # 2) nifti exists - nifti, info = embed_nifti(dcmfiles, 'nifti.nii', 'infofile.json', None, False) - assert op.exists(nifti) - assert op.exists(info) - with open(info) as fp: + embed_dicom_and_nifti_metadata(dcmfiles, niftifile, infofile, None) + assert op.exists(infofile) + with open(infofile) as fp: out2 = json.load(fp) - assert out == out2 - # 3) with existing metadata bids = {"existing": "data"} - nifti, info = embed_nifti(dcmfiles, 'nifti.nii', 'infofile.json', bids, False) - with open(info) as fp: + embed_dicom_and_nifti_metadata(dcmfiles, niftifile, infofile, bids) + with open(infofile) as fp: out3 = json.load(fp) - assert out3["existing"] - del out3["existing"] - assert out3 == out2 == out + assert out3.pop("existing") == "data" + assert out3 == out2 diff --git a/heudiconv/tests/test_main.py b/heudiconv/tests/test_main.py index 1f618767..91291378 100644 --- a/heudiconv/tests/test_main.py +++ b/heudiconv/tests/test_main.py @@ -65,6 +65,17 @@ def test_populate_bids_templates(tmpdir): # it should also be available as a command os.unlink(str(description_file)) + + # it must fail if no heuristic was provided + with pytest.raises(ValueError) as cme: + runner([ + '--command', 'populate-templates', + '--files', str(tmpdir) + ]) + assert str(cme.value).startswith("Specify heuristic using -f. Known are:") + assert "convertall," in str(cme.value) + assert not description_file.exists() + runner([ '--command', 'populate-templates', '-f', 'convertall', '--files', str(tmpdir) diff --git a/heudiconv/tests/test_regression.py b/heudiconv/tests/test_regression.py index 4f68d055..1b283864 100644 --- a/heudiconv/tests/test_regression.py +++ b/heudiconv/tests/test_regression.py @@ -1,27 +1,27 @@ """Testing conversion with conversion saved on datalad""" -import json from glob import glob +import os import os.path as op import pytest +from heudiconv.cli.run import main as runner +from heudiconv.external.pydicom import dcm +from heudiconv.utils import load_json +# testing utilities +from .utils import fetch_data, gen_heudiconv_args, TESTS_DATA_PATH + have_datalad = True try: - from datalad import api # to pull and grab data from datalad.support.exceptions import IncompleteResultsError except ImportError: have_datalad = False -from heudiconv.cli.run import main as runner -from heudiconv.utils import load_json -# testing utilities -from .utils import fetch_data, gen_heudiconv_args - +@pytest.mark.skipif(not have_datalad, reason="no datalad") @pytest.mark.parametrize('subject', ['sub-sid000143']) @pytest.mark.parametrize('heuristic', ['reproin.py']) @pytest.mark.parametrize('anon_cmd', [None, 'anonymize_script.py']) -@pytest.mark.skipif(not have_datalad, reason="no datalad") def test_conversion(tmpdir, subject, heuristic, anon_cmd): tmpdir.chdir() try: @@ -32,17 +32,17 @@ def test_conversion(tmpdir, subject, heuristic, anon_cmd): pytest.skip("Failed to fetch test data: %s" % str(exc)) outdir = tmpdir.mkdir('out').strpath - args = gen_heudiconv_args(datadir, - outdir, - subject, - heuristic, - anon_cmd, - template=op.join('sourcedata/{subject}/*/*/*.tgz')) - runner(args) # run conversion + args = gen_heudiconv_args( + datadir, outdir, subject, heuristic, anon_cmd, + template=op.join('sourcedata/{subject}/*/*/*.tgz') + ) + runner(args) # run conversion # verify functionals were converted - assert glob('{}/{}/func/*'.format(outdir, subject)) == \ - glob('{}/{}/func/*'.format(datadir, subject)) + assert ( + glob('{}/{}/func/*'.format(outdir, subject)) == + glob('{}/{}/func/*'.format(datadir, subject)) + ) # compare some json metadata json_ = '{}/task-rest_acq-24mm64sl1000tr32te600dyn_bold.json'.format @@ -52,6 +52,7 @@ def test_conversion(tmpdir, subject, heuristic, anon_cmd): for key in keys: assert orig[key] == conv[key] + @pytest.mark.skipif(not have_datalad, reason="no datalad") def test_multiecho(tmpdir, subject='MEEPI', heuristic='bids_ME.py'): tmpdir.chdir() @@ -62,7 +63,7 @@ def test_multiecho(tmpdir, subject='MEEPI', heuristic='bids_ME.py'): outdir = tmpdir.mkdir('out').strpath args = gen_heudiconv_args(datadir, outdir, subject, heuristic) - runner(args) # run conversion + runner(args) # run conversion # check if we have echo functionals echoes = glob(op.join('out', 'sub-' + subject, 'func', '*echo*nii.gz')) @@ -81,3 +82,43 @@ def test_multiecho(tmpdir, subject='MEEPI', heuristic='bids_ME.py'): events = glob(op.join('out', 'sub-' + subject, 'func', '*events.tsv')) for event in events: assert 'echo-' not in event + + +@pytest.mark.parametrize('subject', ['merged']) +def test_grouping(tmpdir, subject): + dicoms = [ + op.join(TESTS_DATA_PATH, fl) for fl in ['axasc35.dcm', 'phantom.dcm'] + ] + # ensure DICOMs are different studies + studyuids = { + dcm.read_file(fl, stop_before_pixels=True).StudyInstanceUID for fl + in dicoms + } + assert len(studyuids) == len(dicoms) + # symlink to common location + outdir = tmpdir.mkdir('out') + datadir = tmpdir.mkdir(subject) + for fl in dicoms: + os.symlink(fl, (datadir / op.basename(fl)).strpath) + + template = op.join("{subject}/*.dcm") + hargs = gen_heudiconv_args( + tmpdir.strpath, + outdir.strpath, + subject, + 'convertall.py', + template=template + ) + + with pytest.raises(AssertionError): + runner(hargs) + + # group all found DICOMs under subject, despite conflicts + hargs += ["-g", "all"] + runner(hargs) + assert len([fl for fl in outdir.visit(fil='run0*')]) == 4 + tsv = (outdir / 'participants.tsv') + assert tsv.check() + lines = tsv.open().readlines() + assert len(lines) == 2 + assert lines[1].split('\t')[0] == 'sub-{}'.format(subject) diff --git a/heudiconv/tests/utils.py b/heudiconv/tests/utils.py index cb8cdbdb..fdc0b4be 100644 --- a/heudiconv/tests/utils.py +++ b/heudiconv/tests/utils.py @@ -1,9 +1,17 @@ +from functools import wraps +import os import os.path as op +import sys + import heudiconv.heuristics + HEURISTICS_PATH = op.join(heudiconv.heuristics.__path__[0]) TESTS_DATA_PATH = op.join(op.dirname(__file__), 'data') +import logging +lgr = logging.getLogger(__name__) + def gen_heudiconv_args(datadir, outdir, subject, heuristic_file, anon_cmd=None, template=None, xargs=None): @@ -58,3 +66,51 @@ def fetch_data(tmpdir, dataset, getpath=None): getdir = targetdir + (op.sep + getpath if getpath is not None else '') ds.get(getdir) return targetdir + + +def assert_cwd_unchanged(ok_to_chdir=False): + """Decorator to test whether the current working directory remains unchanged + + Provenance: based on the one in datalad, but simplified. + + Parameters + ---------- + ok_to_chdir: bool, optional + If True, allow to chdir, so this decorator would not then raise exception + if chdir'ed but only return to original directory + """ + + def decorator(func=None): # =None to avoid pytest treating it as a fixture + @wraps(func) + def newfunc(*args, **kwargs): + cwd_before = os.getcwd() + exc = None + try: + return func(*args, **kwargs) + except Exception as exc_: + exc = exc_ + finally: + try: + cwd_after = os.getcwd() + except OSError as e: + lgr.warning("Failed to getcwd: %s" % e) + cwd_after = None + + if cwd_after != cwd_before: + os.chdir(cwd_before) + if not ok_to_chdir: + lgr.warning( + "%s changed cwd to %s. Mitigating and changing back to %s" + % (func, cwd_after, cwd_before)) + # If there was already exception raised, we better reraise + # that one since it must be more important, so not masking it + # here with our assertion + if exc is None: + assert cwd_before == cwd_after, \ + "CWD changed from %s to %s" % (cwd_before, cwd_after) + + if exc is not None: + raise exc + return newfunc + + return decorator diff --git a/heudiconv/utils.py b/heudiconv/utils.py index 59a57a91..da3ebc56 100644 --- a/heudiconv/utils.py +++ b/heudiconv/utils.py @@ -27,8 +27,8 @@ 'example_dcm_file', # 1 'series_id', # 2 'dcm_dir_name', # 3 - 'unspecified2', # 4 - 'unspecified3', # 5 + 'series_files', # 4 + 'unspecified', # 5 'dim1', 'dim2', 'dim3', 'dim4', # 6, 7, 8, 9 'TR', 'TE', # 10, 11 'protocol_name', # 12 @@ -44,7 +44,7 @@ 'patient_age', # 22 'patient_sex', # 23 'date', # 24 - 'series_uid' # 25 + 'series_uid', # 25 ] SeqInfo = namedtuple('SeqInfo', seqinfo_fields) @@ -188,7 +188,7 @@ def assure_no_file_exists(path): os.unlink(path) -def save_json(filename, data, indent=4, sort_keys=True, pretty=False): +def save_json(filename, data, indent=2, sort_keys=True, pretty=False): """Save data to a json file Parameters @@ -203,11 +203,25 @@ def save_json(filename, data, indent=4, sort_keys=True, pretty=False): """ assure_no_file_exists(filename) + dumps_kw = dict(sort_keys=sort_keys, indent=indent) + j = None + if pretty: + try: + j = json_dumps_pretty(data, **dumps_kw) + except AssertionError as exc: + pretty = False + lgr.warning( + "Prettyfication of .json failed (%s). " + "Original .json will be kept as is. Please share (if you " + "could) " + "that file (%s) with HeuDiConv developers" + % (str(exc), filename) + ) + if not pretty: + j = _canonical_dumps(data, **dumps_kw) + assert j is not None # one way or another it should have been set to a str with open(filename, 'w') as fp: - fp.write( - (json_dumps_pretty if pretty else _canonical_dumps)( - data, sort_keys=sort_keys, indent=indent) - ) + fp.write(j) def json_dumps_pretty(j, indent=2, sort_keys=True): @@ -252,25 +266,9 @@ def json_dumps_pretty(j, indent=2, sort_keys=True): def treat_infofile(filename): """Tune up generated .json file (slim down, pretty-print for humans). """ - with open(filename) as f: - j = json.load(f) - + j = load_json(filename) j_slim = slim_down_info(j) - dumps_kw = dict(indent=2, sort_keys=True) - try: - j_pretty = json_dumps_pretty(j_slim, **dumps_kw) - except AssertionError as exc: - lgr.warning( - "Prettyfication of .json failed (%s). " - "Original .json will be kept as is. Please share (if you could) " - "that file (%s) with HeuDiConv developers" - % (str(exc), filename) - ) - j_pretty = json.dumps(j_slim, **dumps_kw) - - set_readonly(filename, False) - with open(filename, 'wt') as fp: - fp.write(j_pretty) + save_json(filename, j_slim, sort_keys=True, pretty=True) set_readonly(filename) @@ -319,7 +317,7 @@ def load_heuristic(heuristic): path, fname = op.split(heuristic_file) try: old_syspath = sys.path[:] - sys.path.append(path) + sys.path.insert(0, path) mod = __import__(fname.split('.')[0]) mod.filename = heuristic_file finally: @@ -488,3 +486,22 @@ def create_tree(path, tree, archives_leading_dir=True): f.write(load) if executable: os.chmod(full_name, os.stat(full_name).st_mode | stat.S_IEXEC) + + +def get_typed_attr(obj, attr, _type, default=None): + """ + Typecasts an object's named attribute. If the attribute cannot be + converted, the default value is returned instead. + + Parameters + ---------- + obj: Object + attr: Attribute + _type: Type + default: value, optional + """ + try: + val = _type(getattr(obj, attr, default)) + except (TypeError, ValueError): + return default + return val diff --git a/setup.py b/setup.py index c3a804a1..686d71d9 100755 --- a/setup.py +++ b/setup.py @@ -19,9 +19,7 @@ def main(): # Get version and release info, which is all stored in heudiconv/info.py info_file = op.join(thispath, 'heudiconv', 'info.py') with open(info_file) as infofile: - # exec(infofile.read(), globals(), ldict) - # Workaround for python 2.7 prior 2.7.8 - eval(compile(infofile.read(), '', 'exec'), globals(), ldict) + exec(infofile.read(), globals(), ldict) def findsome(subdir, extensions): diff --git a/utils/prep_release b/utils/prep_release new file mode 100755 index 00000000..6a0823c6 --- /dev/null +++ b/utils/prep_release @@ -0,0 +1,14 @@ +#!/bin/bash + +set -eu + +read -r newver oldver <<<$(sed -ne 's,## \[\([0-9\.]*\)\] .*,\1,gp' CHANGELOG.md | head -n 2 | tr '\n' ' ') + +echo "Old: $oldver New: $newver" +curver=$(python -c 'import heudiconv; print(heudiconv.__version__)') +# check +test "$oldver" = "$curver" + +sed -i -e "s,${oldver//./\\.},$newver,g" \ + docs/conf.py docs/installation.rst docs/usage.rst heudiconv/info.py + From 756fa0a5865b442ce1849b893099a9721cc06b0c Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Fri, 8 May 2020 13:26:30 -0400 Subject: [PATCH 11/11] Address @yarikoptic's review. - Some refactoring of name-updaters. - Add tests for name-updaters. --- heudiconv/convert.py | 191 ++++++++++++++++++-------------- heudiconv/tests/test_convert.py | 78 +++++++++++++ 2 files changed, 183 insertions(+), 86 deletions(-) create mode 100644 heudiconv/tests/test_convert.py diff --git a/heudiconv/convert.py b/heudiconv/convert.py index f6efc06d..8e2c3c60 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -233,65 +233,91 @@ def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid, getattr(heuristic, 'DEFAULT_FIELDS', {})) -def update_complex_name(metadata, this_prefix_basename, suffix): +def update_complex_name(metadata, filename, suffix): """ - Insert `_part-` entity into filename if data are from a sequence - with magnitude/phase part. + Insert `_rec-` entity into filename if data are from a + sequence with magnitude/phase part. + + Parameters + ---------- + metadata : dict + Scan metadata dictionary from BIDS sidecar file. + filename : str + Incoming filename + suffix : str + An index used for cases where a single scan produces multiple files, + but the differences between those files are unknown. + + Returns + ------- + filename : str + Updated filename with rec entity added in appropriate position. """ - # Functional scans separate magnitude/phase differently - unsupported_types = ['_bold', '_phase'] - if any(ut in this_prefix_basename for ut in unsupported_types): - return this_prefix_basename + # Some scans separate magnitude/phase differently + unsupported_types = ['_bold', '_phase', + '_magnitude', '_magnitude1', '_magnitude2', + '_phasediff', '_phase1', '_phase2'] + if any(ut in filename for ut in unsupported_types): + return filename # Check to see if it is magnitude or phase part: if 'M' in metadata.get('ImageType'): - mag_or_phase = 'mag' + mag_or_phase = 'magnitude' elif 'P' in metadata.get('ImageType'): mag_or_phase = 'phase' else: mag_or_phase = suffix # Determine scan suffix - filetype = '_' + this_prefix_basename.split('_')[-1] + filetype = '_' + filename.split('_')[-1] - # Insert part label - if not ('_part-%s' % mag_or_phase) in this_prefix_basename: - # If "_part-" is specified, prepend the 'mag_or_phase' value. - if '_part-' in this_prefix_basename: + # Insert rec label + if not ('_rec-%s' % mag_or_phase) in filename: + # If "_rec-" is specified, prepend the 'mag_or_phase' value. + if '_rec-' in filename: raise BIDSError( - "Part label for images will be automatically set, remove " - "from heuristic" + "Reconstruction label for images will be automatically set, " + "remove from heuristic" ) - # If not, insert "_part-" + 'mag_or_phase' into the prefix_basename - # **before** "_run", "_echo" or "_sbref", whichever appears first: + # Insert it **before** the following string(s), whichever appears first. for label in ['_dir', '_run', '_mod', '_echo', '_recording', '_proc', '_space', filetype]: - if label == filetype: - this_prefix_basename = this_prefix_basename.replace( - filetype, "_part-%s%s" % (mag_or_phase, filetype) - ) - break - elif (label in this_prefix_basename): - this_prefix_basename = this_prefix_basename.replace( - label, "_part-%s%s" % (mag_or_phase, label) + if (label == filetype) or (label in filename): + filename = filename.replace( + label, "_rec-%s%s" % (mag_or_phase, label) ) break - return this_prefix_basename + return filename -def update_multiecho_name(metadata, this_prefix_basename, echo_times): +def update_multiecho_name(metadata, filename, echo_times): """ Insert `_echo-` entity into filename if data are from a multi-echo sequence. + + Parameters + ---------- + metadata : dict + Scan metadata dictionary from BIDS sidecar file. + filename : str + Incoming filename + echo_times : list + List of all echo times from scan. Used to determine the echo *number* + (i.e., index) if field is missing from metadata. + + Returns + ------- + filename : str + Updated filename with echo entity added, if appropriate. """ # Field maps separate echoes differently unsupported_types = [ '_magnitude', '_magnitude1', '_magnitude2', '_phasediff', '_phase1', '_phase2', '_fieldmap' ] - if any(ut in this_prefix_basename for ut in unsupported_types): - return this_prefix_basename + if any(ut in filename for ut in unsupported_types): + return filename # Get the EchoNumber from json file info. If not present, use EchoTime if 'EchoNumber' in metadata.keys(): @@ -300,57 +326,62 @@ def update_multiecho_name(metadata, this_prefix_basename, echo_times): echo_number = echo_times.index(metadata['EchoTime']) + 1 # Determine scan suffix - filetype = '_' + this_prefix_basename.split('_')[-1] + filetype = '_' + filename.split('_')[-1] # Insert it **before** the following string(s), whichever appears first. for label in ['_recording', '_proc', '_space', filetype]: - if label == filetype: - this_prefix_basename = this_prefix_basename.replace( - filetype, "_echo-%s%s" % (echo_number, filetype) - ) - break - elif (label in this_prefix_basename): - this_prefix_basename = this_prefix_basename.replace( + if (label == filetype) or (label in filename): + filename = filename.replace( label, "_echo-%s%s" % (echo_number, label) ) break - return this_prefix_basename + return filename -def update_uncombined_name(metadata, this_prefix_basename, channel_names): +def update_uncombined_name(metadata, filename, channel_names): """ Insert `_ch-` entity into filename if data are from a sequence with "save uncombined". + + Parameters + ---------- + metadata : dict + Scan metadata dictionary from BIDS sidecar file. + filename : str + Incoming filename + channel_names : list + List of all channel names from scan. Used to determine the channel + *number* (i.e., index) if field is missing from metadata. + + Returns + ------- + filename : str + Updated filename with ch entity added, if appropriate. """ # In case any scan types separate channels differently unsupported_types = [] - if any(ut in this_prefix_basename for ut in unsupported_types): - return this_prefix_basename + if any(ut in filename for ut in unsupported_types): + return filename # Determine the channel number channel_number = ''.join([c for c in metadata['CoilString'] if c.isdigit()]) if not channel_number: channel_number = channel_names.index(metadata['CoilString']) + 1 - channel_number = channel_number.zfill(2) + channel_number = str(channel_number).zfill(2) # Determine scan suffix - filetype = '_' + this_prefix_basename.split('_')[-1] + filetype = '_' + filename.split('_')[-1] # Insert it **before** the following string(s), whichever appears first. # Choosing to put channel near the end since it's not in the specification yet. for label in ['_recording', '_proc', '_space', filetype]: - if label == filetype: - this_prefix_basename = this_prefix_basename.replace( - filetype, "_ch-%s%s" % (channel_number, filetype) - ) - break - elif (label in this_prefix_basename): - this_prefix_basename = this_prefix_basename.replace( + if (label == filetype) or (label in filename): + filename = filename.replace( label, "_ch-%s%s" % (channel_number, label) ) break - return this_prefix_basename + return filename def convert(items, converter, scaninfo_suffix, custom_callable, with_prov, @@ -655,27 +686,16 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # echo times for all bids_files and see if they are all the same or not. # Collect some metadata across all images - echo_times, channel_names, image_types = [], [], [] - for b in bids_files: - if not b: + echo_times, channel_names, image_types = set(), set(), set() + for metadata in bids_metas: + if not metadata: continue - metadata = load_json(b) - echo_times.append(metadata.get('EchoTime', None)) - channel_names.append(metadata.get('CoilString', None)) - image_types.append(metadata.get('ImageType', None)) - echo_times = [v for v in echo_times if v] - echo_times = sorted(list(set(echo_times))) - channel_names = [v for v in channel_names if v] - channel_names = sorted(list(set(channel_names))) - image_types = [v for v in image_types if v] - - is_multiecho = len(echo_times) > 1 # Check for varying echo times - is_uncombined = len(channel_names) > 1 # Check for uncombined data - - # Determine if data are complex (magnitude + phase) - magnitude_found = any(['M' in it for it in image_types]) - phase_found = any(['P' in it for it in image_types]) - is_complex = magnitude_found and phase_found + echo_times.add(metadata.get('EchoTime', nan)) + channel_names.add(metadata.get('CoilString', nan)) + image_types.update(metadata.get('ImageType', [nan])) + is_multiecho = len(set(filter(bool, echo_times))) > 1 # Check for varying echo times + is_uncombined = len(set(filter(bool, channel_names))) > 1 # Check for uncombined data + is_complex = 'M' in image_types and 'P' in image_types # Determine if data are complex (magnitude + phase) ### Loop through the bids_files, set the output name and save files for fl, suffix, bids_file, bids_meta in zip(res_files, suffixes, bids_files, bids_metas): @@ -686,23 +706,22 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # and we don't want to modify it for all the bids_files): this_prefix_basename = prefix_basename - # Update name if multi-echo - if bids_file and is_multiecho: - this_prefix_basename = update_multiecho_name( - bids_meta, this_prefix_basename, echo_times - ) + # Update name for certain criteria + if bids_file: + if is_multiecho: + this_prefix_basename = update_multiecho_name( + bids_meta, this_prefix_basename, echo_times + ) - # Update name if complex data - if bids_file and is_complex: - this_prefix_basename = update_complex_name( - bids_meta, this_prefix_basename, suffix - ) + if is_complex: + this_prefix_basename = update_complex_name( + bids_meta, this_prefix_basename, suffix + ) - # Update name if uncombined (channel-level) data - if bids_file and is_uncombined: - this_prefix_basename = update_uncombined_name( - bids_meta, this_prefix_basename, channel_names - ) + if is_uncombined: + this_prefix_basename = update_uncombined_name( + bids_meta, this_prefix_basename, channel_names + ) # Fallback option: # If we have failed to modify this_prefix_basename, because it didn't fall diff --git a/heudiconv/tests/test_convert.py b/heudiconv/tests/test_convert.py new file mode 100644 index 00000000..7593cb05 --- /dev/null +++ b/heudiconv/tests/test_convert.py @@ -0,0 +1,78 @@ +"""Test functions in heudiconv.convert module. +""" +import pytest + +from heudiconv.convert import (update_complex_name, + update_multiecho_name, + update_uncombined_name) +from heudiconv.bids import BIDSError + + +def test_update_complex_name(): + """Unit testing for heudiconv.convert.update_complex_name(), which updates + filenames with the rec field if appropriate. + """ + # Standard name update + fn = 'sub-X_ses-Y_task-Z_run-01_sbref' + metadata = {'ImageType': ['ORIGINAL', 'PRIMARY', 'P', 'MB', 'TE3', 'ND', 'MOSAIC']} + suffix = 3 + out_fn_true = 'sub-X_ses-Y_task-Z_rec-phase_run-01_sbref' + out_fn_test = update_complex_name(metadata, fn, suffix) + assert out_fn_test == out_fn_true + # Catch an unsupported type and *do not* update + fn = 'sub-X_ses-Y_task-Z_run-01_phase' + out_fn_test = update_complex_name(metadata, fn, suffix) + assert out_fn_test == fn + # Data type is missing from metadata so use suffix + fn = 'sub-X_ses-Y_task-Z_run-01_sbref' + metadata = {'ImageType': ['ORIGINAL', 'PRIMARY', 'MB', 'TE3', 'ND', 'MOSAIC']} + out_fn_true = 'sub-X_ses-Y_task-Z_rec-3_run-01_sbref' + out_fn_test = update_complex_name(metadata, fn, suffix) + assert out_fn_test == out_fn_true + # Catch existing field with value that *does not match* metadata + # and raise Exception + fn = 'sub-X_ses-Y_task-Z_rec-magnitude_run-01_sbref' + metadata = {'ImageType': ['ORIGINAL', 'PRIMARY', 'P', 'MB', 'TE3', 'ND', 'MOSAIC']} + suffix = 3 + with pytest.raises(BIDSError): + assert update_complex_name(metadata, fn, suffix) + + +def test_update_multiecho_name(): + """Unit testing for heudiconv.convert.update_multiecho_name(), which updates + filenames with the echo field if appropriate. + """ + # Standard name update + fn = 'sub-X_ses-Y_task-Z_run-01_bold' + metadata = {'EchoTime': 0.01, + 'EchoNumber': 1} + echo_times = [0.01, 0.02, 0.03] + out_fn_true = 'sub-X_ses-Y_task-Z_run-01_echo-1_bold' + out_fn_test = update_multiecho_name(metadata, fn, echo_times) + assert out_fn_test == out_fn_true + # EchoNumber field is missing from metadata, so use echo_times + metadata = {'EchoTime': 0.01} + out_fn_test = update_multiecho_name(metadata, fn, echo_times) + assert out_fn_test == out_fn_true + # Catch an unsupported type and *do not* update + fn = 'sub-X_ses-Y_task-Z_run-01_phasediff' + out_fn_test = update_multiecho_name(metadata, fn, echo_times) + assert out_fn_test == fn + + +def test_update_uncombined_name(): + """Unit testing for heudiconv.convert.update_uncombined_name(), which updates + filenames with the ch field if appropriate. + """ + # Standard name update + fn = 'sub-X_ses-Y_task-Z_run-01_bold' + metadata = {'CoilString': 'H1'} + channel_names = ['H1', 'H2', 'H3', 'HEA;HEP'] + out_fn_true = 'sub-X_ses-Y_task-Z_run-01_ch-01_bold' + out_fn_test = update_uncombined_name(metadata, fn, channel_names) + assert out_fn_test == out_fn_true + # CoilString field has no number in it + metadata = {'CoilString': 'HEA;HEP'} + out_fn_true = 'sub-X_ses-Y_task-Z_run-01_ch-04_bold' + out_fn_test = update_uncombined_name(metadata, fn, channel_names) + assert out_fn_test == out_fn_true