From d076a07ceb85922f60156735790c6b8d791cadc1 Mon Sep 17 00:00:00 2001 From: Lea Waller Date: Tue, 27 Oct 2020 17:19:54 +0100 Subject: [PATCH 001/176] ENH: Support parsing DICOM datasets from zip files --- heudiconv/parser.py | 55 +++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/heudiconv/parser.py b/heudiconv/parser.py index 0c590319..620d2efd 100644 --- a/heudiconv/parser.py +++ b/heudiconv/parser.py @@ -7,6 +7,7 @@ from collections import defaultdict import tarfile +import zipfile from tempfile import mkdtemp from .dicoms import group_dicoms_into_seqinfos @@ -76,24 +77,44 @@ def get_extracted_dicoms(fl): # needs sorting to keep the generated "session" label deterministic for i, t in enumerate(sorted(fl)): # "classical" heudiconv has that heuristic to handle multiple - # tarballs as providing different sessions per each tarball - if not tarfile.is_tarfile(t): + # tarballs as providing different sessions per each tarball or zipfile + + if tarfile.is_tarfile(t): + # cannot use TempDirs since will trigger cleanup with __del__ + tmpdir = mkdtemp(prefix='heudiconvDCM') + # load file + tf = tarfile.open(t) + # check content and sanitize permission bits + tmembers = tf.getmembers() + for tm in tmembers: + tm.mode = 0o700 + # get all files, assemble full path in tmp dir + tf_content = [m.name for m in tmembers if m.isfile()] + # store full paths to each file, so we don't need to drag along + # tmpdir as some basedir + sessions[session] = [op.join(tmpdir, f) for f in tf_content] + session += 1 + # extract into tmp dir + tf.extractall(path=tmpdir, members=tmembers) + + elif zipfile.is_zipfile(t): + # cannot use TempDirs since will trigger cleanup with __del__ + tmpdir = mkdtemp(prefix='heudiconvDCM') + # load file + zf = zipfile.ZipFile(t) + # check content + zmembers = zf.infolist() + # get all files, assemble full path in tmp dir + zf_content = [m.filename for m in zmembers if not m.is_dir()] + # store full paths to each file, so we don't need to drag along + # tmpdir as some basedir + sessions[session] = [op.join(tmpdir, f) for f in zf_content] + session += 1 + # extract into tmp dir + zf.extractall(path=tmpdir) + + else: sessions[None].append(t) - continue - - tf = tarfile.open(t) - # check content and sanitize permission bits - tmembers = tf.getmembers() - for tm in tmembers: - tm.mode = 0o700 - # get all files, assemble full path in tmp dir - tf_content = [m.name for m in tmembers if m.isfile()] - # store full paths to each file, so we don't need to drag along - # tmpdir as some basedir - sessions[session] = [op.join(tmpdir, f) for f in tf_content] - session += 1 - # extract into tmp dir - tf.extractall(path=tmpdir, members=tmembers) if session == 1: # we had only 1 session, so no really multiple sessions according From 34242f32863852c92d266151c3e630d9c9b8336b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 2 Jun 2022 13:38:09 -0400 Subject: [PATCH 002/176] DOC+tinyRF: reproin - use BIDS terminology BIDS, in particular in its schema, formalized terms to be "datatype" and "suffix", so let"s use those in reproin heuristic documentation. --- heudiconv/heuristics/reproin.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index a8de120d..ea7d29d5 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -28,7 +28,7 @@ Sequence names on the scanner must follow this specification to avoid manual conversion/handling: - [PREFIX:][WIP ][_ses-][_task-][_acq-][_run-][_dir-][][__] + [PREFIX:][WIP ]]>[_ses-][_task-][_acq-][_run-][_dir-][][__] where [PREFIX:] - leading capital letters followed by : are stripped/ignored @@ -42,17 +42,19 @@ descriptive ones for e.g. SESID (_ses-movie, _ses-localizer) - - a known BIDS sequence type which is usually a name of the folder under - subject's directory. And (optional) label is specific per sequence type - (e.g. typical "bold" for func, or "T1w" for "anat"), which could often - (but not always) be deduced from DICOM. Known to BIDS modalities are: + + a known BIDS sequence datatype which is usually a name of the folder under + subject's directory. And (optional) suffix is a specific sequence type + (e.g., "bold" for func, or "T1w" for "anat"), which could often + (but not always) be deduced from DICOM. Known to ReproIn BIDS modalities + are: anat - anatomical data. Might also be collected multiple times across runs (e.g. if subject is taken out of magnet etc), so could (optionally) have "_run" definition attached. For "standard anat" - labels, please consult to "8.3 Anatomy imaging data" but most - common are 'T1w', 'T2w', 'angio' + suffixes, please consult to "8.3 Anatomy imaging data" but most + common are 'T1w', 'T2w', 'angio'. + beh - behavioral data. known but not "treated". func - functional (AKA task, including resting state) data. Typically contains multiple runs, and might have multiple different tasks different per each run @@ -60,6 +62,13 @@ fmap - field maps dwi - diffusion weighted imaging (also can as well have runs) + The other BIDS modalities are not known ATM and their data will not be + converted and will be just skipped (with a warning). Full list of datatypes + can be found at + https://github.com/bids-standard/bids-specification/blob/v1.7.0/src/schema/objects/datatypes.yaml + and their corresponding suffixes at + https://github.com/bids-standard/bids-specification/tree/v1.7.0/src/schema/rules/datatypes + _ses- (optional) a session. Having a single sequence within a study would make that study follow "multi-session" layout. A common practice to have a _ses specifier From 364551b86f051a5963827f70305ee993f385c7e9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 2 Jun 2022 13:38:32 -0400 Subject: [PATCH 003/176] tinyRF: reproin - define a set of known BIDS datatypes to check against --- heudiconv/heuristics/reproin.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index ea7d29d5..1af00404 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -213,6 +213,10 @@ 'criterion': 'Closest' } + +KNOWN_DATATYPES = {'anat', 'func', 'dwi', 'behav', 'fmap'} + + def _delete_chars(from_str, deletechars): """ Delete characters from string allowing for Python 2 / 3 difference """ @@ -859,11 +863,11 @@ def split2(s): # Let's analyze first element which should tell us sequence type seqtype, seqtype_label = split2(split[0]) - if seqtype not in {'anat', 'func', 'dwi', 'behav', 'fmap'}: + if seqtype not in KNOWN_DATATYPES: # It is not something we don't consume if bids: - lgr.warning("It was instructed to be BIDS sequence but unknown " - "type %s found", seqtype) + lgr.warning("It was instructed to be BIDS datatype but unknown " + "%s found. Known are: %s", seqtype, ', '.join(KNOWN_DATATYPES)) return {} regd = dict(seqtype=seqtype) From d011b6c2dfce2df7c1d10bb68b4437d0b042fb08 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 2 Jun 2022 13:55:38 -0400 Subject: [PATCH 004/176] RF: reproin - used datatype and suffix terms in reproin code Also added few extra comments / pointers. There should be no behavior change, unless someone used parts of reproin heuristic and relied on those returned by helper parse_series_spec data structures --- heudiconv/heuristics/reproin.py | 81 +++++++++++++++++---------------- 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index 1af00404..a4e1b29f 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -417,9 +417,9 @@ def infotodict(seqinfo): # 1 - PRIMARY/SECONDARY # 3 - Image IOD specific specialization (optional) dcm_image_iod_spec = s.image_type[2] - image_type_seqtype = { + image_type_datatype = { # Note: P and M are too generic to make a decision here, could be - # for different seqtypes (bold, fmap, etc) + # for different datatypes (bold, fmap, etc) 'FMRI': 'func', 'MPR': 'anat', 'DIFFUSION': 'dwi', @@ -428,7 +428,7 @@ def infotodict(seqinfo): 'MIP_TRA': 'anat', # angiography }.get(dcm_image_iod_spec, None) else: - dcm_image_iod_spec = image_type_seqtype = None + dcm_image_iod_spec = image_type_datatype = None series_info = {} # For please lintian and its friends for sfield in series_spec_fields: @@ -453,19 +453,19 @@ def infotodict(seqinfo): if dcm_image_iod_spec and dcm_image_iod_spec.startswith('MIP'): series_info['acq'] = series_info.get('acq', '') + sanitize_str(dcm_image_iod_spec) - seqtype = series_info.pop('seqtype') - seqtype_label = series_info.pop('seqtype_label', None) + datatype = series_info.pop('datatype') + datatype_suffix = series_info.pop('datatype_suffix', None) - if image_type_seqtype and seqtype != image_type_seqtype: + if image_type_datatype and datatype != image_type_datatype: lgr.warning( - "Deduced seqtype to be %s from DICOM, but got %s out of %s", - image_type_seqtype, seqtype, series_spec) + "Deduced datatype to be %s from DICOM, but got %s out of %s", + image_type_datatype, datatype, series_spec) # if s.is_derived: # # Let's for now stash those close to original images # # TODO: we might want a separate tree for all of this!? # # so more of a parameter to the create_key - # #seqtype += '/derivative' + # #datatype += '/derivative' # # just keep it lower case and without special characters # # XXXX what for??? # #seq.append(s.series_description.lower()) @@ -475,26 +475,26 @@ def infotodict(seqinfo): prefix = '' # - # Figure out the seqtype_label (BIDS _suffix) + # Figure out the datatype_suffix (BIDS _suffix) # # If none was provided -- let's deduce it from the information we find: # analyze s.protocol_name (series_id is based on it) for full name mapping etc - if not seqtype_label: - if seqtype == 'func': + if not datatype_suffix: + if datatype == 'func': if '_pace_' in series_spec: - seqtype_label = 'pace' # or should it be part of seq- + datatype_suffix = 'pace' # or should it be part of seq- elif 'P' in s.image_type: - seqtype_label = 'phase' + datatype_suffix = 'phase' elif 'M' in s.image_type: - seqtype_label = 'bold' + datatype_suffix = 'bold' else: # assume bold by default - seqtype_label = 'bold' - elif seqtype == 'fmap': + datatype_suffix = 'bold' + elif datatype == 'fmap': # TODO: support phase1 phase2 like in "Case 2: Two phase images ..." if not dcm_image_iod_spec: raise ValueError("Do not know image data type yet to make decision") - seqtype_label = { + datatype_suffix = { # might want explicit {file_index} ? # _epi for pepolar fieldmaps, see # https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/01-magnetic-resonance-imaging-data.html#case-4-multiple-phase-encoded-directions-pepolar @@ -502,19 +502,19 @@ def infotodict(seqinfo): 'P': 'phasediff', 'DIFFUSION': 'epi', # according to KODI those DWI are the EPIs we need }[dcm_image_iod_spec] - elif seqtype == 'dwi': + elif datatype == 'dwi': # label for dwi as well - seqtype_label = 'dwi' + datatype_suffix = 'dwi' # - # Even if seqtype_label was provided, for some data we might need to override, + # Even if datatype_suffix was provided, for some data we might need to override, # since they are complementary files produced along-side with original # ones. # if s.series_description.endswith('_SBRef'): - seqtype_label = 'sbref' + datatype_suffix = 'sbref' - if not seqtype_label: + if not datatype_suffix: # Might be provided by the bids ending within series_spec, we would # just want to check if that the last element is not _key-value pair bids_ending = series_info.get('bids', None) @@ -584,7 +584,12 @@ def from_series_info(name): else: return None - suffix_parts = [ + # TODO: get order from schema, do not hardcode. ATM could be checked at + # https://bids-specification.readthedocs.io/en/stable/99-appendices/04-entity-table.html + # https://github.com/bids-standard/bids-specification/blob/HEAD/src/schema/rules/entities.yaml + # ATM we at large rely on possible (re)ordering according to schema to be done + # by heudiconv, not reproin here. + filename_suffix_parts = [ from_series_info('task'), from_series_info('acq'), # But we want to add an indicator in case it was motion corrected @@ -593,10 +598,10 @@ def from_series_info(name): from_series_info('dir'), series_info.get('bids'), run_label, - seqtype_label, + datatype_suffix, ] # filter those which are None, and join with _ - suffix = '_'.join(filter(bool, suffix_parts)) + suffix = '_'.join(filter(bool, filename_suffix_parts)) # # .series_description in case of # sdesc = s.study_description @@ -615,12 +620,12 @@ def from_series_info(name): # For scouts -- we want only dicoms # https://github.com/nipy/heudiconv/issues/145 if "_Scout" in s.series_description or \ - (seqtype == 'anat' and seqtype_label and seqtype_label.startswith('scout')): + (datatype == 'anat' and datatype_suffix and datatype_suffix.startswith('scout')): outtype = ('dicom',) else: outtype = ('nii.gz', 'dicom') - template = create_key(seqtype, suffix, prefix=prefix, outtype=outtype) + template = create_key(datatype, suffix, prefix=prefix, outtype=outtype) # we wanted ordered dict for consistent demarcation of dups if template not in info: info[template] = [] @@ -862,17 +867,17 @@ def split2(s): return s, None # Let's analyze first element which should tell us sequence type - seqtype, seqtype_label = split2(split[0]) - if seqtype not in KNOWN_DATATYPES: + datatype, datatype_suffix = split2(split[0]) + if datatype not in KNOWN_DATATYPES: # It is not something we don't consume if bids: lgr.warning("It was instructed to be BIDS datatype but unknown " - "%s found. Known are: %s", seqtype, ', '.join(KNOWN_DATATYPES)) + "%s found. Known are: %s", datatype, ', '.join(KNOWN_DATATYPES)) return {} - regd = dict(seqtype=seqtype) - if seqtype_label: - regd['seqtype_label'] = seqtype_label + regd = dict(datatype=datatype) + if datatype_suffix: + regd['datatype_suffix'] = datatype_suffix # now go through each to see if one which we care bids_leftovers = [] for s in split[1:]: @@ -899,12 +904,12 @@ def split2(s): # TODO: might want to check for all known "standard" BIDS suffixes here # among bids_leftovers, thus serve some kind of BIDS validator - # if not regd.get('seqtype_label', None): - # # might need to assign a default label for each seqtype if was not + # if not regd.get('datatype_suffix', None): + # # might need to assign a default label for each datatype if was not # # given - # regd['seqtype_label'] = { + # regd['datatype_suffix'] = { # 'func': 'bold' - # }.get(regd['seqtype'], None) + # }.get(regd['datatype'], None) return regd From 2305f5f509bcefe9a46101b769d9b6dfa470e78e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 2 Jun 2022 14:09:41 -0400 Subject: [PATCH 005/176] [DATALAD RUNCMD] Also adjust names in the tests === Do not change lines below === { "chain": [], "cmd": "sed -i -e 's,seqtype_label,datatype_suffix,g' -e 's,seqtype,datatype,g' {inputs}", "exit": 0, "extra_inputs": [], "inputs": [ "heudiconv/heuristics/*reproin.py" ], "outputs": [ "heudiconv/heuristics/*reproin.py" ], "pwd": "." } ^^^ Do not change lines above ^^^ --- heudiconv/heuristics/test_reproin.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/heudiconv/heuristics/test_reproin.py b/heudiconv/heuristics/test_reproin.py index d8777f21..4b878f91 100644 --- a/heudiconv/heuristics/test_reproin.py +++ b/heudiconv/heuristics/test_reproin.py @@ -169,7 +169,7 @@ def test_parse_series_spec(): assert pdpn("bids_func-bold") == \ pdpn("func-bold") == \ - {'seqtype': 'func', 'seqtype_label': 'bold'} + {'datatype': 'func', 'datatype_suffix': 'bold'} # pdpn("bids_func_ses+_task-boo_run+") == \ # order and PREFIX: should not matter, as well as trailing spaces @@ -179,8 +179,8 @@ def test_parse_series_spec(): pdpn("WIP func_ses+_task-boo_run+") == \ pdpn("bids_func_ses+_run+_task-boo") == \ { - 'seqtype': 'func', - # 'seqtype_label': 'bold', + 'datatype': 'func', + # 'datatype_suffix': 'bold', 'session': '+', 'run': '+', 'task': 'boo', @@ -191,7 +191,7 @@ def test_parse_series_spec(): pdpn("bids_func-pace_ses-1_run-2_task-boo_acq-bu_bids-please__therest") == \ pdpn("func-pace_ses-1_task-boo_acq-bu_bids-please_run-2") == \ { - 'seqtype': 'func', 'seqtype_label': 'pace', + 'datatype': 'func', 'datatype_suffix': 'pace', 'session': '1', 'run': '2', 'task': 'boo', @@ -201,24 +201,24 @@ def test_parse_series_spec(): assert pdpn("bids_anat-scout_ses+") == \ { - 'seqtype': 'anat', - 'seqtype_label': 'scout', + 'datatype': 'anat', + 'datatype_suffix': 'scout', 'session': '+', } assert pdpn("anat_T1w_acq-MPRAGE_run+") == \ { - 'seqtype': 'anat', + 'datatype': 'anat', 'run': '+', 'acq': 'MPRAGE', - 'seqtype_label': 'T1w' + 'datatype_suffix': 'T1w' } # Check for currently used {date}, which should also should get adjusted # from (date) since Philips does not allow for {} assert pdpn("func_ses-{date}") == \ pdpn("func_ses-(date)") == \ - {'seqtype': 'func', 'session': '{date}'} + {'datatype': 'func', 'session': '{date}'} assert pdpn("fmap_dir-AP_ses-01") == \ - {'seqtype': 'fmap', 'session': '01', 'dir': 'AP'} \ No newline at end of file + {'datatype': 'fmap', 'session': '01', 'dir': 'AP'} \ No newline at end of file From 198dfbe4bde6898e3fba5c1ab8c5906cb61ac820 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 2 Jun 2022 14:13:15 -0400 Subject: [PATCH 006/176] BF: address DepWarn about invalid escapes -- give those "r" --- heudiconv/heuristics/reproin.py | 2 +- heudiconv/heuristics/test_reproin.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index a4e1b29f..21882961 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -918,7 +918,7 @@ def fixup_subjectid(subjectid): """Just in case someone managed to miss a zero or added an extra one""" # make it lowercase subjectid = subjectid.lower() - reg = re.match("sid0*(\d+)$", subjectid) + reg = re.match(r"sid0*(\d+)$", subjectid) if not reg: # some completely other pattern # just filter out possible _- in it diff --git a/heudiconv/heuristics/test_reproin.py b/heudiconv/heuristics/test_reproin.py index 4b878f91..0753a37b 100644 --- a/heudiconv/heuristics/test_reproin.py +++ b/heudiconv/heuristics/test_reproin.py @@ -114,7 +114,7 @@ def test_fix_dbic_protocol(): seqinfos = [seq1, seq2] protocols2fix = { md5sum('mystudy'): - [('scout_run\+', 'THESCOUT-runX'), + [(r'scout_run\+', 'THESCOUT-runX'), ('run-life[0-9]', 'run+_task-life')], re.compile('^my.*'): [('THESCOUT-runX', 'THESCOUT')], @@ -221,4 +221,4 @@ def test_parse_series_spec(): {'datatype': 'func', 'session': '{date}'} assert pdpn("fmap_dir-AP_ses-01") == \ - {'datatype': 'fmap', 'session': '01', 'dir': 'AP'} \ No newline at end of file + {'datatype': 'fmap', 'session': '01', 'dir': 'AP'} From ce2089c55a478bd80513cad8bb72e5a6fc55d2b8 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 2 Jun 2022 14:23:13 -0400 Subject: [PATCH 007/176] DOC: adjust NotImplementedError message to correspond (it is _rec, not _acq used now) --- heudiconv/heuristics/reproin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index 21882961..bbf657f0 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -572,7 +572,7 @@ def infotodict(seqinfo): # assert s.is_derived, "Motion corrected images must be 'derived'" if s.is_motion_corrected and 'rec-' in series_info.get('bids', ''): - raise NotImplementedError("want to add _acq-moco but there is _acq- already") + raise NotImplementedError("want to add _rec-moco but there is _rec- already") def from_series_info(name): """A little helper to provide _name-value if series_info knows it From 1287cab9ddd22044e1185582163aa3fae61e6654 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 9 Aug 2022 14:32:19 -0400 Subject: [PATCH 008/176] DOC: provide rudimentary How to contribute section in README.rst --- README.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.rst b/README.rst index e0ce6aff..8f62c1e2 100644 --- a/README.rst +++ b/README.rst @@ -44,3 +44,22 @@ How to cite Please use `Zenodo record `_ for your specific version of HeuDiConv. We also support gathering all relevant citations via `DueCredit `_. + + +How to contribute +----------------- + +HeuDiConv sources are managed with Git on `GitHub `_. +Please file issues and suggest changes via Pull Requests. + +HeuDiConv requires installation of +`dcm2niix `_ and optionally +`DataLad `_. + +For development you will need a non-shallow clone (so there is a +recent released tag) of the aforementioned repository. You can then +install all necessary development requirements using ``pip install -r +dev-requirements.txt``. Testing is done using `pytest +`_. Releases are packaged using Intuit +auto. Workflow for releases and preparation of Docker images is in +``.github/workflows/release.yml``. From cc97c2f3fec8a7ba705d8467113c3761f0708555 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 9 Aug 2022 15:08:48 -0400 Subject: [PATCH 009/176] adjusted script for neurodocker although it does not work as is atm, see https://github.com/ReproNim/neurodocker/issues/466 --- utils/gen-docker-image.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/gen-docker-image.sh b/utils/gen-docker-image.sh index 968a5597..20f54e19 100755 --- a/utils/gen-docker-image.sh +++ b/utils/gen-docker-image.sh @@ -2,9 +2,10 @@ set -eu -VER=$(grep -Po '(?<=^__version__ = ).*' ../heudiconv/info.py | sed 's/"//g') +thisd=$(dirname $0) +VER=$(grep -Po '(?<=^__version__ = ).*' $thisd/../heudiconv/info.py | sed 's/"//g') -image="kaczmarj/neurodocker:master@sha256:936401fe8f677e0d294f688f352cbb643c9693f8de371475de1d593650e42a66" +image="kaczmarj/neurodocker:0.9.1" docker run --rm $image generate docker -b neurodebian:bullseye -p apt \ --dcm2niix version=v1.0.20211006 method=source \ From e284072365e250d95de3f7ccc8e168298a403828 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 9 Aug 2022 15:09:36 -0400 Subject: [PATCH 010/176] BF(docker): replace old -tipsy with -y -all for conda clean as neurodocker does now --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d03e5451..79344b52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -88,7 +88,7 @@ RUN export PATH="/opt/miniconda-latest/bin:$PATH" \ && conda config --system --prepend channels conda-forge \ && conda config --system --set auto_update_conda false \ && conda config --system --set show_channel_urls true \ - && sync && conda clean -tipsy && sync \ + && sync && conda clean -y --all && sync \ && conda install -y -q --name base \ 'python=3.7' \ 'traits>=4.6.0' \ @@ -96,7 +96,7 @@ RUN export PATH="/opt/miniconda-latest/bin:$PATH" \ 'numpy' \ 'nomkl' \ 'pandas' \ - && sync && conda clean -tipsy && sync \ + && sync && conda clean -y --all && sync \ && bash -c "source activate base \ && pip install --no-cache-dir --editable \ '/src/heudiconv[all]'" \ From ddf6733b809e82a8fe8f5989c14f1035267d6df7 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Fri, 23 Sep 2022 15:21:35 -0400 Subject: [PATCH 011/176] Convert Travis workflow to GitHub Actions --- .github/workflows/test.yml | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..3841eac1 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,59 @@ +name: Test + +on: + pull_request: + push: + +jobs: + test: + runs-on: ubuntu-latest + env: + BOTO_CONFIG: /tmp/nowhere + DATALAD_TESTS_SSH: '1' + strategy: + fail-fast: false + matrix: + python-version: + - '3.7' + - '3.8' + - '3.9' + # causes issues, disabled for now + # - '3.10' + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install git-annex + run: | + # The ultimate one-liner setup for NeuroDebian repository + bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) + sudo apt-get update -qq + sudo apt-get install git-annex-standalone dcm2niix + + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel + pip install -r dev-requirements.txt + pip install requests # below installs pyld but that assumes we have requests already + pip install datalad + pip install coverage pytest + + - name: Configure Git identity + run: | + git config --global user.email "test@github.land" + git config --global user.name "GitHub Almighty" + + - name: Run tests with coverage + run: coverage run `which pytest` -s -v heudiconv + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + fail_ci_if_error: false + +# vim:set et sts=2: From 33a0262781c58f347a1b1dbf2fdc9fd683be479d Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Fri, 23 Sep 2022 15:24:02 -0400 Subject: [PATCH 012/176] Check out a full clone when testing --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3841eac1..03f5a7d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,6 +22,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v4 From 4cdfb17fde70f2e7d7479cad0959736d614225c7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 23 Sep 2022 15:46:38 -0400 Subject: [PATCH 013/176] install dcmstack straight from github until it is released --- dev-requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index aa41cc1d..de37dce9 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,3 +3,6 @@ pytest==3.6.4 tinydb inotify +# we need master version until it gets released +# see https://github.com/moloney/dcmstack/issues/75 +git+https://github.com/moloney/dcmstack.git From 38e889c2e652045d45414aba60e04fbba81d09e5 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 23 Sep 2022 16:34:12 -0400 Subject: [PATCH 014/176] TST: Account for a bug in datalad which breeds entries in .gitmodules --- heudiconv/tests/test_heuristics.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/heudiconv/tests/test_heuristics.py b/heudiconv/tests/test_heuristics.py index 99a439e7..1f453247 100644 --- a/heudiconv/tests/test_heuristics.py +++ b/heudiconv/tests/test_heuristics.py @@ -90,7 +90,10 @@ def test_reproin_largely_smoke(tmpdir, heuristic, invocation): # but there should be nothing new assert not ds.repo.dirty - assert head == ds.repo.get_hexsha() + # TODO: remove whenever https://github.com/datalad/datalad/issues/6843 + # is fixed/released + buggy_datalad = (ds.pathobj / ".gitmodules").read_text().splitlines().count('[submodule "Halchenko"]') > 1 + assert head == ds.repo.get_hexsha() or buggy_datalad # unless we pass 'overwrite' flag runner(args + ['--overwrite']) @@ -98,7 +101,7 @@ def test_reproin_largely_smoke(tmpdir, heuristic, invocation): # and at the same commit assert ds.is_installed() assert not ds.repo.dirty - assert head == ds.repo.get_hexsha() + assert head == ds.repo.get_hexsha() or buggy_datalad @pytest.mark.parametrize( From adafefaff95aca343241a68b535ab7b6914a267c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 23 Sep 2022 16:37:19 -0400 Subject: [PATCH 015/176] BF: make strings using \ into r"" to address warnings --- heudiconv/convert.py | 2 +- heudiconv/parser.py | 4 ++-- heudiconv/tests/test_heuristics.py | 2 +- heudiconv/utils.py | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index d69baa25..4e090628 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -895,7 +895,7 @@ def add_taskname_to_infofile(infofiles): for infofile in infofiles: meta_info = load_json(infofile) try: - meta_info['TaskName'] = (re.search('(?<=_task-)\w+', + meta_info['TaskName'] = (re.search(r'(?<=_task-)\w+', op.basename(infofile)) .group(0).split('_')[0]) except AttributeError: diff --git a/heudiconv/parser.py b/heudiconv/parser.py index 3db1fc2c..e0678762 100644 --- a/heudiconv/parser.py +++ b/heudiconv/parser.py @@ -22,7 +22,7 @@ # Ensure they are cleaned up upon exit atexit.register(tempdirs.cleanup) -_VCS_REGEX = '%s\.(?:git|gitattributes|svn|bzr|hg)(?:%s|$)' % (op.sep, op.sep) +_VCS_REGEX = r'%s\.(?:git|gitattributes|svn|bzr|hg)(?:%s|$)' % (op.sep, op.sep) @docstring_parameter(_VCS_REGEX) @@ -161,7 +161,7 @@ def get_study_sessions(dicom_dir_template, files_opt, heuristic, outdir, for f in files_opt: if op.isdir(f): files += sorted(find_files( - '.*', topdir=f, exclude_vcs=True, exclude="/\.datalad/")) + '.*', topdir=f, exclude_vcs=True, exclude=r"/\.datalad/")) else: files.append(f) diff --git a/heudiconv/tests/test_heuristics.py b/heudiconv/tests/test_heuristics.py index 1f453247..740a561b 100644 --- a/heudiconv/tests/test_heuristics.py +++ b/heudiconv/tests/test_heuristics.py @@ -124,7 +124,7 @@ def test_scans_keys_reproin(tmpdir, invocation): if i != 0: assert(os.path.exists(pjoin(dirname(scans_keys[0]), row[0]))) assert(re.match( - '^[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}.[\d]{6}$', + r'^[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}.[\d]{6}$', row[1])) diff --git a/heudiconv/utils.py b/heudiconv/utils.py index e3b74dfa..5910586d 100644 --- a/heudiconv/utils.py +++ b/heudiconv/utils.py @@ -259,20 +259,20 @@ def json_dumps_pretty(j, indent=2, sort_keys=True): '[\n ]+("?[-+.0-9e]+"?,?) *\n(?= *"?[-+.0-9e]+"?)', r' \1', js, flags=re.MULTILINE) # uniform no spaces before ] - js_ = re.sub(" *\]", "]", js_) + js_ = re.sub(r" *\]", "]", js_) # uniform spacing before numbers # But that thing could screw up dates within strings which would have 2 spaces # in a date like Mar 3 2017, so we do negative lookahead to avoid changing # in those cases #import pdb; pdb.set_trace() js_ = re.sub( - '(? ?)[ \n]*', + r'(? ?)[ \n]*', r' \1\g', js_) # no spaces after [ - js_ = re.sub('\[ ', '[', js_) + js_ = re.sub(r'\[ ', '[', js_) # the load from the original dump and reload from tuned up # version should result in identical values since no value # must be changed, just formatting. From 593785b8e710526b44ba701e0990b8157c37a3d2 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 23 Sep 2022 16:39:00 -0400 Subject: [PATCH 016/176] no more Travis --- .travis.yml | 45 --------------------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0c093198..00000000 --- a/.travis.yml +++ /dev/null @@ -1,45 +0,0 @@ -# vim ft=yaml -language: python -dist: bionic -python: - - 3.7 - - 3.8 - - 3.9 - # causes issues, disabled for now - # - '3.10' - -cache: - - apt - -env: - global: - # will be used in the matrix, where neither other variable is used - - BOTO_CONFIG=/tmp/nowhere - - DATALAD_TESTS_SSH=1 - -before_install: - # The ultimate one-liner setup for NeuroDebian repository - - bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) - - travis_retry sudo apt-get update -qq - - travis_retry sudo apt-get install git-annex-standalone dcm2niix - # Install in our own virtualenv - - python -m pip install --upgrade pip - - pip install --upgrade virtualenv - - virtualenv --python=python venv - - source venv/bin/activate - - pip --version # check again since seems that python_requires='>=3.5' in secretstorage is not in effect - - python --version # just to check - - pip install -r dev-requirements.txt - - pip install requests # below installs pyld but that assumes we have requests already - - pip install datalad - - pip install codecov pytest - -install: - - git config --global user.email "test@travis.land" - - git config --global user.name "Travis Almighty" - -script: - - coverage run `which py.test` -s -v heudiconv - -after_success: - - codecov From ebd1efd38cc457cfedc5049183e7e4e42f18568c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 23 Sep 2022 16:39:09 -0400 Subject: [PATCH 017/176] Test python 3.10 as well --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03f5a7d3..29366c99 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,8 +17,7 @@ jobs: - '3.7' - '3.8' - '3.9' - # causes issues, disabled for now - # - '3.10' + - '3.10' steps: - name: Check out repository uses: actions/checkout@v3 From 431ad0f74e050b71a44ccacadb2718f5771988ce Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 23 Sep 2022 16:50:26 -0400 Subject: [PATCH 018/176] Try without locking pytest to old version - incompatible with py 3.10 --- dev-requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index de37dce9..3ff82c9c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,4 @@ -r requirements.txt -# Fix version to older pytest to ease backward compatibility testing -pytest==3.6.4 tinydb inotify # we need master version until it gets released From ebbc024e0a0d4e0ffe57090b63b9b2494f8e4174 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 24 Sep 2022 14:33:32 -0400 Subject: [PATCH 019/176] 0.9 of dcmstack was released, no need for github version --- dev-requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 3ff82c9c..2f8b1f2f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,3 @@ -r requirements.txt tinydb inotify -# we need master version until it gets released -# see https://github.com/moloney/dcmstack/issues/75 -git+https://github.com/moloney/dcmstack.git From 7effb64befae85c78f745e9868b318901ac689d0 Mon Sep 17 00:00:00 2001 From: auto Date: Thu, 29 Sep 2022 21:26:37 +0000 Subject: [PATCH 020/176] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b2cb1b3..ac671e16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,29 @@ +# v0.11.4 (Thu Sep 29 2022) + +#### 🐛 Bug Fix + +- install dcmstack straight from github until it is released [#593](https://github.com/nipy/heudiconv/pull/593) ([@yarikoptic](https://github.com/yarikoptic)) +- DOC: provide rudimentary How to contribute section in README.rst ([@yarikoptic](https://github.com/yarikoptic)) + +#### ⚠️ Pushed to `master` + +- Check out a full clone when testing ([@jwodder](https://github.com/jwodder)) +- Convert Travis workflow to GitHub Actions ([@jwodder](https://github.com/jwodder)) +- BF(docker): replace old -tipsy with -y -all for conda clean as neurodocker does now ([@yarikoptic](https://github.com/yarikoptic)) +- adjusted script for neurodocker although it does not work ([@yarikoptic](https://github.com/yarikoptic)) + +#### 🏠 Internal + +- 0.9 of dcmstack was released, no need for github version [#594](https://github.com/nipy/heudiconv/pull/594) ([@yarikoptic](https://github.com/yarikoptic)) +- Minor face-lifts to ReproIn: align doc and code better to BIDS terms, address deprecation warnings etc [#569](https://github.com/nipy/heudiconv/pull/569) ([@yarikoptic](https://github.com/yarikoptic)) + +#### Authors: 2 + +- John T. Wodder II ([@jwodder](https://github.com/jwodder)) +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + # v0.11.3 (Thu May 12 2022) #### 🏠 Internal From 4a4dc853116fb57a54d63f869fedfe735f40604e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 29 Sep 2022 19:08:58 -0400 Subject: [PATCH 021/176] Fixup miniconda spec for neurodocker so it produces dockerfile now --- Dockerfile | 253 +++++++++++++++++++------------------- utils/gen-docker-image.sh | 2 +- 2 files changed, 125 insertions(+), 130 deletions(-) diff --git a/Dockerfile b/Dockerfile index 79344b52..47d59dd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,47 +1,10 @@ -# Generated by Neurodocker version 0.4.2-3-gf7055a1 -# Timestamp: 2022-04-06 17:10:29 UTC -# -# Thank you for using Neurodocker. If you discover any issues -# or ways to improve this software, please submit an issue or -# pull request on our GitHub repository: -# -# https://github.com/kaczmarj/neurodocker +# Generated by Neurodocker and Reproenv. FROM neurodebian:bullseye - -ARG DEBIAN_FRONTEND="noninteractive" - -ENV LANG="en_US.UTF-8" \ - LC_ALL="en_US.UTF-8" \ - ND_ENTRYPOINT="/neurodocker/startup.sh" -RUN export ND_ENTRYPOINT="/neurodocker/startup.sh" \ - && apt-get update -qq \ - && apt-get install -y -q --no-install-recommends \ - apt-utils \ - bzip2 \ - ca-certificates \ - curl \ - locales \ - unzip \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ - && dpkg-reconfigure --frontend=noninteractive locales \ - && update-locale LANG="en_US.UTF-8" \ - && chmod 777 /opt && chmod a+s /opt \ - && mkdir -p /neurodocker \ - && if [ ! -f "$ND_ENTRYPOINT" ]; then \ - echo '#!/usr/bin/env bash' >> "$ND_ENTRYPOINT" \ - && echo 'set -e' >> "$ND_ENTRYPOINT" \ - && echo 'if [ -n "$1" ]; then "$@"; else /usr/bin/env bash; fi' >> "$ND_ENTRYPOINT"; \ - fi \ - && chmod -R 777 /neurodocker && chmod a+s /neurodocker - -ENTRYPOINT ["/neurodocker/startup.sh"] - ENV PATH="/opt/dcm2niix-v1.0.20211006/bin:$PATH" RUN apt-get update -qq \ && apt-get install -y -q --no-install-recommends \ + ca-certificates \ cmake \ g++ \ gcc \ @@ -49,8 +12,7 @@ RUN apt-get update -qq \ make \ pigz \ zlib1g-dev \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && rm -rf /var/lib/apt/lists/* \ && git clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix \ && cd /tmp/dcm2niix \ && git fetch --tags \ @@ -58,107 +20,140 @@ RUN apt-get update -qq \ && mkdir /tmp/dcm2niix/build \ && cd /tmp/dcm2niix/build \ && cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20211006 .. \ - && make \ + && make -j1 \ && make install \ && rm -rf /tmp/dcm2niix - RUN apt-get update -qq \ - && apt-get install -y -q --no-install-recommends \ - git \ - gcc \ - pigz \ - liblzma-dev \ - libc-dev \ - git-annex-standalone \ - netbase \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -COPY [".", "/src/heudiconv"] - + && apt-get install -y -q --no-install-recommends \ + gcc \ + git \ + git-annex-standalone \ + libc-dev \ + liblzma-dev \ + netbase \ + pigz \ + && rm -rf /var/lib/apt/lists/* +COPY [".", \ + "/src/heudiconv"] ENV CONDA_DIR="/opt/miniconda-latest" \ PATH="/opt/miniconda-latest/bin:$PATH" -RUN export PATH="/opt/miniconda-latest/bin:$PATH" \ +RUN apt-get update -qq \ + && apt-get install -y -q --no-install-recommends \ + bzip2 \ + ca-certificates \ + curl \ + && rm -rf /var/lib/apt/lists/* \ + # Install dependencies. + && export PATH="/opt/miniconda-latest/bin:$PATH" \ && echo "Downloading Miniconda installer ..." \ && conda_installer="/tmp/miniconda.sh" \ - && curl -fsSL --retry 5 -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && curl -fsSL -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ && bash "$conda_installer" -b -p /opt/miniconda-latest \ && rm -f "$conda_installer" \ && conda update -yq -nbase conda \ + # Prefer packages in conda-forge && conda config --system --prepend channels conda-forge \ + # Packages in lower-priority channels not considered if a package with the same + # name exists in a higher priority channel. Can dramatically speed up installations. + # Conda recommends this as a default + # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html + && conda config --set channel_priority strict \ && conda config --system --set auto_update_conda false \ && conda config --system --set show_channel_urls true \ - && sync && conda clean -y --all && sync \ - && conda install -y -q --name base \ - 'python=3.7' \ - 'traits>=4.6.0' \ - 'scipy' \ - 'numpy' \ - 'nomkl' \ - 'pandas' \ - && sync && conda clean -y --all && sync \ + # Enable `conda activate` + && conda init bash \ + && conda install -y --name base \ + "python=3.7" \ + "traits>=4.6.0" \ + "scipy" \ + "numpy" \ + "nomkl" \ + "pandas" \ && bash -c "source activate base \ - && pip install --no-cache-dir --editable \ - '/src/heudiconv[all]'" \ - && rm -rf ~/.cache/pip/* \ - && sync - + && python -m pip install --no-cache-dir --editable \ + "/src/heudiconv[all]"" \ + # Clean up + && sync && conda clean --all --yes && sync \ + && rm -rf ~/.cache/pip/* ENTRYPOINT ["heudiconv"] -RUN echo '{ \ - \n "pkg_manager": "apt", \ - \n "instructions": [ \ - \n [ \ - \n "base", \ - \n "neurodebian:bullseye" \ - \n ], \ - \n [ \ - \n "dcm2niix", \ - \n { \ - \n "version": "v1.0.20211006", \ - \n "method": "source" \ - \n } \ - \n ], \ - \n [ \ - \n "install", \ - \n [ \ - \n "git", \ - \n "gcc", \ - \n "pigz", \ - \n "liblzma-dev", \ - \n "libc-dev", \ - \n "git-annex-standalone", \ - \n "netbase" \ - \n ] \ - \n ], \ - \n [ \ - \n "copy", \ - \n [ \ - \n ".", \ - \n "/src/heudiconv" \ - \n ] \ - \n ], \ - \n [ \ - \n "miniconda", \ - \n { \ - \n "use_env": "base", \ - \n "conda_install": [ \ - \n "python=3.7", \ - \n "traits>=4.6.0", \ - \n "scipy", \ - \n "numpy", \ - \n "nomkl", \ - \n "pandas" \ - \n ], \ - \n "pip_install": [ \ - \n "/src/heudiconv[all]" \ - \n ], \ - \n "pip_opts": "--editable" \ - \n } \ - \n ], \ - \n [ \ - \n "entrypoint", \ - \n "heudiconv" \ - \n ] \ - \n ] \ - \n}' > /neurodocker/neurodocker_specs.json +# Save specification to JSON. +RUN printf '{ \ + "pkg_manager": "apt", \ + "existing_users": [ \ + "root" \ + ], \ + "instructions": [ \ + { \ + "name": "from_", \ + "kwds": { \ + "base_image": "neurodebian:bullseye" \ + } \ + }, \ + { \ + "name": "env", \ + "kwds": { \ + "PATH": "/opt/dcm2niix-v1.0.20211006/bin:$PATH" \ + } \ + }, \ + { \ + "name": "run", \ + "kwds": { \ + "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n ca-certificates \\\\\\n cmake \\\\\\n g++ \\\\\\n gcc \\\\\\n git \\\\\\n make \\\\\\n pigz \\\\\\n zlib1g-dev\\nrm -rf /var/lib/apt/lists/*\\ngit clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix\\ncd /tmp/dcm2niix\\ngit fetch --tags\\ngit checkout v1.0.20211006\\nmkdir /tmp/dcm2niix/build\\ncd /tmp/dcm2niix/build\\ncmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20211006 ..\\nmake -j1\\nmake install\\nrm -rf /tmp/dcm2niix" \ + } \ + }, \ + { \ + "name": "install", \ + "kwds": { \ + "pkgs": [ \ + "git", \ + "gcc", \ + "pigz", \ + "liblzma-dev", \ + "libc-dev", \ + "git-annex-standalone", \ + "netbase" \ + ], \ + "opts": null \ + } \ + }, \ + { \ + "name": "run", \ + "kwds": { \ + "command": "apt-get update -qq \\\\\\n && apt-get install -y -q --no-install-recommends \\\\\\n gcc \\\\\\n git \\\\\\n git-annex-standalone \\\\\\n libc-dev \\\\\\n liblzma-dev \\\\\\n netbase \\\\\\n pigz \\\\\\n && rm -rf /var/lib/apt/lists/*" \ + } \ + }, \ + { \ + "name": "copy", \ + "kwds": { \ + "source": [ \ + ".", \ + "/src/heudiconv" \ + ], \ + "destination": "/src/heudiconv" \ + } \ + }, \ + { \ + "name": "env", \ + "kwds": { \ + "CONDA_DIR": "/opt/miniconda-latest", \ + "PATH": "/opt/miniconda-latest/bin:$PATH" \ + } \ + }, \ + { \ + "name": "run", \ + "kwds": { \ + "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n bzip2 \\\\\\n ca-certificates \\\\\\n curl\\nrm -rf /var/lib/apt/lists/*\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-latest/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-latest\\nrm -f \\"$conda_installer\\"\\nconda update -yq -nbase conda\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda install -y --name base \\\\\\n \\"python=3.7\\" \\\\\\n \\"traits>=4.6.0\\" \\\\\\n \\"scipy\\" \\\\\\n \\"numpy\\" \\\\\\n \\"nomkl\\" \\\\\\n \\"pandas\\"\\nbash -c \\"source activate base\\n python -m pip install --no-cache-dir --editable \\\\\\n \\"/src/heudiconv[all]\\"\\"\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \ + } \ + }, \ + { \ + "name": "entrypoint", \ + "kwds": { \ + "args": [ \ + "heudiconv" \ + ] \ + } \ + } \ + ] \ +}' > /.reproenv.json +# End saving to specification to JSON. diff --git a/utils/gen-docker-image.sh b/utils/gen-docker-image.sh index 20f54e19..08049095 100755 --- a/utils/gen-docker-image.sh +++ b/utils/gen-docker-image.sh @@ -11,7 +11,7 @@ docker run --rm $image generate docker -b neurodebian:bullseye -p apt \ --dcm2niix version=v1.0.20211006 method=source \ --install git gcc pigz liblzma-dev libc-dev git-annex-standalone netbase \ --copy . /src/heudiconv \ - --miniconda use_env=base conda_install="python=3.7 traits>=4.6.0 scipy numpy nomkl pandas" \ + --miniconda version=latest conda_install="python=3.7 traits>=4.6.0 scipy numpy nomkl pandas" \ pip_install="/src/heudiconv[all]" \ pip_opts="--editable" \ --entrypoint "heudiconv" \ From 6fe77040e376bc8e98ee99ff7cf5bd605ba187c9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 5 Oct 2022 09:48:52 -0400 Subject: [PATCH 022/176] docker image Use python 3.9 and newer dcm2niix from this year --- utils/gen-docker-image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/gen-docker-image.sh b/utils/gen-docker-image.sh index 08049095..13c8c4f4 100755 --- a/utils/gen-docker-image.sh +++ b/utils/gen-docker-image.sh @@ -8,10 +8,10 @@ VER=$(grep -Po '(?<=^__version__ = ).*' $thisd/../heudiconv/info.py | sed 's/"// image="kaczmarj/neurodocker:0.9.1" docker run --rm $image generate docker -b neurodebian:bullseye -p apt \ - --dcm2niix version=v1.0.20211006 method=source \ + --dcm2niix version=v1.0.20220720 method=source \ --install git gcc pigz liblzma-dev libc-dev git-annex-standalone netbase \ --copy . /src/heudiconv \ - --miniconda version=latest conda_install="python=3.7 traits>=4.6.0 scipy numpy nomkl pandas" \ + --miniconda version=latest conda_install="python=3.9 traits>=4.6.0 scipy numpy nomkl pandas" \ pip_install="/src/heudiconv[all]" \ pip_opts="--editable" \ --entrypoint "heudiconv" \ From 4e7d1d295d91a2ef4db96d2bc408f28b2c70da87 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 5 Oct 2022 09:51:43 -0400 Subject: [PATCH 023/176] BF: make utils/gen-docker-image.sh work regardless of location of invocation --- utils/gen-docker-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/gen-docker-image.sh b/utils/gen-docker-image.sh index 13c8c4f4..e58a774a 100755 --- a/utils/gen-docker-image.sh +++ b/utils/gen-docker-image.sh @@ -15,4 +15,4 @@ docker run --rm $image generate docker -b neurodebian:bullseye -p apt \ pip_install="/src/heudiconv[all]" \ pip_opts="--editable" \ --entrypoint "heudiconv" \ -> ../Dockerfile +> $thisd/../Dockerfile From e179ee94e55f524620bea09c4d29fe56a7908fff Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 5 Oct 2022 09:51:47 -0400 Subject: [PATCH 024/176] [DATALAD RUNCMD] produce updated dockerfile === Do not change lines below === { "chain": [], "cmd": "utils/gen-docker-image.sh", "exit": 0, "extra_inputs": [], "inputs": [], "outputs": [], "pwd": "." } ^^^ Do not change lines above ^^^ --- Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 47d59dd1..bddb26e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Generated by Neurodocker and Reproenv. FROM neurodebian:bullseye -ENV PATH="/opt/dcm2niix-v1.0.20211006/bin:$PATH" +ENV PATH="/opt/dcm2niix-v1.0.20220720/bin:$PATH" RUN apt-get update -qq \ && apt-get install -y -q --no-install-recommends \ ca-certificates \ @@ -16,10 +16,10 @@ RUN apt-get update -qq \ && git clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix \ && cd /tmp/dcm2niix \ && git fetch --tags \ - && git checkout v1.0.20211006 \ + && git checkout v1.0.20220720 \ && mkdir /tmp/dcm2niix/build \ && cd /tmp/dcm2niix/build \ - && cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20211006 .. \ + && cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20220720 .. \ && make -j1 \ && make install \ && rm -rf /tmp/dcm2niix @@ -63,7 +63,7 @@ RUN apt-get update -qq \ # Enable `conda activate` && conda init bash \ && conda install -y --name base \ - "python=3.7" \ + "python=3.9" \ "traits>=4.6.0" \ "scipy" \ "numpy" \ @@ -93,13 +93,13 @@ RUN printf '{ \ { \ "name": "env", \ "kwds": { \ - "PATH": "/opt/dcm2niix-v1.0.20211006/bin:$PATH" \ + "PATH": "/opt/dcm2niix-v1.0.20220720/bin:$PATH" \ } \ }, \ { \ "name": "run", \ "kwds": { \ - "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n ca-certificates \\\\\\n cmake \\\\\\n g++ \\\\\\n gcc \\\\\\n git \\\\\\n make \\\\\\n pigz \\\\\\n zlib1g-dev\\nrm -rf /var/lib/apt/lists/*\\ngit clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix\\ncd /tmp/dcm2niix\\ngit fetch --tags\\ngit checkout v1.0.20211006\\nmkdir /tmp/dcm2niix/build\\ncd /tmp/dcm2niix/build\\ncmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20211006 ..\\nmake -j1\\nmake install\\nrm -rf /tmp/dcm2niix" \ + "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n ca-certificates \\\\\\n cmake \\\\\\n g++ \\\\\\n gcc \\\\\\n git \\\\\\n make \\\\\\n pigz \\\\\\n zlib1g-dev\\nrm -rf /var/lib/apt/lists/*\\ngit clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix\\ncd /tmp/dcm2niix\\ngit fetch --tags\\ngit checkout v1.0.20220720\\nmkdir /tmp/dcm2niix/build\\ncd /tmp/dcm2niix/build\\ncmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20220720 ..\\nmake -j1\\nmake install\\nrm -rf /tmp/dcm2niix" \ } \ }, \ { \ @@ -143,7 +143,7 @@ RUN printf '{ \ { \ "name": "run", \ "kwds": { \ - "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n bzip2 \\\\\\n ca-certificates \\\\\\n curl\\nrm -rf /var/lib/apt/lists/*\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-latest/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-latest\\nrm -f \\"$conda_installer\\"\\nconda update -yq -nbase conda\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda install -y --name base \\\\\\n \\"python=3.7\\" \\\\\\n \\"traits>=4.6.0\\" \\\\\\n \\"scipy\\" \\\\\\n \\"numpy\\" \\\\\\n \\"nomkl\\" \\\\\\n \\"pandas\\"\\nbash -c \\"source activate base\\n python -m pip install --no-cache-dir --editable \\\\\\n \\"/src/heudiconv[all]\\"\\"\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \ + "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n bzip2 \\\\\\n ca-certificates \\\\\\n curl\\nrm -rf /var/lib/apt/lists/*\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-latest/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-latest\\nrm -f \\"$conda_installer\\"\\nconda update -yq -nbase conda\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda install -y --name base \\\\\\n \\"python=3.9\\" \\\\\\n \\"traits>=4.6.0\\" \\\\\\n \\"scipy\\" \\\\\\n \\"numpy\\" \\\\\\n \\"nomkl\\" \\\\\\n \\"pandas\\"\\nbash -c \\"source activate base\\n python -m pip install --no-cache-dir --editable \\\\\\n \\"/src/heudiconv[all]\\"\\"\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \ } \ }, \ { \ From 643022174d7bfe1c69b218187c079afb7d431e78 Mon Sep 17 00:00:00 2001 From: Michael Dayan Date: Wed, 5 Oct 2022 15:54:37 +0200 Subject: [PATCH 025/176] Fix certificate issue as indicated in #595 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 47d59dd1..1628b8ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ RUN apt-get update -qq \ && curl -fsSL -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ && bash "$conda_installer" -b -p /opt/miniconda-latest \ && rm -f "$conda_installer" \ - && conda update -yq -nbase conda \ + && conda install -yq -nbase conda==4.13.0 \ # Prefer packages in conda-forge && conda config --system --prepend channels conda-forge \ # Packages in lower-priority channels not considered if a package with the same From 401069bc459bccb13e99d6cacbc57776480dec43 Mon Sep 17 00:00:00 2001 From: Michael Dayan Date: Wed, 5 Oct 2022 17:10:39 +0200 Subject: [PATCH 026/176] Change neurodocker factory to set version of miniconda in order to set version of conda --- Dockerfile | 17 ++++++++--------- utils/gen-docker-image.sh | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1628b8ca..a0f16992 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,8 +35,8 @@ RUN apt-get update -qq \ && rm -rf /var/lib/apt/lists/* COPY [".", \ "/src/heudiconv"] -ENV CONDA_DIR="/opt/miniconda-latest" \ - PATH="/opt/miniconda-latest/bin:$PATH" +ENV CONDA_DIR="/opt/miniconda-py39_4.12.0" \ + PATH="/opt/miniconda-py39_4.12.0/bin:$PATH" RUN apt-get update -qq \ && apt-get install -y -q --no-install-recommends \ bzip2 \ @@ -44,13 +44,12 @@ RUN apt-get update -qq \ curl \ && rm -rf /var/lib/apt/lists/* \ # Install dependencies. - && export PATH="/opt/miniconda-latest/bin:$PATH" \ + && export PATH="/opt/miniconda-py39_4.12.0/bin:$PATH" \ && echo "Downloading Miniconda installer ..." \ && conda_installer="/tmp/miniconda.sh" \ - && curl -fsSL -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && bash "$conda_installer" -b -p /opt/miniconda-latest \ + && curl -fsSL -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh \ + && bash "$conda_installer" -b -p /opt/miniconda-py39_4.12.0 \ && rm -f "$conda_installer" \ - && conda install -yq -nbase conda==4.13.0 \ # Prefer packages in conda-forge && conda config --system --prepend channels conda-forge \ # Packages in lower-priority channels not considered if a package with the same @@ -136,14 +135,14 @@ RUN printf '{ \ { \ "name": "env", \ "kwds": { \ - "CONDA_DIR": "/opt/miniconda-latest", \ - "PATH": "/opt/miniconda-latest/bin:$PATH" \ + "CONDA_DIR": "/opt/miniconda-py39_4.12.0", \ + "PATH": "/opt/miniconda-py39_4.12.0/bin:$PATH" \ } \ }, \ { \ "name": "run", \ "kwds": { \ - "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n bzip2 \\\\\\n ca-certificates \\\\\\n curl\\nrm -rf /var/lib/apt/lists/*\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-latest/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-latest\\nrm -f \\"$conda_installer\\"\\nconda update -yq -nbase conda\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda install -y --name base \\\\\\n \\"python=3.7\\" \\\\\\n \\"traits>=4.6.0\\" \\\\\\n \\"scipy\\" \\\\\\n \\"numpy\\" \\\\\\n \\"nomkl\\" \\\\\\n \\"pandas\\"\\nbash -c \\"source activate base\\n python -m pip install --no-cache-dir --editable \\\\\\n \\"/src/heudiconv[all]\\"\\"\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \ + "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n bzip2 \\\\\\n ca-certificates \\\\\\n curl\\nrm -rf /var/lib/apt/lists/*\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-py39_4.12.0/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-py39_4.12.0\\nrm -f \\"$conda_installer\\"\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda install -y --name base \\\\\\n \\"python=3.7\\" \\\\\\n \\"traits>=4.6.0\\" \\\\\\n \\"scipy\\" \\\\\\n \\"numpy\\" \\\\\\n \\"nomkl\\" \\\\\\n \\"pandas\\"\\nbash -c \\"source activate base\\n python -m pip install --no-cache-dir --editable \\\\\\n \\"/src/heudiconv[all]\\"\\"\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \ } \ }, \ { \ diff --git a/utils/gen-docker-image.sh b/utils/gen-docker-image.sh index 08049095..6fd342bb 100755 --- a/utils/gen-docker-image.sh +++ b/utils/gen-docker-image.sh @@ -11,7 +11,7 @@ docker run --rm $image generate docker -b neurodebian:bullseye -p apt \ --dcm2niix version=v1.0.20211006 method=source \ --install git gcc pigz liblzma-dev libc-dev git-annex-standalone netbase \ --copy . /src/heudiconv \ - --miniconda version=latest conda_install="python=3.7 traits>=4.6.0 scipy numpy nomkl pandas" \ + --miniconda version="py39_4.12.0" conda_install="python=3.7 traits>=4.6.0 scipy numpy nomkl pandas" \ pip_install="/src/heudiconv[all]" \ pip_opts="--editable" \ --entrypoint "heudiconv" \ From a643c4c784bb6be68905dd61208b72beb20067de Mon Sep 17 00:00:00 2001 From: Michael Dayan Date: Wed, 5 Oct 2022 17:19:04 +0200 Subject: [PATCH 027/176] Change miniconda python version from 3.9 to 3.7 to be consistent with python env version --- Dockerfile | 16 ++++++++-------- utils/gen-docker-image.sh | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index a0f16992..ef7aaf49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,8 +35,8 @@ RUN apt-get update -qq \ && rm -rf /var/lib/apt/lists/* COPY [".", \ "/src/heudiconv"] -ENV CONDA_DIR="/opt/miniconda-py39_4.12.0" \ - PATH="/opt/miniconda-py39_4.12.0/bin:$PATH" +ENV CONDA_DIR="/opt/miniconda-py37_4.12.0" \ + PATH="/opt/miniconda-py37_4.12.0/bin:$PATH" RUN apt-get update -qq \ && apt-get install -y -q --no-install-recommends \ bzip2 \ @@ -44,11 +44,11 @@ RUN apt-get update -qq \ curl \ && rm -rf /var/lib/apt/lists/* \ # Install dependencies. - && export PATH="/opt/miniconda-py39_4.12.0/bin:$PATH" \ + && export PATH="/opt/miniconda-py37_4.12.0/bin:$PATH" \ && echo "Downloading Miniconda installer ..." \ && conda_installer="/tmp/miniconda.sh" \ - && curl -fsSL -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh \ - && bash "$conda_installer" -b -p /opt/miniconda-py39_4.12.0 \ + && curl -fsSL -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh \ + && bash "$conda_installer" -b -p /opt/miniconda-py37_4.12.0 \ && rm -f "$conda_installer" \ # Prefer packages in conda-forge && conda config --system --prepend channels conda-forge \ @@ -135,14 +135,14 @@ RUN printf '{ \ { \ "name": "env", \ "kwds": { \ - "CONDA_DIR": "/opt/miniconda-py39_4.12.0", \ - "PATH": "/opt/miniconda-py39_4.12.0/bin:$PATH" \ + "CONDA_DIR": "/opt/miniconda-py37_4.12.0", \ + "PATH": "/opt/miniconda-py37_4.12.0/bin:$PATH" \ } \ }, \ { \ "name": "run", \ "kwds": { \ - "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n bzip2 \\\\\\n ca-certificates \\\\\\n curl\\nrm -rf /var/lib/apt/lists/*\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-py39_4.12.0/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-py39_4.12.0\\nrm -f \\"$conda_installer\\"\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda install -y --name base \\\\\\n \\"python=3.7\\" \\\\\\n \\"traits>=4.6.0\\" \\\\\\n \\"scipy\\" \\\\\\n \\"numpy\\" \\\\\\n \\"nomkl\\" \\\\\\n \\"pandas\\"\\nbash -c \\"source activate base\\n python -m pip install --no-cache-dir --editable \\\\\\n \\"/src/heudiconv[all]\\"\\"\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \ + "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n bzip2 \\\\\\n ca-certificates \\\\\\n curl\\nrm -rf /var/lib/apt/lists/*\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-py37_4.12.0/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-py37_4.12.0\\nrm -f \\"$conda_installer\\"\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda install -y --name base \\\\\\n \\"python=3.7\\" \\\\\\n \\"traits>=4.6.0\\" \\\\\\n \\"scipy\\" \\\\\\n \\"numpy\\" \\\\\\n \\"nomkl\\" \\\\\\n \\"pandas\\"\\nbash -c \\"source activate base\\n python -m pip install --no-cache-dir --editable \\\\\\n \\"/src/heudiconv[all]\\"\\"\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \ } \ }, \ { \ diff --git a/utils/gen-docker-image.sh b/utils/gen-docker-image.sh index 6fd342bb..5f080e6a 100755 --- a/utils/gen-docker-image.sh +++ b/utils/gen-docker-image.sh @@ -11,7 +11,7 @@ docker run --rm $image generate docker -b neurodebian:bullseye -p apt \ --dcm2niix version=v1.0.20211006 method=source \ --install git gcc pigz liblzma-dev libc-dev git-annex-standalone netbase \ --copy . /src/heudiconv \ - --miniconda version="py39_4.12.0" conda_install="python=3.7 traits>=4.6.0 scipy numpy nomkl pandas" \ + --miniconda version="py37_4.12.0" conda_install="python=3.7 traits>=4.6.0 scipy numpy nomkl pandas" \ pip_install="/src/heudiconv[all]" \ pip_opts="--editable" \ --entrypoint "heudiconv" \ From 49e79db8a5e4f853f23f145097bd4a4a27caf5d5 Mon Sep 17 00:00:00 2001 From: Michael Dayan Date: Wed, 5 Oct 2022 17:31:11 +0200 Subject: [PATCH 028/176] Update miniconda and python version to 3.9 for neurodocker --- Dockerfile | 18 +++++++++--------- utils/gen-docker-image.sh | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index ef7aaf49..cda3b0e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,8 +35,8 @@ RUN apt-get update -qq \ && rm -rf /var/lib/apt/lists/* COPY [".", \ "/src/heudiconv"] -ENV CONDA_DIR="/opt/miniconda-py37_4.12.0" \ - PATH="/opt/miniconda-py37_4.12.0/bin:$PATH" +ENV CONDA_DIR="/opt/miniconda-py39_4.12.0" \ + PATH="/opt/miniconda-py39_4.12.0/bin:$PATH" RUN apt-get update -qq \ && apt-get install -y -q --no-install-recommends \ bzip2 \ @@ -44,11 +44,11 @@ RUN apt-get update -qq \ curl \ && rm -rf /var/lib/apt/lists/* \ # Install dependencies. - && export PATH="/opt/miniconda-py37_4.12.0/bin:$PATH" \ + && export PATH="/opt/miniconda-py39_4.12.0/bin:$PATH" \ && echo "Downloading Miniconda installer ..." \ && conda_installer="/tmp/miniconda.sh" \ - && curl -fsSL -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh \ - && bash "$conda_installer" -b -p /opt/miniconda-py37_4.12.0 \ + && curl -fsSL -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh \ + && bash "$conda_installer" -b -p /opt/miniconda-py39_4.12.0 \ && rm -f "$conda_installer" \ # Prefer packages in conda-forge && conda config --system --prepend channels conda-forge \ @@ -62,7 +62,7 @@ RUN apt-get update -qq \ # Enable `conda activate` && conda init bash \ && conda install -y --name base \ - "python=3.7" \ + "python=3.9" \ "traits>=4.6.0" \ "scipy" \ "numpy" \ @@ -135,14 +135,14 @@ RUN printf '{ \ { \ "name": "env", \ "kwds": { \ - "CONDA_DIR": "/opt/miniconda-py37_4.12.0", \ - "PATH": "/opt/miniconda-py37_4.12.0/bin:$PATH" \ + "CONDA_DIR": "/opt/miniconda-py39_4.12.0", \ + "PATH": "/opt/miniconda-py39_4.12.0/bin:$PATH" \ } \ }, \ { \ "name": "run", \ "kwds": { \ - "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n bzip2 \\\\\\n ca-certificates \\\\\\n curl\\nrm -rf /var/lib/apt/lists/*\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-py37_4.12.0/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-py37_4.12.0\\nrm -f \\"$conda_installer\\"\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda install -y --name base \\\\\\n \\"python=3.7\\" \\\\\\n \\"traits>=4.6.0\\" \\\\\\n \\"scipy\\" \\\\\\n \\"numpy\\" \\\\\\n \\"nomkl\\" \\\\\\n \\"pandas\\"\\nbash -c \\"source activate base\\n python -m pip install --no-cache-dir --editable \\\\\\n \\"/src/heudiconv[all]\\"\\"\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \ + "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n bzip2 \\\\\\n ca-certificates \\\\\\n curl\\nrm -rf /var/lib/apt/lists/*\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-py39_4.12.0/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-py39_4.12.0\\nrm -f \\"$conda_installer\\"\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda install -y --name base \\\\\\n \\"python=3.9\\" \\\\\\n \\"traits>=4.6.0\\" \\\\\\n \\"scipy\\" \\\\\\n \\"numpy\\" \\\\\\n \\"nomkl\\" \\\\\\n \\"pandas\\"\\nbash -c \\"source activate base\\n python -m pip install --no-cache-dir --editable \\\\\\n \\"/src/heudiconv[all]\\"\\"\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \ } \ }, \ { \ diff --git a/utils/gen-docker-image.sh b/utils/gen-docker-image.sh index 5f080e6a..acf3bfd4 100755 --- a/utils/gen-docker-image.sh +++ b/utils/gen-docker-image.sh @@ -11,7 +11,7 @@ docker run --rm $image generate docker -b neurodebian:bullseye -p apt \ --dcm2niix version=v1.0.20211006 method=source \ --install git gcc pigz liblzma-dev libc-dev git-annex-standalone netbase \ --copy . /src/heudiconv \ - --miniconda version="py37_4.12.0" conda_install="python=3.7 traits>=4.6.0 scipy numpy nomkl pandas" \ + --miniconda version="py39_4.12.0" conda_install="python=3.9 traits>=4.6.0 scipy numpy nomkl pandas" \ pip_install="/src/heudiconv[all]" \ pip_opts="--editable" \ --entrypoint "heudiconv" \ From 9adfc47a6ce929ecd52c20eb3d9110a11d9fe83d Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Wed, 12 Oct 2022 09:10:20 -0400 Subject: [PATCH 029/176] Set action step outputs via $GITHUB_OUTPUT --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7416074e..2767142a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: id: auto-version run: | version="$(~/auto version)" - echo "::set-output name=version::$version" + echo "version=$version" >> "$GITHUB_OUTPUT" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 205c36f8cf5c47d788492c638d18177259bdfada Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Wed, 12 Oct 2022 14:49:11 -0400 Subject: [PATCH 030/176] Update GitHub Actions action versions --- .github/workflows/docker.yml | 2 +- .github/workflows/release.yml | 4 ++-- .github/workflows/test.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ac09ea54..8c66c2ef 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout source - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2767142a..05957516 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ jobs: if: "!contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.head_commit.message, 'skip ci')" steps: - name: Checkout source - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -34,7 +34,7 @@ jobs: - name: Set up Python if: steps.auto-version.outputs.version != '' - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '^3.7' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29366c99..72883a62 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,7 +53,7 @@ jobs: run: coverage run `which pytest` -s -v heudiconv - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: fail_ci_if_error: false From 8b8d80543046ab60cf1063c50f9261e7fd9d2810 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Wed, 12 Oct 2022 14:49:30 -0400 Subject: [PATCH 031/176] Configure Dependabot to update GitHub Actions action versions --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..aa807cd2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + commit-message: + prefix: "[gh-actions]" + include: scope + labels: + - internal From 151f25adf46ed61d275a6544e35ef28ceb99b8d2 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 3 Nov 2022 13:20:59 -0400 Subject: [PATCH 032/176] DOC: codespell fix a few typos in code comments --- heudiconv/heuristics/multires_7Tbold.py | 2 +- heudiconv/tests/test_heuristics.py | 4 ++-- heudiconv/tests/test_queue.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/heudiconv/heuristics/multires_7Tbold.py b/heudiconv/heuristics/multires_7Tbold.py index 0489b00d..9bbbf06d 100644 --- a/heudiconv/heuristics/multires_7Tbold.py +++ b/heudiconv/heuristics/multires_7Tbold.py @@ -27,7 +27,7 @@ def extract_moco_params(basename, outypes, dicoms): dcm_times = [(d, float(dcm_read(d, stop_before_pixels=True).AcquisitionTime)) for d in dicoms] - # store MoCo info from image comments sorted by acqusition time + # store MoCo info from image comments sorted by acquisition time moco = ['\t'.join( [str(float(i)) for i in dcm_read(fn, stop_before_pixels=True).ImageComments.split()[1].split(',')]) for fn, t in sorted(dcm_times, key=lambda x: x[1])] diff --git a/heudiconv/tests/test_heuristics.py b/heudiconv/tests/test_heuristics.py index 740a561b..906e15f7 100644 --- a/heudiconv/tests/test_heuristics.py +++ b/heudiconv/tests/test_heuristics.py @@ -45,7 +45,7 @@ def test_smoke_convertall(tmpdir): @pytest.mark.parametrize('heuristic', ['reproin', 'convertall']) @pytest.mark.parametrize( 'invocation', [ - "--files %s" % TESTS_DATA_PATH, # our new way with automated groupping + "--files %s" % TESTS_DATA_PATH, # our new way with automated grouping "-d %s/{subject}/* -s 01-fmap_acq-3mm" % TESTS_DATA_PATH # "old" way specifying subject # should produce the same results ]) @@ -106,7 +106,7 @@ def test_reproin_largely_smoke(tmpdir, heuristic, invocation): @pytest.mark.parametrize( 'invocation', [ - "--files %s" % TESTS_DATA_PATH, # our new way with automated groupping + "--files %s" % TESTS_DATA_PATH, # our new way with automated grouping ]) def test_scans_keys_reproin(tmpdir, invocation): args = "-f reproin -c dcm2niix -o %s -b " % (tmpdir) diff --git a/heudiconv/tests/test_queue.py b/heudiconv/tests/test_queue.py index a65f4532..58f5addc 100644 --- a/heudiconv/tests/test_queue.py +++ b/heudiconv/tests/test_queue.py @@ -10,7 +10,7 @@ @pytest.mark.skipif(bool(which("sbatch")), reason="skip a real slurm call") @pytest.mark.parametrize( 'invocation', [ - "--files %s/01-fmap_acq-3mm" % TESTS_DATA_PATH, # our new way with automated groupping + "--files %s/01-fmap_acq-3mm" % TESTS_DATA_PATH, # our new way with automated grouping "-d %s/{subject}/* -s 01-fmap_acq-3mm" % TESTS_DATA_PATH # "old" way specifying subject ]) def test_queue_no_slurm(tmpdir, invocation): From c2ce3c4486398729b760d0505250a3d337b84d16 Mon Sep 17 00:00:00 2001 From: auto Date: Thu, 3 Nov 2022 17:31:24 +0000 Subject: [PATCH 033/176] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac671e16..6754a43f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,28 @@ +# v0.11.5 (Thu Nov 03 2022) + +#### 🐛 Bug Fix + +- Fix certificate issue as indicated in #595 [#597](https://github.com/nipy/heudiconv/pull/597) ([@neurorepro](https://github.com/neurorepro)) +- BF docker build: use python3.9 (not 3.7 which gets upgraded to 3.9) and newer dcm2niix [#596](https://github.com/nipy/heudiconv/pull/596) ([@yarikoptic](https://github.com/yarikoptic)) +- Fixup miniconda spec for neurodocker so it produces dockerfile now [#596](https://github.com/nipy/heudiconv/pull/596) ([@yarikoptic](https://github.com/yarikoptic)) + +#### 🏠 Internal + +- Update GitHub Actions action versions [#601](https://github.com/nipy/heudiconv/pull/601) ([@jwodder](https://github.com/jwodder)) +- Set action step outputs via $GITHUB_OUTPUT [#600](https://github.com/nipy/heudiconv/pull/600) ([@jwodder](https://github.com/jwodder)) + +#### 📝 Documentation + +- DOC: codespell fix a few typos in code comments [#605](https://github.com/nipy/heudiconv/pull/605) ([@yarikoptic](https://github.com/yarikoptic)) + +#### Authors: 3 + +- John T. Wodder II ([@jwodder](https://github.com/jwodder)) +- Michael ([@neurorepro](https://github.com/neurorepro)) +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + # v0.11.4 (Thu Sep 29 2022) #### 🐛 Bug Fix From 0d16623f158a5bee85ecd1b0a3082aaa993fb8b5 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Thu, 3 Nov 2022 15:18:56 -0400 Subject: [PATCH 034/176] Delete .dockerignore Excluding version-controlled files from the Docker image causes the resulting copy of the repository inside the image to be dirty, which in turn results in "+dDATE" added to the version number. --- .dockerignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 94143827..00000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -Dockerfile From 22f9ea1217a9cd806db440fc201e4a177a5271f1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 3 Nov 2022 16:59:17 -0400 Subject: [PATCH 035/176] DOC: Set language in Sphinx config to en Starting with Sphinx version 5.0, the configuration for the doc's language should not be None anymore and causes docbuild failures otherwise. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 4743807a..f515c78d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -63,7 +63,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From de9b4c65a916a5458a9dbea0e992bd0b175d87a8 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 3 Nov 2022 17:25:48 -0400 Subject: [PATCH 036/176] update copyright and go away from m2r since not compatible with most recent sphinx etc yet to figure out a better solution -- for now just include verbatim changes.md --- docs/changes.rst | 2 +- docs/conf.py | 3 +-- docs/requirements.txt | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 3d2cfd1a..4f28b30e 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -2,4 +2,4 @@ Changes ======= -.. mdinclude:: ../CHANGELOG.md +.. literalinclude:: ../CHANGELOG.md diff --git a/docs/conf.py b/docs/conf.py index f515c78d..8e281ee0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,7 +22,7 @@ from heudiconv import __version__ project = 'heudiconv' -copyright = '2019, Heudiconv team' +copyright = '2014-2022, Heudiconv team' author = 'Heudiconv team' # The short X.Y version @@ -43,7 +43,6 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinxarg.ext', - 'm2r', ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/requirements.txt b/docs/requirements.txt index 46f4b641..80151ce2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,2 @@ sphinx-argparse -m2r -r ../dev-requirements.txt From 057c8d33c8284c2efabc7c1deef71b8c338a3a56 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 3 Nov 2022 17:32:56 -0400 Subject: [PATCH 037/176] Make html_static_path empty since we do not have _static --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 8e281ee0..1e15db19 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -89,7 +89,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = [] # Custom sidebar templates, must be a dictionary that maps document names # to template names. From 2f7086544e4e9d86ce0a8357406019fa5c9d6620 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 3 Nov 2022 19:07:01 -0400 Subject: [PATCH 038/176] Fix numpy docstring styles, use napoleon extension Not sure if all were needed but most were. Still have 2 warnings locally cannot figure out --- README.rst | 2 +- docs/conf.py | 1 + docs/requirements.txt | 1 + heudiconv/bids.py | 39 +++++++++++++------------ heudiconv/convert.py | 56 ++++++++++++------------------------ heudiconv/parser.py | 16 +++++++---- heudiconv/tests/test_bids.py | 26 ++++++++--------- heudiconv/utils.py | 10 +++---- 8 files changed, 69 insertions(+), 82 deletions(-) diff --git a/README.rst b/README.rst index 8f62c1e2..0ac8ab7f 100644 --- a/README.rst +++ b/README.rst @@ -54,7 +54,7 @@ Please file issues and suggest changes via Pull Requests. HeuDiConv requires installation of `dcm2niix `_ and optionally -`DataLad `_. +`DataLad`_. For development you will need a non-shallow clone (so there is a recent released tag) of the aforementioned repository. You can then diff --git a/docs/conf.py b/docs/conf.py index 1e15db19..cf550f02 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -43,6 +43,7 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinxarg.ext', + 'sphinx.ext.napoleon', ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/requirements.txt b/docs/requirements.txt index 80151ce2..83948e26 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ sphinx-argparse +sphinxcontrib-napoleon -r ../dev-requirements.txt diff --git a/heudiconv/bids.py b/heudiconv/bids.py index 40deb1e5..088cd2d6 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -1,5 +1,7 @@ """Handle BIDS specific operations""" +__docformat__ = "numpy" + import hashlib import os import os.path as op @@ -415,15 +417,14 @@ def save_scans_key(item, bids_files): def add_rows_to_scans_keys_file(fn, newrows): - """ - Add new rows to file fn for scans key filename and generate accompanying json - descriptor to make BIDS validator happy. + """Add new rows to the _scans file. Parameters ---------- - fn: filename - newrows: extra rows to add - dict fn: [acquisition time, referring physician, random string] + fn: str + filename + newrows: dict + extra rows to add (acquisition time, referring physician, random string) """ if op.lexists(fn): with open(fn, 'r') as csvfile: @@ -527,11 +528,11 @@ def get_shim_setting(json_file): Gets the "ShimSetting" field from a json_file. If no "ShimSetting" present, return error - Parameters: + Parameters ---------- json_file : str - Returns: + Returns ------- str with "ShimSetting" value """ @@ -552,13 +553,13 @@ def find_fmap_groups(fmap_dir): By groups here we mean fmaps that are intended to go together (with reversed PE polarity, magnitude/phase, etc.) - Parameters: + Parameters ---------- fmap_dir : str or os.path path to the session folder (or to the subject folder, if there are no sessions). - Returns: + Returns ------- fmap_groups : dict key: prefix common to the group (e.g. no "dir" entity, "_phase"/"_magnitude", ...) @@ -598,14 +599,14 @@ def get_key_info_for_fmap_assignment(json_file, matching_parameter): (Note: It is the responsibility of the calling function to make sure the arguments are OK) - Parameters: + Parameters ---------- json_file : str or os.path path to the json file matching_parameter : str in AllowedFmapParameterMatching matching_parameter that will be used to match runs - Returns: + Returns ------- key_info : dict part of the json file that will need to match between the fmap and @@ -666,7 +667,7 @@ def find_compatible_fmaps_for_run(json_file, fmap_groups, matching_parameters): (Note: It is the responsibility of the calling function to make sure the arguments are OK) - Parameters: + Parameters ---------- json_file : str or os.path path to the json file @@ -676,7 +677,7 @@ def find_compatible_fmaps_for_run(json_file, fmap_groups, matching_parameters): matching_parameters : list of str from AllowedFmapParameterMatching matching_parameters that will be used to match runs - Returns: + Returns ------- compatible_fmap_groups : dict Subset of the fmap_groups which match json_file, according @@ -720,7 +721,7 @@ def find_compatible_fmaps_for_session(path_to_bids_session, matching_parameters) (Note: It is the responsibility of the calling function to make sure the arguments are OK) - Parameters: + Parameters ---------- path_to_bids_session : str or os.path path to the session folder (or to the subject folder, if there are no @@ -728,7 +729,7 @@ def find_compatible_fmaps_for_session(path_to_bids_session, matching_parameters) matching_parameters : list of str from AllowedFmapParameterMatching matching_parameters that will be used to match runs - Returns: + Returns ------- compatible_fmap : dict Dict of compatible_fmaps_groups (values) for each non-fmap run (keys) @@ -768,7 +769,7 @@ def select_fmap_from_compatible_groups(json_file, compatible_fmap_groups, criter (Note: It is the responsibility of the calling function to make sure the arguments are OK) - Parameters: + Parameters ---------- json_file : str or os.path path to the json file @@ -777,7 +778,7 @@ def select_fmap_from_compatible_groups(json_file, compatible_fmap_groups, criter criterion : str in ['First', 'Closest'] matching_parameters that will be used to decide which fmap to use - Returns: + Returns ------- selected_fmap_key : str or os.path key from the compatible_fmap_groups for the selected fmap group @@ -859,7 +860,7 @@ def populate_intended_for(path_to_bids_session, matching_parameters, criterion): Because fmaps come in groups (with reversed PE polarity, or magnitude/ phase), we work with fmap_groups. - Parameters: + Parameters ---------- path_to_bids_session : str or os.path path to the session folder (or to the subject folder, if there are no diff --git a/heudiconv/convert.py b/heudiconv/convert.py index 4e090628..362c0544 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -1,3 +1,5 @@ +__docformat__ = "numpy" + import filelock import os import os.path as op @@ -453,23 +455,6 @@ def convert(items, converter, scaninfo_suffix, custom_callable, with_prov, dcmconfig=None, populate_intended_for_opts={}): """Perform actual conversion (calls to converter etc) given info from heuristic's `infotodict` - - Parameters - ---------- - items - symlink - converter - scaninfo_suffix - custom_callable - with_prov - is_bids - sourcedir - outdir - min_meta - - Returns - ------- - None """ prov_files = [] tempdirs = TempDirs() @@ -624,10 +609,6 @@ def convert_dicom(item_dicoms, bids_options, prefix, Create softlink to DICOMs - if False, create hardlink instead. overwrite : bool If True, allows overwriting of previous conversion - - Returns - ------- - None """ if bids_options is not None: # mimic the same hierarchy location as the prefix @@ -666,18 +647,18 @@ def nipype_convert(item_dicoms, prefix, with_prov, bids_options, tmpdir, dcmconf Parameters ---------- - item_dicoms : List + item_dicoms : list DICOM files to convert - prefix : String + prefix : str Heuristic output path - with_prov : Bool + with_prov : bool Store provenance information - bids_options : List or None + bids_options : list or None If not None then output BIDS sidecar JSONs List may contain bids specific options - tmpdir : Directory + tmpdir : str Conversion working directory - dcmconfig : File (optional) + dcmconfig : str, optional JSON file used for additional Dcm2niix configuration """ import nipype @@ -723,18 +704,19 @@ def nipype_convert(item_dicoms, prefix, with_prov, bids_options, tmpdir, dcmconf def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outname_bids, overwrite): """Copy converted files from tempdir to output directory. + Will rename files if necessary. Parameters ---------- res : Node Nipype conversion Node with results - item_dicoms: list of filenames - DICOMs converted + item_dicoms: list + Filenames of converted DICOMs bids : list or None If not list save to BIDS List may contain bids specific options - prefix : string + prefix : str Returns ------- @@ -877,15 +859,13 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam return bids_outfiles -def add_taskname_to_infofile(infofiles): +def add_taskname_to_infofile(infofiles): """Add the "TaskName" field to json files corresponding to func images. Parameters ---------- - infofiles : list with json filenames or single filename - - Returns - ------- + infofiles: list or str + json filenames or a single filename. """ # in case they pass a string with a path: @@ -908,15 +888,15 @@ def add_taskname_to_infofile(infofiles): def bvals_are_zero(bval_file): """Checks if all entries in a bvals file are zero (or 5, for Siemens files). - Returns True if that is the case, otherwise returns False Parameters ---------- - bval_file : file with the bvals + bval_file : str + file with the bvals Returns ------- - True if all are zero; False otherwise. + True if all are all 0 or 5; False otherwise. """ with open(bval_file) as f: diff --git a/heudiconv/parser.py b/heudiconv/parser.py index e0678762..fe3cd808 100644 --- a/heudiconv/parser.py +++ b/heudiconv/parser.py @@ -29,6 +29,7 @@ def find_files(regex, topdir=op.curdir, exclude=None, exclude_vcs=True, dirs=False): """Generator to find files matching regex + Parameters ---------- regex: basestring @@ -60,7 +61,8 @@ def find_files(regex, topdir=op.curdir, exclude=None, def get_extracted_dicoms(fl): - """Given a list of files, possibly extract some from tarballs + """Given a list of files, possibly extract some from tarballs. + For 'classical' heudiconv, if multiple tarballs are provided, they correspond to different sessions, so here we would group into sessions and return pairs `sessionid`, `files` with `sessionid` being None if no "sessions" @@ -116,13 +118,15 @@ def get_extracted_dicoms(fl): def get_study_sessions(dicom_dir_template, files_opt, heuristic, outdir, session, sids, grouping='studyUID'): - """Given options from cmdline sort files or dicom seqinfos into - study_sessions which put together files for a single session of a subject - in a study - Two major possible workflows: + """Sort files or dicom seqinfos into study_sessions. + + study_sessions put together files for a single session of a subject + in a study. Two major possible workflows: + - if dicom_dir_template provided -- doesn't pre-load DICOMs and just loads files pointed by each subject and possibly sessions as corresponding - to different tarballs + to different tarballs. + - if files_opt is provided, sorts all DICOMs it can find under those paths """ study_sessions = {} diff --git a/heudiconv/tests/test_bids.py b/heudiconv/tests/test_bids.py index ca26c640..83fdf3d5 100644 --- a/heudiconv/tests/test_bids.py +++ b/heudiconv/tests/test_bids.py @@ -199,12 +199,12 @@ def generate_scans_tsv(session_struct): Currently, it will have the columns "filename" and "acq_time". The acq_time will increase by one minute from run to run. - Parameters: + Parameters ---------- session_struct : dict structure for the session, as a dict with modality: files - Returns: + Returns ------- scans_file_content : str multi-line string with the content of the file @@ -231,12 +231,12 @@ def create_dummy_pepolar_bids_session(session_path): The fmap files are pepolar The json files have ShimSettings - Parameters: + Parameters ---------- session_path : str or os.path path to the session (or subject) level folder - Returns: + Returns ------- session_struct : dict Structure of the directory that was created @@ -407,12 +407,12 @@ def create_dummy_no_shim_settings_bids_session(session_path): The fmap files are pepolar The json files don't have ShimSettings - Parameters: + Parameters ---------- session_path : str or os.path path to the session (or subject) level folder - Returns: + Returns ------- session_struct : dict Structure of the directory that was created @@ -553,7 +553,7 @@ def create_dummy_no_shim_settings_custom_label_bids_session(session_path, label_ - TASK label for modality - ACQ label for any other modality (e.g. ) - Parameters: + Parameters ---------- session_path : str or os.path path to the session (or subject) level folder @@ -562,7 +562,7 @@ def create_dummy_no_shim_settings_custom_label_bids_session(session_path, label_ label_seed : int, optional seed for the random label creation - Returns: + Returns ------- session_struct : dict Structure of the directory that was created @@ -705,12 +705,12 @@ def create_dummy_magnitude_phase_bids_session(session_path): We just need to test a very simple case to make sure the mag/phase have the same "IntendedFor" field: - Parameters: + Parameters ---------- session_path : str or os.path path to the session (or subject) level folder - Returns: + Returns ------- session_struct : dict Structure of the directory that was created @@ -890,7 +890,7 @@ def test_find_compatible_fmaps_for_run(tmpdir, simulation_function, match_param) """ Test find_compatible_fmaps_for_run. - Parameters: + Parameters ---------- tmpdir simulation_function : function @@ -939,7 +939,7 @@ def test_find_compatible_fmaps_for_session( """ Test find_compatible_fmaps_for_session. - Parameters: + Parameters ---------- tmpdir folder : str or os.path @@ -1029,7 +1029,7 @@ def test_populate_intended_for( ): """ Test populate_intended_for. - Parameters: + Parameters ---------- tmpdir folder : str or os.path diff --git a/heudiconv/utils.py b/heudiconv/utils.py index 5910586d..d51dd5ce 100644 --- a/heudiconv/utils.py +++ b/heudiconv/utils.py @@ -290,7 +290,7 @@ def update_json(json_file, new_data, pretty=False): """ Adds a given field (and its value) to a json file - Parameters: + Parameters ----------- json_file : str or Path path for the corresponding json file @@ -613,12 +613,12 @@ def remove_suffix(s, suf): """ Remove suffix from the end of the string - Parameters: + Parameters ---------- s : str suf : str - Returns: + Returns ------- s : str string with "suf" removed from the end (if present) @@ -632,12 +632,12 @@ def remove_prefix(s, pre): """ Remove prefix from the beginning of the string - Parameters: + Parameters ---------- s : str pre : str - Returns: + Returns ------- s : str string with "pre" removed from the beginning (if present) From 994180e0bfe79aad4d7a9e74214b37d53ada7920 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 3 Nov 2022 19:11:28 -0400 Subject: [PATCH 039/176] RTD: use python 3.9 to overcome problem with napoleon use of collections --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 7e347cbc..faf51743 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,7 +8,7 @@ python: build: os: ubuntu-20.04 tools: - python: "3" + python: "3.9" sphinx: configuration: docs/conf.py fail_on_warning: true From f78290b42e621de8b194714e25379fc86456ff2c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 3 Nov 2022 19:37:06 -0400 Subject: [PATCH 040/176] Finish final warning about backticks local sphinx was not specific and was just saying about bids file, not that particular function, uff --- heudiconv/bids.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/heudiconv/bids.py b/heudiconv/bids.py index 088cd2d6..56339fc2 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -159,12 +159,12 @@ def populate_bids_templates(path, defaults={}): def populate_aggregated_jsons(path): - """Aggregate across the entire BIDS dataset .json's into top level .json's + """Aggregate across the entire BIDS dataset ``.json``\s into top level ``.json``\s Top level .json files would contain only the fields which are - common to all subject[/session]/type/*_modality.json's. + common to all ``subject[/session]/type/*_modality.json``\s. - ATM aggregating only for *_task*_bold.json files. Only the task- and + ATM aggregating only for ``*_task*_bold.json`` files. Only the task- and OPTIONAL _acq- field is retained within the aggregated filename. The other BIDS _key-value pairs are "aggregated over". From ac478a2729aa47ee402a9f0be9385176f49a2bf3 Mon Sep 17 00:00:00 2001 From: auto Date: Thu, 3 Nov 2022 23:44:07 +0000 Subject: [PATCH 041/176] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6754a43f..8d59ff6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +# v0.11.6 (Thu Nov 03 2022) + +#### 🏠 Internal + +- Delete .dockerignore [#607](https://github.com/nipy/heudiconv/pull/607) ([@jwodder](https://github.com/jwodder)) + +#### 📝 Documentation + +- DOC: Various fixes to make RTD build the docs again [#608](https://github.com/nipy/heudiconv/pull/608) ([@yarikoptic](https://github.com/yarikoptic)) + +#### Authors: 2 + +- John T. Wodder II ([@jwodder](https://github.com/jwodder)) +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + # v0.11.5 (Thu Nov 03 2022) #### 🐛 Bug Fix From 1018c65954b153a1dc1a052623b0687226574fc3 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 3 Nov 2022 20:13:31 -0400 Subject: [PATCH 042/176] DOC: do provide short version for sphinx Otherwise we get WARNING: conf value "version" should not be empty for EPUB3 which I believe is what errors out our build on RTD. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index cf550f02..b5d762f7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,7 +26,7 @@ author = 'Heudiconv team' # The short X.Y version -version = '' +version = '.'.join(__version__.split('.')[:2]) # The full version, including alpha/beta/rc tags release = __version__ From 5db2ede88280635dd39f58d283e5bb1fb480861f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 5 Dec 2022 20:20:05 -0500 Subject: [PATCH 043/176] DOC: fixed the comment. Original was copy/pasted from DataLad --- heudiconv/dicoms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/heudiconv/dicoms.py b/heudiconv/dicoms.py index 98c14772..00692fe1 100644 --- a/heudiconv/dicoms.py +++ b/heudiconv/dicoms.py @@ -367,8 +367,8 @@ def compress_dicoms(dicom_list, out_prefix, tempdirs, overwrite): dcm_time = get_dicom_series_time(dicom_list) def _assign_dicom_time(ti): - # Reset the date to match the one of the last commit, not from the - # filesystem since git doesn't track those at all + # Reset the date to match the one from the dicom, not from the + # filesystem so we could sort reproducibly ti.mtime = dcm_time return ti From e7a240998623a8db54b1741085ca23126c20d4d6 Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Fri, 9 Dec 2022 13:42:25 -0500 Subject: [PATCH 044/176] reduce requirements for date and time fields in dicom headers --- heudiconv/bids.py | 16 ++--- heudiconv/dicoms.py | 61 ++++++++++++++---- ...1.3.1.2022.11.16.15.50.20.357.31204541.dcm | Bin 0 -> 316172 bytes heudiconv/tests/test_dicoms.py | 50 ++++++++++++++ 4 files changed, 104 insertions(+), 23 deletions(-) create mode 100644 heudiconv/tests/data/MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm diff --git a/heudiconv/bids.py b/heudiconv/bids.py index 56339fc2..3d039a86 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -31,6 +31,7 @@ remove_prefix, ) from . import __version__ +from . import dicoms lgr = logging.getLogger(__name__) @@ -457,7 +458,7 @@ def add_rows_to_scans_keys_file(fn, newrows): writer.writerows([header] + data_rows_sorted) -def get_formatted_scans_key_row(dcm_fn): +def get_formatted_scans_key_row(dcm_fn) -> list[str]: """ Parameters ---------- @@ -470,15 +471,8 @@ def get_formatted_scans_key_row(dcm_fn): """ dcm_data = dcm.read_file(dcm_fn, stop_before_pixels=True, force=True) - # we need to store filenames and acquisition times - # parse date and time of start of run acquisition and get it into isoformat - try: - date = dcm_data.AcquisitionDate - time = dcm_data.AcquisitionTime - acq_time = get_datetime(date, time) - except (AttributeError, ValueError) as exc: - lgr.warning("Failed to get date/time for the content: %s", str(exc)) - acq_time = '' + # we need to store filenames and acquisition datetimes + acq_datetime = dicoms.get_datetime_from_dcm(dcm_data=dcm_data) # add random string # But let's make it reproducible by using all UIDs # (might change across versions?) @@ -491,7 +485,7 @@ def get_formatted_scans_key_row(dcm_fn): perfphys = dcm_data.PerformingPhysicianName except AttributeError: perfphys = '' - row = [acq_time, perfphys, randstr] + row = [acq_datetime.isoformat() if acq_datetime else "", perfphys, randstr] # empty entries should be 'n/a' # https://github.com/dartmouth-pbs/heudiconv/issues/32 row = ['n/a' if not str(e) else e for e in row] diff --git a/heudiconv/dicoms.py b/heudiconv/dicoms.py index 98c14772..0613c692 100644 --- a/heudiconv/dicoms.py +++ b/heudiconv/dicoms.py @@ -1,4 +1,5 @@ # dicom operations +import datetime import os import os.path as op import logging @@ -314,19 +315,55 @@ def group_dicoms_into_seqinfos(files, grouping, file_filter=None, return seqinfos -def get_dicom_series_time(dicom_list): - """Get time in seconds since epoch from dicom series date and time - Primarily to be used for reproducible time stamping +def get_timestamp_from_series(dicom_list: list[str]) -> int: + """try to return a timestamp indicating when the first dicom in dicom_list was created, or if that + info isn't present then a reproducible integer. This is used in setting mtimes reproducibly + + Args: + dicom_list (list[str]): list of strings pointing to existing dicom files + + Returns: + int: either an int representing when the first dicom was created, + in number of seconds since epoch, or if no datetime info is found then a hash of the + SeriesInstanceUID (meaningless value, but reproducible) + """ - import time import calendar dicom = dcm.read_file(dicom_list[0], stop_before_pixels=True, force=True) - dcm_date = dicom.SeriesDate # YYYYMMDD - dcm_time = dicom.SeriesTime # HHMMSS.MICROSEC - dicom_time_str = dcm_date + dcm_time.split('.', 1)[0] # YYYYMMDDHHMMSS - # convert to epoch - return calendar.timegm(time.strptime(dicom_time_str, '%Y%m%d%H%M%S')) + if (dicom_datetime := get_datetime_from_dcm(dicom)): + return calendar.timegm(dicom_datetime.timetuple()) + else: + logging.warning("unable to get real timestamp from series. returning arbitrary time based on hash of SeriesInstanceUID") + return hash(dicom.SeriesInstanceUID) + + +def get_datetime_from_dcm(dcm_data: dcm.FileDataset) -> datetime.datetime | None: + """try to extract datetime from filedataset + + Args: + dcm_data (pydicom.FileDataset): dicom with header, e.g., as ready by pydicom.dcmread + + Returns: + datetime.datetime | None: one of several datetimes that are related to when the scan occurred. + + """ + if ( + ( + (acq_date := dcm_data.get("AcquisitionDate")) + and (acq_time := dcm_data.get("AcquisitionTime")) + ) + or ( + (acq_date := dcm_data.get("SeriesDate")) + and (acq_time := dcm_data.get("SeriesTime")) + ) + ): + acq_datetime = datetime.datetime.strptime(acq_date+acq_time, "%Y%m%d%H%M%S.%f") + elif (acq_dt := dcm_data.get("AcquisitionDateTime")): + acq_datetime = datetime.datetime.strptime(acq_dt, "%Y%m%d%H%M%S.%f") + else: + acq_datetime = None + return acq_datetime def compress_dicoms(dicom_list, out_prefix, tempdirs, overwrite): @@ -364,10 +401,10 @@ def compress_dicoms(dicom_list, out_prefix, tempdirs, overwrite): # Solution from DataLad although ugly enough: dicom_list = sorted(dicom_list) - dcm_time = get_dicom_series_time(dicom_list) + dcm_time = get_timestamp_from_series(dicom_list) - def _assign_dicom_time(ti): - # Reset the date to match the one of the last commit, not from the + def _assign_dicom_time(ti: tarfile.TarInfo) -> tarfile.TarInfo: + # Try to reset the date to match the one of the last commit, not from the # filesystem since git doesn't track those at all ti.mtime = dcm_time return ti diff --git a/heudiconv/tests/data/MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm b/heudiconv/tests/data/MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm new file mode 100644 index 0000000000000000000000000000000000000000..65030dd230001d5bdb16231daa96b9624f4b8495 GIT binary patch literal 316172 zcmeFa3!G%vSs!@2t0axuk{QXe8G~V~N7x8kQ@-!2vLsvm)HLpwx_U;ERk*3Hx;;~< zx~g4|(G19egs_mXWF-UPkz{wB1+oc`uuDk1KVV~$%|ZwXuLMHCtN;V%4JJHx0W1IS z-1n>do<65#0$Dl^K~ zBK^hB%q;$1S$mVBRrPA!RI8e*s`aW>)v6}`vHy{xgfuqN7*$P2LQPX^NTXLxT(E0u z!>;P8u4|fRYo=zKH3MG&q?t7nxz8i_1Bnu-JPr5m$G!X0$hh-5rFr?aO7oe`C${!B zcJ`iLFw})}${os5b52=nZtbn@R8Yn_#|chL#oR@Qb0vS!HAAoN}bB zZ#|$id(NRV7*{Sk?f&F?Z{Rqi@j2xyaPNY$*uK>nIpc|Uv(oJJoI%II{lBg>cOO*@ z%T_Ck?aB4&2#K1Lk>gA%5AO~~!@*=&X*~Ib&+Jx=MeFJ2!&`$%d(;~~ytMG}hP_bH zSG04=i^}#UNp${v^YS@Jv}9PCj&@ehuWwb9rR&aMZ=*e#Tz8%yg8=&O^4_v@!|4x? zAnyci_)etzETuhYPcDr+!|7yi?2Ip|+8!9O*BOqG;&G%X19=UfU&QAJl#PWaHdl7H zHY(er-uSRxS?v+hefKq=RFqfn^NClKm+9|0W&6G>YZnq+h7o|xM%7TxDOYg+9ceyC zEXAPqaIMACOrv4JX6o#ks+!6tUb**WGR#lB@)7p;oU;2;T8h_aNcJrm*GSsqZQGTY z!?sP!H0>I=R@Z9E`W5G=DRnyp4zP3jUIJ<>rF~?ZDnI+vsqz%WMyr0dFAO!b7kl1 z+R_Tr{10HZf|TcBPj4W$@442peR*N8xv^s#hE+L_{%Cz`PQf+r(s^YbS16y#dLpd> zRcWer9&JrM#1`_LKfeH3HC=xO`Oh!Y=PF^T?q~Ei=M|;ml|D~@?^pI&?oIR_ug9md za(QK8`GWEwyK3OmS1ZdvH!2rU^!gUq!-ka$z+c~btunqfxHhb8>{MtJKpN@`?oZn| z?o?j4igE7w@0fWPke*~Etu}GJ{32;^duMC;%F-^r-|=Ge@|^P4gTB$#_R!B=P|mRw zPwXr#udQtEUQq7G_gRG`ejxbt`ry-N;!_!SH&-qw7lNyw7kqkiVH12tjzlUXXv>-u52e#HD}3wt~B>Mo&B2G?RG46 z-!vXqc4v$DLXRs~?g+W=>h*D@T!~7u4U3?jacvo$-rDXn%FY+0*n2^_^TiOy<=O@1 z8_{Ez70YO#2dhMSsf+mR_L~nyu72~2tIOw<_jYIh+SmQWdmj&(>q-TmGNXj;`fNuQ1VI!+7syUG45D2eq3gf zJ+soRgp&C6^3!Pr)9D4%(b!g`T&iV(bhU!@s1>b;RjgdN_J!oL3zlma)Sq2Ye~cvs z^k)~XuU)XdwSwvD1=EpT6{rXEhl->t*d9i~_Am;zhf$Ev@J0*O*DT0CvtWJAg7q~E z*4HdpU$ba^3-S+cRWbb)Y>#@u_NW)Mk9slt*euweZWiRhrcq4Kn+5r@S+IZJ%;lBq zafN=5?}4pl_)Jg211l?w!|tsM$~HW&b3t72g7PHup%%B6pGjlg6^U8-`1ePy`M%ay zFdq`}GwXu#k~jaueEW423!r}LMn<0JG0Q!tg!A;9_>PG1%F^EE*3QPldMJ%w3gwtp zc2+Ma`DuMAzXEArt8B0A>}?{Fva;9QUE8>z{Ee5$d&SSoAN#IfLu8?m;UnK-IZ2ME zE^n=`?2%?JD1V*hy!r!g`i|fK*{^w5S~kjco=LyGu(P|iupY}d|C7pRe&t8MBPSoc zc9iBuQrvQ;Kkz4eU+^veM^1WIOY2)p3*;@kIez_b{^7T8f5yMh&4IAc-lDd%(F}6T zKC}7z|G|7YCkG<-lw)gWC0@?PH^2RDosFN0CuW6Wv(M8zP_QgA{LxT>bpWXB(gIQI@tg7S}cx*i7<*(ql6fYHUQIAHSeH&(hOe z0sO(|JBfSS+Y8WO?!>SLTAe_aH@CdE5H-<$2}a z^Df)wmCwL^#NM#T)zHkAW>*`vMy;jW)w%)S2NI&hbBb*s`iThWymBAIqtfp`@AU>m zk;mti*9CVTKpxEZk>(A-wa*MbU4R^O=uJmXWqsHoY2SFBR^op8vu2e?pWYq~C&SLL z|IW(Xm(5iwk8ZXPop)9)ZeJfdgWk;zXVUglT&&RjwdHqKNHnY^u{CkVTVZVBDZM-(%+md>cyqu zbTBDejz+RqDk&%|FJZ-?deK*oS`_eNTIHciS!s{PRBGFHrcz^SjcR*&|DoU-c^?yJXw~V zaqk)y(uSh~xttBBJwR!Rq`quX%^>BNl6lH-8A;-M7kvHv6|S&b?O3I!{YkGmagKtT z`}K}-!nH=2;|8T4d+8&M>|!pAM(tZL>E?7lh8d;Lqsz`g8z}GeIsw<}E(Yb3#(f#+ z^P}tSeFy!|#fvvD2KLQvgtqV6b>yz!zQjX>@x!lPt^5nF*k0|L?HkVKa4=wkXlFH0TjJVfJaE+di7~Zn$5T zdj|(o>f3Sa{Yd6#NT?`Xs5>x1#~Gz=9UQQdo^smHu@4(gw>LdxA7Ojwq`lJIgo&~O z>ufrso-;;g(HZq1aZtf{(|an`mzTlnrBs*Vs@6-id}~l3RkPC`EKHWL?C*@0r-z5P zVomFo?!tNkC~vGddKMu2V;zNL7RDr{5+}d$|LuPGVUE=Uqeiu z&^DqnVJ&>n4aAAsqgPt0FWeqEL8b`l(&>gEr;i><7;hD~t_#!2aDCV=B#2i_v`-uE z0Oi~rwFlz^CvTAOGEmAUh7zBMX)E`u;AJ3zc89CI{$V((6wGwB=R6+o$1Z}j{&@wt&mNKLnO9Og7*} zo-D2bKs zU-Jq<`?-R!kuV9mMX7=o@~P}QOoT@xa(gr(YYc}1qHx%?BU{3I@`xSjtCKm9=NLzc znFyOKxkEmiMNzHY854{)=<%b{lV26RbNd@9l78Zv2$UBp%QZ@`-JTkbo`W%Nw84&; zS@{I<)3tk()!~Sye!Cd2K$D#Gfki##=3r@aJNG8a?mLx3%zUOJ+FA)%wEd2AWEoBo5)(kZWy^ZugEHxR5toZ+YeFf_FPK$48g??eUlrPwZ@6*$(uw)b4kteQziU4Ks$m zys*0vNB2n>OZL>l?g}E*JMZ?uK?&#{5BE4wR+O8bd|YO_tCdy`w#l*~XOaI`jl1`lmIADCpuv_X;RA89nNK?hSer2X#j8jHO|k zSi*MZ(Xh{MMD23r2CN$*utYn^?Yl}BFTOegK(GPEceFL6=x9OpqXMD2tj=*gh1`jA zGvpBB4UO^`DpOXQJ$1EDT*Bf-vn=#)a zHcYwQfydx@qJnZT&F@{KaE8~e8R7&(lPkNM=P=vdi!>8Cgl<-hWrJ>eb%DaxgH^=a-X4LNX z+Wj0RpWpU++wavIc#1)p?llI!-QJ=yq-mVn7 zRPnZ#JmznTsdR<|T*LC)V{hJA0pQkVbN9_}NB!KZ>saY~J0a`^m#UMSlM0HToA$j8 z33pc>tK97)&P|%m`=<0XG1h$?3Q)1z7cct358QrpYhhuXJ6S!|q(t@U;2de)3*C>$I~np!K!QVQ=g#biZUep0LaSg)ZR{jwj>r!r?v^ zjk_xy>=p8_`T%PbqzELMcD!}FrR%g~hpor4uz;IQh0xQ^+L*%l?m7fLG#AOeut3v0 zc0otVg#}kB!9~hW7Y|}r;!CNqTU!H){b5NKF;^_K(9RIATGI}mO5JWoQ}>_3(%Y4z zjrO&Hm#Du0rqR{`?|aw(zC4^ld+t><6KlX-xXDeY<6hL~`n$tPJAK2P>q7qK<_=3| zsHRqH;CXpG^{o4M_O5QNW!#b$ssc zA_x|yau+P9g@eST`_+E$2)h9K;Z<-0&#-$L`(zkv%kDRbjzu5Wu)J%!4GfA-ZB5xq zP*5J`$z7wn-R^`U>>jp{7F7xyx?!atE!}|a^kWwopQ2UhZ;h_C2Ui9NqpjR*V>Op4 z$jj5+?TyD%XIFE-lTG=Tbau(2=35)wNYZ}VhxG58VpkePx&qZ`bACmaD8R=cPcil>pQNe zzzTunDo(yIYI?1%skW7oI@Dk!pGGw%g9}&Y*82S^+fm2L@X*b}^N zJR^dq!57o$L>65&^IGpDJTt`8w7NwsG&DQcYO>+TH=%AAf%l1|2984dZH#^4n%+>s z)`qV0RX7G0G%@-cl`IecbMh~S7qqcv)oYfis_6cSl4Z{VKf>I15Qvl0LA*+ zlbht(NF%FeqhUge_`hCU#%Cnka2(HXUuMH~qpsI%>XA5%B!L$+Fq*>DO#_cX)XYY4 z4JP2c_Ps>%X?oo-(8LBzxp0dl@WS?0wT9lnDFhA6)N@C^R0AjAyw+vn={P3HLWizd z`J+`fUX;<|Hq1nfLi%m2RU1~rR8eDH<5XY_E@%-X+tkLEk?*yq`kFLcV9AzEr!Y1eDvsde zYvY)BfhFrWCa?xPm>(wzV|dY4#;{q(iIM26RS^**rmUjBq3)|e=e*_CIzGb_^{9_S z^IMn3y1I=MvFdPsF-)CGv}c31MR38wIEV3AZ{!>Bv4REdioqBC9vymFHyinVbOu zXe|mDDo%%VWT|l9xvCmN3kav-{w{oXU-aEB&}ha&LuT}9LiRL1a=k#_9e!F#toc`4I{r3+gs*O@Xtt8E3uKikII?BK&qkXhpyEKgVcFRC)H}r4BJF`QoQ^{+yVVKG#nyvF zD`gAOU(|@{K|x_Nw#S>(PT!g0cp~gf$k_}K$`wlv{#Penb|%=xi`r#u8uP1{N{oFE z{?pNG*skKXb=ZXI^s6IxSIsTA-Z}#5SSiVop2XP?a%3waRm1_dx!`OC+7cJ60JFld z#8Wk~Sc`=`ikEr~oyxzT&R)1=+S)+-L)_gY03r#v3eQA1W%-NRHlE;#NztfP)mj~W zJPR%4svs^!45zRxf%3ymJZ_s~`%b>{GjbIyaX&6Ja!s9_ZZg&8Sy`*-OtAlhj`-TW z8MGsUMb#SS@s@3xVW(FN5KmO&AQyB~toLBOd8~1Y@iSA_09v}x^g!zr8!6~XaO^6& zS+F%SrF_qOJPgQ;d4f%A;-ZsY*>P_!7xMbmZ~!Sri3F_gf0%Z?itY+=G3*A(x0cK*Cm}t5 zg&?rd z?M({vbG2X+(yKg!#2zjk@Af%nNX#TfvSX+=vD47ql*%hzCX~CGRTzq7?>je-+ZoAg z4k>XXWsu8)?a*wb!^`2YhXyM~wvE^&XN#R!$l9SdoN8FIMGUfrT_Q5e(87w)3=un8 z5Q{UkI@_tmMZROJLdAGhp<;9dQcdg+#KODO+UW7xpagV=*zRL%yj}LOwV`4nwb2YL zrPF>smD^&EEyf6ILyM-wFDyGFvQ1mIN1-j88Z8xgpZvkM_!BM!RGm@vhWjLpe)$qhN`?Wj%lJyjf;&Tc?!=p<5P%z zI({OQcxAo0l2@Ki+tPe$dTFVUJql!qbd&TvS5U_|Y@{PPCY2d_L&v1D3Z=3OrK%N5RWFpPk&|kB z_h~QAoY#J#+7`7i$i|Lqa1ituSNF4nHF>EQaPH9!Z|Xu%x_P|INq&H?)PKIweL4@l zVk*Lyn|T{YDC0CsI)Xp)Y#*G9x$%y-;^9uN{pW+xM|(GL)S&ycDoyHka2lXH+ai(u ztn76G6dmv40#0{Ua@SDZU_2AikEZ7( z&K#V=TE%3+=Cl04%rBr%v5BSj(bXXq9`Mj_UIpM_1zDD#clv!=Da}jF`lNJn)Rt9} zHgjdc`Mc%f_s6l8$}18BDIV!&bNFcwpRgtnUHORJ!}3?|z2uTfEaOw{3Am7(cV`tx zKj+@vSdKn;9hY{Oo=e{y_VLsy7A2UHz?fLs(PxjtTi^piuk<1z__-1wqt9VG3j*d<}Wt)_exoeBJ*q_~- zhCxWF_3L& z3E5KR>ef}3PPg%%;OuntSUS!f@Kfd^j@{GkA%b@#0}YsXqI)OL^2pffzHi-zhNBp8 zVeNzr%O~7F*5d*GieDf?C@2sQ?+>3R$M9I#AXSh#lGE>hwCFW&sXlhiU!0Go^Ooad z=`6Rui(RubuGKQG)ibU&GOlS}ry46n^P{&>3AL=MXjyg9vMQrx)ke#zj+Rv)EvrIW zR*m$m8tGXz(z9x$XVpm0s*#>mBR#7|dRC3}tQzTAH8QelWCS%Dvrf-FO&ezH>&a7u$ zDVrul)4JNiSjona!qprsj61~><3O9DiO0Lm_9Ur0cbtfb?WUSyYf(Em|76VG+>$}J z$aOr+OXr~;D@-~$wdNp7;Orz_9GEj9m);!#t8jIq2?scue%~48mkoAgr1yJowyBDQ zcWXG_&ec-{#7&jZ(++aFAnox=|1eJ=h?7W3dN>}7eDf1hVE%hT3e0^^NP&6p2`Mn= zJs}0=yCVI^S_X;)A96^|R62!hMwmX~7^{p*^xX}fL2xSS1I4h`cH9^JWnqvWrbn39=G;WEz!|}b zn$W>=8EIg|tO1s%cpCs6ET6Lx6K+;SrerR^<1?9l-`zaIA!F4ps~!x3HXv2+WpCVH$iKVh>?=jjZtEwj^vZY{b=>9Db| zT<3jEHkj4~N6D@Zr#Pr?goEn*!{M0UTxc zC6N-`sWh#D`S9*3eK?M+*>CT=Ekw5O-`-)T-BR$v+b5mAiIV`GQQyJw=3~GxJcbiN zVPa7k*M}|(%&-De9Dq2akc1x%(1Ox&R3B|uj_ZkGqc`&;#D-z^UZj{VQ)%4e`%prZ z#_GrG%HVy{;i%*+SF{zkW*z)87vjfjLNQ+{6jLLv zp-rp(mF=}ie+Xe?cXy|w{oyh9S<9c8$bD;?X9e*RrhWHN&7IZe5jJ}JL}rg1L?!X~ z3}f9{VfCN`0b{a)&2BlYH;ghPii4TVFgT_&mcLJgg7SL^DC-JpAjoCf5&pc!Q3&tNr9M zmhd8sUb{oJ-5Ts($4DJSTiy3BA6H^D^T9zpH(DD%$L01MY-UzB@M7g-g=IS9(O^77 z0Udf7c(v$di>iW0j~UsyG#RgEFVLhAK}uQ^fC8d}2`yu>iJdnIxYeOSEBc^TB8Bf? zW+(A>R_5#7Gq5;7G`W8ccJ4%t<_BSh zG0o(IukOk;mTTef(`n64^A=7A_BLprn9YCVG^1I%q8UZCnIy}-(N*T8=gACPX0ta~ z)%FruOVCcRTK~@O)+(M9#7$N*CIRk^zUL{kqsW7Nuu1Rsu5*ZO6P^cqG<0A1=L_$q z8B>u3Vua+NV-ieu;j4m9v&ZBMf^T_`n9@_OxrqY#P%m=6v7ZC3H|CcUEKGmiD}X52xnJ&P9akWW$rJvdzei02G?U7AD9%O)R=}Id0iK& zJ0`mOMsU`6UbXNZX&m()3}}N>0+9Q?G9*& z*!{!qIG8cJ(*gH3d{CAj|0PfPX9>K5_cQK?n|s)tBLTBA=mxJ5NP~EHkB11}MSyoC zV^Zdyc%G8j1ItR)g;YCjygmSaL1e;>(|dfC_M6W;k$#M7ff*BB3%COQ)8+2+Qj?B5 zKlDz8ht+t;_T#8puYKHykkLIvm5&4PvE75VN3fU(w$Hk%B_4U@QM_SwNVD(QOwYaR z^#|;M-{*Ui&h?Bm%q>7P)X%oJC)VlAWtSW2v;`gQ|X_lcK4FL~w4)H1T-* z-r-@m^&z5uN|o$f&;Wt%VU9jnMlvv@^IVV?hMt?Yzq!2v18|!K)mT_q!;|y~mAMCv zSJyFgpijj?-L3|)(}G@j>xH=3iQlv{Ya@3Z(KhGW-5}(;>|8r8(yph|`T@r}xYWk7 zkNcKc=27JYFs-?14}`)7rkl6)=5xLLRmHh+_^>#wq*UGSh?HQZV@Sc&XuKP5fm+fo z)gIYg!uvQ=S=&1|Lon8d=ZFV|B#TYuWea-l`TFh{5VR$8;KG8K)S1~cADS|+0PJ3$t#kTg0xYgb?!_gnRmEPAfli=E#(m!r#A~6u)V<$A z!yn;|Hh3XPX!M1U+!`z{tjzK4(~FZgLBu`VEK00U{Rw2Ym3V1* z6vXevVb<0p(g5pUwd#!ODP=umxFGmPQt_ zvNYNxX=Q~xAL_wFmL;m5G=`d`n^vPvM;P<9F-tDV7_j$Af-w+^Bx4{bDaJrRxyHaZ zq5?{SF#sjO2wBFEMiw!KG+N3Sbk30>4Vg3s4JZ3+ctDp<r9Q7lp9BaUC98REIYvAjFXfvd^0|t~cvW7IWh&80qQr2K_T0MrO$+Sp)X8O0Wh(kz@@7CB+&DDAyVa&icJ^9#9gj0Vro=4QXT%Ye=J|tbupF zl!i=N!zCQrLoZmd_@;P1uq121{%r}?Kqz_Ez&SZ5s2pqHQxpN^S_5AX1SkpC0F*Pb zhBUH>HKfs!)==Y}oTnj^)_^DfRh-3Mr)9eA&r)@2Hr_@8Zu=KI4BsWEU^PXIjq44mShchE=Phj z5Q-#gASfx;KtQ?Hz&FeQN`f^2<&3N$jVxjfX|$v@SQh8JAq|`#2nIS zNprAyU8M|2Qor~*MjAL}XBxfC6TvFl0nOLuEV(2@!*N9t42@7E85%*!F*HKTH8ei! z2b2Uu1Iig0S{hlz(9&orL*w{$>@yx zWf)lClrU1@u$yCs3QofyEJ=nzP`QS|7bc-9$}kMToRwjup@j@14VN+u-USX)Vxsmb z!>BcE3>Ox?Fq6kHd}LXMfdxbvhCwiSBZke%K*Gu~4E+>Pd4_?r!?-jLv1AwqU=q}j zwF-_?x0298hLMI#83ylKM}~+=!_ctH(PA&B=WZURS6POEMM@clK`^omgRmqS20`T+ zhRHVr$GL7&37B8ht;5pGvI{IQ%CHN9k!2TzCCM%bD%URfRw{A4rxd#Y%vsq*8d}IM(r_ue zsPWoG8Z&7ZHr}R;*TENk>d4=>>jTTO3oJFtunU68wF|yQ3>+n!Zx^S4%C!s5xgErk zFuJoSU0l-4dJS`Lyw{6w#PFeISq7FJWmpEm$g&K= z${R5_ErXzPErV|b1IIi|u?)bRm1U%%g)Actm$Hlo?}(AcOj`zCmx9--X(oUBNcE+# z0x82X2qw=mIG?)1q2jrg!Ko}l$}Sw#l4h*f03rL2N?F+Bq_Wfhu+cPrZ(o{Zv}xCGF$>;j9B z5=INWXyusEg3~StOO9O-Ql4G#O@XL_GVB5{XJr>@Xd$~u!=>zkcP*Win5ccqE_6&- zunk@}_}hm;%d!isn98sVg2^2(IIBolId;M4`rr*n`F25X@8k&55S9$X08D}!vQ|b& z!UVLCVWimts|_=(Sp+p226q(((ED)Eo2vIxTIZ}yiqPvVpi?Nqg)xt zNsGH=V>+)I*zU+T5cx98GC3@5$}l;Ck!5m(l`|0WnjAspnViAtpaDyU$pI#dbQ0oR zc%x<#TFB(ma4D1HjdNv)m@*7gQ|-EG+MHdPFRd)Yz{;i!!yuSk!{FPVgO{V`8wO`= zfS_^>gH!Xsl3^HtIV;0RLkk&38ZKoR+@Y>CX38+|_N}^Bx2y)|+{K5MWf@pBm0=kK zBg--fD{si)v@Xd$~u!=>zk zJKmMXOxlH^Vqrwrc&C#-uq3-Mb*n`B9U>5{)Ms|?_ zEn*iLa4EaseQqQJGieuCG_)JKhGBy%Dit8hG7KzmN*FK4g@hzbKnod08ZKoRyibl~h?q2tO9uAp>J6jD z-_)$~vJ3-DkunT}U}PBvVM#I!g32`vzMY8%uw)npU=r+=W*BK`A;U<+r3{1j!4Xno zqV_4nu(6`vu(Uc~lqEoxWf)j6lwlYIlV=#5rz04Im17th=NyBe@(hD>X_dsvGz`uL z1WbY&(hMUFEo2yJxTIm|yld*D#H`wjuc>DsCrwV*Fnhsir<^XdFS0C)!%CzKiz66W z7Drf;ERLXZEsj(3z>;BcfXU*Vge7$&4J~AGX}FZd@h+)nh?umCx?x*J9S@K2&0Tz9 zW!VMRA!XPF!Q|Nm=K)p*VddEcr?veRN0 z+9&M-Z%nAw8?}0!v)l5KWf=yRA!Qf_!N@WU!jfbd1eI$ToO2go$uJDSB&Z?HMN2~q z8Aci|Wf;5*>P8wfWf(eMu2ZwIO2Ri{_{g#h1M84741-|u41;r>b!!Q7opoX*&BMmKN7-_haVel@f8)?k6VW_HV)f;ur$KC>H zNtR*CSyE>(d6vN$qheS&mcch-Fr+-g;D^ez3%>XpNpwbbkpV4Y7a4F#yU=)pU0Im4 zI4B3W1Uym3-%d$8uo=O;tOoEYRafBtu;s`0v;yCpVC>a(9n6t9D zG_;V#rQuQ*#~bV-mLzJQwhJ5xfrI3AoiBhBK+CcVtjEf*3xdhD3(hJMR*qe8l9`b5 z?1FO$0hA2808D}zvOF|14J~9BX}FYK@CLgwL`>O*Zn4*nXna$b09lq(D;sVU{HB>!FfUrVPzTyr+oqDtPCR!Eo2yJxTIm!xo0jJ zn385LY2=j2X*k@mfk0Y~b13qmW!W4SIAz!z!N{^X!jfcj1eI%ZoV^{eWY`>F&dTP} z&_XtshD+HT_uM6onYN62%`{9LHq6|EQ4TjErZke2`k4kIM4Pq2`bMr zOukbg2up@#0OqVLBMmKN8ELqbWpK}4(wHgBuxq-F{r8-2ZlLC2dX;4uSmBgm7z88B zFbGSMVGvZFVHlh)AFyN?24K$0Fw)RMhLMI#83yzV1(r=^*agAl*#+l` zSQcUB*ahc4I0ltx7o71e!pgJ@K7Wj063me0NM%3^*+qt=D}CqC8BFf3JXX2;;>Atw zS72p`$Z9XXw<`lVX>phEJ`Td zHO?W!hn8g-Sc;Tk83ZHCG6+kOWe`-ZWpG*C3?!8wvaEE$FYn6omB zG_;Umq~Vf=Veqc0Q-dUGpE3*!>*);AYfJsn8+K7>c7BY-9T+%Q!Zo^1prVImzcF|M58t;(dLknBR>gw)Zb7^6HrMkD* zo=ir){prNn+pCUyU+S=shE+6F8X!H3@c%)7wcVKvM-0NOTAEh3ExbInUX+J1v2-XTN zQKDA1Z8z*%-LCU@rvOV;s_43Is2ZRMRtjTIqEhv`)zDSdtQB6d3FQG=s#g5E0g7O) zunQ+vEA$m;r#i;vM!jf&BditBlC?7UI}4x))(V?5Vzn}>v=_tH%!c0J?*f6Asuh1{ z0TjVnX-Urljb)g1+cb5{=C^Z#OV#UiFrpPpI~RESmR?s4#Gv^*cMWijH z^fs(2d?yo+nqu?aDJquXZ&>_em?rLUXlaiF571gBe&Bg=iBkVT@ozTP(xr zFdAk}w`w?0g1=%JUdQQBMC+AyJK*rz*a3=m<9AuL48P+P%`WbD=xM)0tD6mYh?uPK zYntGaMC7+qLJ@2)J;Uv=)tai-8>jGE2`yPI{zfMh(Q2ig4$Vf7Sl9IWDY|Gq!|5oz z=_(w;2u8GG8BT}Rs9~c*%_w{(WJID2r-OgWqXR{>UP+(BteR%ShBrkaL+<{NUW%S1 zV&NBdM$`is(Tb&g4s?e)jz?+W4ZHjm%kVi0zg#?0F>#-R-VmLbU?O0M=fli~af*6n zxE!a15pA%f%R%7)yx6jipg`dmeMHk4F2^ZCkh-|bL2ufOH5g5u?HWDPr=6xx%y2mh zKT00a4`4)FEW_tOObYKr(N5t?=@~AE$?qc)j9|U!O?R;dqY0+QDpo7}d}63+f=l*^ zCciU9FrpPpIvo@eu(6N=BRj>Em%Vx~T=sMqocMh*f)TA)((S;yR}CwBSoEqFex^Cn zVi|6S$?s3mo3%tMmUKI)Q>?+CGPS~o8Aa-q;dK;#rZ^5GTCcR*f!Iw0K{rG?_>G>u zZ79LS!fytOR7~9Mpf`0S`i_Qa)HGc;PGj>KUdJgSK_rxDgQdL=>|RF@r>2?wQEm24 zj%0hC!ob7>9duBBti5P(Rxx$Pe5HO0B4&6UI=>rDQi(QL((Aw=hW936lN`MfgWLa2 z9AxkI_5AK8!H8BY>2;vREChg2FZC4u0v%4B8l6wUXmPiL4(*Lm5rY_Y^cb75fX>}w z1edHBe}fTh>ul%5XVOu|`8uiMCkU*>vE11m;YoCOt6Q8Bt#KGZD$>CQiHg|jH z;dB@V*7*?EE_^z6sA_i7LSn*litsqWh*m7=cF>Zuij6p$X%s$XH&U?-zoYP@=y4d) ziX|NntVA^6*JBtv%|;x08Z=48Q}{0gBU-Vf=RpIsg`;sBH7urc`$CBaH`zE=_<7Ap zi-~(4^z>As^RO)2#KFHhVv76~%kVr-v9v}~iMCkU^T6}SSng9B#wjA(iRY%=7UOsB z2}ZPHNzVgR#Je#ml7sVT`CBZ*^C5zMIX=!-4-y1j^!>;?hMG+5$I`HQ7=xF4O#~bZIw~Z$)ZiSD6@QJYR)?jgIeQD)p zr$6oX2G=+dd|6ojaqot6T3JGMwDBm)&Aew)3M#ifUhj=3c_2ml1}M56P(%a>1DpU+ zkUP{24^fa&5>W_Aj3`lA#E248hcHoqCQcLrkwO%V^a4ad?$Z&491P!z#d;J%5+h1f z7BQm4)FDh1potTOK%@`_o|6Dkko$B*A%{&4Q3y$lC{bC&h!RtWFj0UeP80%>LKJwS z0z^UX(-DOnU^zq~Br&2yWf3DvOdY~R0h%~b2t*1|;Mofh1-Va06q-=TAqpXh5hW^% z7*S&C5GD%H#EC*6Qiy_?k^oVV`*cL137QCqOyn)C8iExq5w^tCJTOh(8P&CAX12er3e9{AouBrf{c=gLP%mniOM2I zl$bh%i2^inq7aA_qR=C|0!*Ror=tqxl*APVB}SH*G-7Ott3;SC1SifHh9ZYBf+5<| z$7m%6X$eY@Fi~j)2@_L^IAH)MfiQ$3hcF1`iS#hB;3QA1mY!{w)x!{!AYr1?2offy z5^=%+P6A;FMGj#Q(i0&}EJ(>k803^i7=jWcOjH^{!o*Y}P8h&RAPk|%Aq+x$B7}(r zE4c`RoU#Z*P=bVsN+U>^m`cP612_qUArv`;L5NR;FtMN|7h#Z77GVfVkT6kc1PK#U zi8x^ZCxI}8B8M;t^@$KB7QEyl406gM3_%GJCMt~}VPYx~Ck)^u5Qb3X5C$PX5yHfR zm|TQGPFaK@C_%zRr4b}dOeNxk0h|QF5Q-eaAoM3fm{>5Ai!jJ3i!cNwNSLTJf`o~w zM4T{ylRy|kkwX}S07VEB3uk!kRoi3_Gz(5@8^OU=I_M zMvyQum537ta1sbZC~^pc5TFQQVqr}#!XT%lA&DI-CfLJ7r4b}dOeNxk0h|QF5Q-ea zAOt8vm{?eoi!jJ3i!cNwNSLTJf`o~wM4T{ylRy|kkwX}S07VEB3u|%_203LBhM)uq z6O~4gFfo;g69#Y+2tz1x2!jxy2w`GjO)kP9r!2w{lptZE(g+eJrV??&08Rp72t^KI z5CRk-Of0O)MHu9iMHqq-BurEqLBhmTB2E~UgfOwNCKq9lQx;(eN{}#7X#@!qQ;9fX04ISkgd&G9 z2my)^CKlG@A`EiMA`C$Z5+*8*AYo!E5ho1bBoKyBq!5PAlocQhWj|e{mmOv+CveB0 z#0V3UMvO3Vl?W4t;KT{TP~;E>AwUtr#KM|9Jq$Y}SQ23%gkTR7lSYs*F_nlD25=Gx zLnv|xgAkwyVPauTF2W$Eq#=o&<1E<2M5PfVOiU%>gaMoc!Vroa!XN}FLYP=slZ!CO zDT^=!B}ka4G=hYQsYIMGfRjKNLXkrlgaAbd6ANo{5e7MB5r&`y2@{n@kT5Zoh!X~I z5(q;matMPEpa@}NVNEW=Ag3(C5R@QcqS6QwCZ-Z`!T?SJVF*PIVGsfoAxtc+$we6C zltmbV5+qDi8bQLuR3c6oz)2tsp~xW&LVzNKiG?+}2!oum2t!bUgo#QcNSK&P#0dj9 z34|dOIfOw7P=qkCuqGE_kW&_62uhGJQE3DT6H|#eVE`wAFoYt9FbDyP5GEGZt2onoy^7JtDx)@1441*FROjH^{!o*Y}P8h&RAPk|%Aq+x*B7}*B zHMt0boU#Z*P=bVsN+U>^m`cP612_qUArv`;K?qQUFtM;E7h#Z77GVfVkT6kc1PK#U zi8x^ZCxI}8B8M;t0g4bN7S`k<406gM3_%GJCMt~}VPYx~Ck)^u5Qb3X5C$PY5yHg6 znp}iIPFaK@C_%zRr4b}dOeNxk0h|QF5Q-eaAOt8vm{?eoi!jJ3i!cNwNSLTJf`o~w zM4T{ylRy|kkwX}S07VEB3u|%_203LBhM)uq6O~4gFfo;g69#Y+2tz1x2!jxy2w`Gj zO)kP9r!2w{lptZE(g+eJrV??&08Rp72t^KI5CRk-Of0O)MHu9iMHqq-BurEqLBhmT zB2E~Ky(UiQ{TIgwrlB}SN-G-8B_t3;SE1Sd`yh9ZYB2my)^ zCKlG@>0#L0FeMQNLJ0OSF=+$|6H|#eVE`wAFoYt9FbDyP5GEGZ zAwUtr#KM|fgh5VOgdr$F!bGJJBuq>t;)DU51i}!C9Ks+3C_t2onoyauEhOWf6v;1PK$BMvyQum537ta1sbZC{hSx zi$!`V`{^RRlv5I67?c=cV$z5aCaw};!VsJ|VHk=W!XN}F(8JiVuqICrLpfy;hM)uq z6O~4gFfo;g69#Y+2tz1x2!jxy2w`GjO)kP9r!2w{lptZE(g+eJrV??&08Rp72t^KI z5CRk-Of0O)MHu9iMHqq-BurEqLBhmTB2E~srwQj@PkK zuzoHI)wI9TcMhGw#8sWsWnyc^DjT9OrFuwf%hd_#I8edjuzQQIdL&o^vQLmp!=qbr z1oUL}8fkG!6&8@yd8ywX`=nW162_eNuMB$cn>z1y+-hIaY}M2(wPAC&^LY1Wv(xT7 ztL@HYI9l4Sc6VHhP^&ey#^2yExRofJS+n@1qt@LX4Ue4Br00yQ4t?Lfb>xc3FI@tT zwarF*Fg-x+rz2VUYvTW4VaGm(`)1E zHvZ4^O|-NS@r+dTT6eoWn&d&8RCI`dPmI&LWt|`{8Jl3KmS9C0>Yqz2wD+4d#ZP|;T$gZ zhn?rD?f&Y>aq^7PHs{8%)ZOD!Yfy4L^$r}Ooau})H|;ELtuLcbXm@+p4i`qF_N~0c zx&>%TdFWtf)Y)hs9rXs+;Du-_<4NzZJ@LFPRaLXnO*^r?toJ~V6}!j)o4vznA86g( zmE+|>F>5t`2)0r|C>4kZ8PxzB+*ty7dc&0)?S8fIUcu)mSJKHghg%2RBj?DS`ojQIN!tA# zr!$11-ddi*u+cr_X{lIle^Aa&-E1EovCisLWhHaR%7VRO4&A%qz-o8#@i`c!J2|?^ zG#I_lsJ}YwqkHpHF?sXT#MK<8>-$A5qQq-p+@Q-kY*sRyp|wFq+9syoX2I(c`S&-ISBuiwIOusiBqyXFNR)TFBV%g)jD z_83Zfg2r)#<6I`CH0rBiYtVP=l!o$?EIT*Qhpi2|&P`Xkc(vN;GMNtYXi1U9<=)NV z!0kvo>@yk3l(9KfK$vK}F=Ah~j)?DL5QcqSZBIB-KZQ}y4cD-$o`}9S*oGz}&2?gq zt)rvi*y(a-Lxc7P?fy#V`q1&3i^kq`d7+jPn!pWKy;WvX!@+TCISvFG6@}QI@j? zm`06^G!9F;;Y_X%BO&iN4kD!2;K+`8Um8-pQEjlFnprdI&Lx%2h~n9tWoIz#kugVm zvy_lQDtLO%;M(Lmla)>E;&@$(+XBsv+`1g8W>l%W>l{LnhKk0s|uzZEVY%@k`X1eyC-=U$E!TdAB_2hOGKE+tc;3G(%WR+zA|=3K@xQ?E6Flu zpR{zD=K~-#u`ycc-XQBo=ZL=6)jfH${p{;OiCTWvvDaVU2z$;dyc4ezjdhO(C(26q znzJ&%8cWAr*<7yR`^Cy!1wZe{|L>oB z|J=Ot+2@y=cPeVtuGg%2kWnTH5^J~pnMa8F! z=bM+$Db2N&jg`$NzCRe;dGq<^`aO#O^SOWi$}98ATX1(>QCob!4LdV~PFLrZ&p&?! zvdk#+%3Fb6Q4li54GO@rT=-zow*X^adE5Es?wb_tk;|*iM>JJ^+hM($6ZJ!+E97(&#n2YO;2EBuMkv`K4xA+1%P(nOEL*{>rYR{MdO#slR<;VZ&SP zQs05kyX|X_U3S{tJ+{1nZ3&}CR=0K@)2c>Q)2n(_)6`nk!Uq!<>>8GO@mW`OUDGsM zGchRZjT+QIzmIIJJ*FAeTD^`>Z0Y&2D}(Wo)9D@bI*;sbKZX=6it?CBZcE+JbW^Rr zfSjtK>Nc1MRQjsbHLc!IRnvOGtkyNlgcD#}ETLAfHySmxnSrkj+tgGO?N@&Ri&wgB z8LC}34E=>mIuv3W4c#ydx&=hNW|)w=hHOZvH*7;SZH8eq4BM>X3mT?s)NBY-#~t>o zz3|A^=vsTQIJ`+7!#kER7ji~dF_aHSk40pAI9lSvF`(vkc#{h^dt)zGwC)d6-NPz9 zoDg+=IJg$e8C4YWhV{pnNUee*mwShgsW132?zhWM@7nc=`+2d4z|;7-jib?HwHG4P z(0pnvj@rG!-elDNq_9G3z(Nf$ff|anA=p;0nJ-vX7^-HWZ^~jtt!_53_GmFusJ3ny zwyF^+)GB&4yH?lG)+ti3goci6P;9Mk=wO-sg059H%dqsC1!kU1iHKmJL|ER#QJkQ8 zq%a~>5;4NBI}swRQBWc^yQVj)u<95sG{Zz!-*_P;hi2B$xxEl!h#6&w?lWXoBZ_W-uzKbzL(ot7bM>_o11#Vb$@~Mt?zn$QD==tEOsz04Amn%+}FgXf?!0 z88xUIFpav7q?(C=(5Pz-({!m}>kZAUo9J1XWohVh^qS5v4D?-U4J^E*n^nUAb_0Ye zK#ov0^blczAW8%$XsQ@FSj}1Y5UG*dU4)YQb{^zZBYN_?3s!x%yYMR&6nWb2!tlx9 zyVC2!eMgWL(2v%zit!>ckTf*a(5+fcr}076tcFGHs#@qWR9l6Q(9klvjJjs1pq6bn zsP!#S0=BDd>lbK4WkMi0hCqO^t#p1t-A4ohHl0}H)9s-Ghlhx za97i6wh1b*@@&-NVZfm01AA@zg`kOowhaeRG$!XD5-JIK!@A>3q)I`Nd3}a$!$p9O z!al>)t97_4(M}^ate8GewnqECNxQ!{ZeLp$=7|CGJDDfsMqpI1T!+A_)$K-Itu=@x zD!L93qh^9MR@KnKAovH=sK{Yj=u9*dU+f0F2(Y42V`S1`9Y6))`G3 z)&*Z8LMD0*;=ivPBn&G}5MgbC!cN;~SQ%W{5#&Na-?NTQi8Qc9IUSrq9_hG*FwhKe_=}rB(C3#(D<5UPotdVK9Np5Rrrh zs|I>!atqL$7Hr-!Qk{cYZNN#eO)r~9ZiU*wOa-|t-9+yKr<08!k?z5$VgiajMK{3F zBK<=k8~TT^M0|+=1&tJR5EMIRb$4Bhutu(H5sI2(1>&-Sra*s8xAyOB(7 z7RjM#(Ej;VVE)LzdvNg+uiX1Gfq&wak9dFM-<2Kp1I;HD0KyqD6~ z?w?Oz+rID08e(CH;-DMCJP<=3;yds)@sItloKvnKjg2(N#DF3ZAQNM!X(Ldm!iz=j zFTQWB`3*B#OK%x1vt_mHR;^WUH4sf~XBytE zbfcx4E!}GAc1y3d^m3WOM4+)&Z`B&Dy4tF1t-9W-BW7y05HLj0w2s==TlGe( zp|%=YtD(0VMyp}A8deK&%|@-&fHFuMP=^YAs8EOsji^wG3Z1AFcZF6|s6|COrZ%J_ z#gXPnb)-8|9%+x%2mNVKARJ2kLptaXu{HcdI;asn6#msKk1IzXOfy26Xhd||cb@#v z=YG|1|M9z$7#EZ`X5{%wg<`l`s(u%gw`A1kt{0b@pQ|jd?5tf~S#E9btZgjpJk!EH zuGY> z?p8iPZ9zX@@unYryrNuvks{k#rJ_9b;uZ9VGq_Vx9(pPKeDx(a`i~U2dJjuMspwx? z-q)q%#d-nS^0O!kufv_STbCN}mm26%U|k4zo2fRgC~wF>sR8AkSU+)ngH~j?XP?;+ z4siGjuv|C@7^K`GpRQpz#5kTR>9|(>#C=K2JuY?WK3d&DKfp$rJJIH!r98c-1;2Li zN!!zndF8Y2qg99wEBET?VG zD{BG9Cd&hbfN5KdiIE5-g7Tq1pa1HQedl{W{9kr|?$K}jmrs583qF72J9nOFJ&y62l}*g=uuoLZwfma8 zKRNR#Mf7*DeV}sH-68wf#pNBns$P8OT;|ES9q*v+vY4`rCzLyqVFg!jy%UQokG|c5yyadZ)ZkLVGVmaa*X3rGg03QjJbfwH zW`Y&e%o1p&NJbG?v5g|L0PMm&2|u{c+UC)>`;G9VK%l2j{WGsj(F11p z-kV!H8!gsR&MP;-&s`<98TcKW%a394-`QK**mz9aTUy5dt%ao}t+h!*Rv)Wt3;k=u zcVHh0_QVWYYqTq})0?c($m;Gren;~%`1v)k-gC+&%RqF(fLHss1Ll4$vxVk%WdS#F z=kI{Y>M@obF>@HdXaA{w=ON{Ms)xIvY@>%O!wa-44=E}>pQFz$`kw4(A5!#G{)gaW zv;3-UJOnSBeYP8VI`2wG-W{Z;{^ud3$v$cLvQ~AP(iMKA<8X{4cKf@2MVlgY#)93rbk9xi^rs)(TW1AZ4uK&yS zUU(m2o(&{mQBRGz!Zy}ZQ{^1z+sgHXYmp)H)4>yd@4@QoSAFYyKm03~*NwmZw>Cfg zz`}>VxAp_KJ>k>%+qXU8PZ3WTAycfaz%zC|Vd}|4PZ&4rS)Op*vLj~xN%MpoYTyai z3_J8&-4XjC<+dk$J1TrTD(uhVZbyZ0M}@Pa!lgo+9IHT5Ddd;Tpw9ZvWiYz-9vosew;VcAibGX4u^QQ?niasz^1X&~x&kCZjDCUj3_5o8#m> zyV|(=%f`>XdYqh}iuam z9;ZtloWH7RU9FMyzWz?CiJva6`IB;e`rapfgg^0b{`F92Cx3GDgz0>Y=-+6G(d${C zdD3R(e|4@MFQuyuUy@Ka;V1td{SB{1KSfh~I(-f`7k)j8eXICzID1+XO*&YN9?z(l%QMs@wrJkLYBH$-P~dM?sp$M zTN*#jr^4S*a#tszOi|gRpIt4utRoGP=G-Tq!o0~(Gti}**8ShtXdKtg6-pFE@vpi$ zqIX@bD@$^V<3{_L-2aUm=< zswH20r9zr>xktRqs$mJ-b@8H_jFy(pe{Nh4R46SIm5T=GocrpH)hal$_F+ zm%gs_V(BYO*Gumzoh$vD*`wLl%zo?4TW5Ym87Xg9?tv!#A8Hvw_It@VzMIOIFw|vu zg;HeY@$p~HzoAs_r)b%JJ)^Rgl?%!h<-^MQ%pcBt)9iz#Z!G<8sapP`@|TytvHZdE zx0SyGzyDMD1NiUz}Z<{RcCzDDP7~Gg^SBN0*Dy8-9sV>F9p) zdo<4{>vL45xDH+Yk%!{f=MLp_lpSzydFFR!zIj$H{cojLN{#Yf`DXd6%P*C`sr>iw z`z`qUP35mIe_i>j%6|=aM&(}l+45@n!Se5xzPa?e(zCPQF!OUt6PolH>5lDgDWL@P zDf;7o4^u?%esVv%GDZJ)CY%8!<}p`UiS zTRtdXD__UIPWip%7JlDTe!6_6yo=u}<)`qwTiz@`QErq!yZno#=Su%-_77$rm^oD5 z=vCb<=}uCav+{Y>dI z%Jbznm(}vS%8U3{FTcI~mU0F9xln$fe7^iz_Iq#nHRXHD_u<-uWewV1DQ}c_p!22j zrE;nKZKXe${k@srR37!%HiOo>Te*vk*CGG>Pb61dYELY7HT?YQesZOAW%H%;HAr4Z z33a*)|7e~ut9-8VWr{j85B`07>5odUE#F`MOz7qU{(nvR6Qw^XeZ2Gsr9Z@-KaTu< zy!89%Cw~XOzlW=zDBTOa*UMiBK3*=r6Z$Wg|NGM4opokzVa#@|`c7pIdb|VWh0;Yo zx$=>od`Y8!UtRsVQuvZ&$y8>n!1*6cuzBU%m9L-qd$UgIC!ndnDBV%MtNaRd^O4ej zDg8YD{bK2tO21tCXzABWA7j72S^6#f{$}YnaOKx<{nzmKH*n?SrJ3>@%5P(wTq@h; z*Ofn1`jgq8pZRe-QB1m;1@B*jf36luKdv@?-T8lg$$bqnA(>v?Pp|I(zEr+6?%xvX zaW`sn7Zc`Q<$qMZXXfSE?*g0tE4cL+aFBnk^uwj^EB#RE$4ft5`pMFNfNp=f^q)%q zx%3Or>o1jFhDLt{zrTz=^OxA~%lP)I(Bto*4gR!rH!Ob%mfa}7zWfXDhwh#Icglaq z%5t@M7qmztjIYCdow+Qftl=jHB;h|QKl&5@=H7AjMW6lDuGD4d?r!*TbIigj%HLN$ zF!TMhUjq&N5&nG?n)#{Hhf3cG9e%j<|KRsum;Ue4zl9!uru1{*WT?Ymh1N)qFGG{` z`!e$W7PIf)F8zO{&xRH^%NFeWFG`E0kIa0t@_N?ymeF=|kbt!4&SZQE{W^qyeYwLL zxHXgczh7Ds)74&F0{0qu8Fzt&_bB(W8r`jYiE?%3%V5R70RDWe^s`{zzbgIn(s!4> z2mk+I>HmTr|26dZW2Ju&KK^v+KZ1=v2mAg7X5TM+jQmw*&1BIZ0~3EAzaIye-w117 zLC^S5`DaSM1Apgl!s{g4C42Tq7t$it!k5_pVQB83mi`HT|0(?De*r%J5OhdP{1K+dp8!YyBeeLR znI1pF`1i};1ti;yL9_%2z7ypZRCA-&*=h{Qf*x z_kG~sKQ4U-{(W!hpFxY?U-|*)^@pLwAAvppI4s)j7k?h9e-Rq|W&HjMZ1`i)-S49> z{J8h~$LLl692PwfKk_~B8{bxbuJqBFzlF1qaC#D2@LtcJNhV@q-mj~-@b1~_(PzD1 zid$}|L2B6&cyNxf@I07sk5X6uQ2Cr$3%y*a{82FOyU`2&V|X{;j=%pq^hv)Tf@c3E zboir8gFgjZChfVsBH6HO!M_dm{Q>iR{3=PKu=J0hzki5--xcWapM!}%gg){|!NH$m1`7|nj6eG}Y;`g_h4O1WZ zU!lD}XPo{^@bHz=o$!&~2yf8=7auHNExkPRbtv^SAOW?ZJF@sZBuPWQk^KHQw+^rN z|8Wia($KubwdDI?OZOlu^M5M8G4t8r)oaVY4Xge8rT@9~ZP4HcnGU~~Y4Q8P!ykk- z{~NH7H2AX^OBl06+}fq!~{VQL=aFg5dniR02Ne16iKlVR1o8a+qAiwu#t=Y~T)bG7l z2Jij<@16ZR(`V1#pZ$4qJ?mL|%ow;N!E2M5xRiMD8~=9ieXd_lG;%gD%*4^8NrhC1 zTI$Dt61{lj?DOy)9g}6DY+@2;kQp|#a!L`|P4Iu=Z1;fM{rV-{YWm65M{~{2(f(Z8!Fian$A!h6zTpo-GiU!&I>)pQi zIO3wDIXxIklK*I(Ep8Ue*z21KMyazRhl|vGkx2{2I+C{7xgUzwxL0b7rH}A=sq}(t%wD{xT9xXC}gK z41$w_Zoxf%Uhh$On3HRm9UjWdkNgFO;nBx%(>wpfE%qz%$6#3~dy<~(p93alB{%)D zlfi2Ri?R_*JnYZ)9{|ILj66Sqby)?jYmwuv$nh?$@j>E|Pmo|mB0m|u_nYs*INvZX zEy(Fvz+;9W6FG?_(93Lc6hTlvxGIdm|m^O-`qR!OE9FLiO z9y`0lVwrlsu~(^&M=VSOhLYs$9D6pWg;U18)tl#MAdYz1zsFw)j!XOp4Tg^+%d7Bi zp5v2O@oC;b4~m1I;{*Nx>XKOp#L|M7HsxbK7RcON(@C>&j$8Y2P95D!f2M2#eZmrB_1TAgZKr1Jh*;gTc zxxud>8MD0P%vuH41$PBk1P}Qcy!jkK4p7W)ESt@SosVLOpdNJ#&&i%)V(!Cz|JmMR zzZV{f?1{C>y8hYWVoB#|C!==>eqrw54SyaOJ^+Rf_=~~tQL@+4i0klawh^7Yi7h#d zEtVZVjufBezaq$Ndf1lJpr<@pKJsLGr7f3vCIky%Q3``?m0(9*%88q|pPEj%r|;dis0tpZvPW^GWf|l=RzlD07Es+-!NAn$8*L zDsKrnzTdDMvykLR*cLGcx(NHS6n(hb*yL?^HIn3y@d}miR}>-^%8KMdf(w9H5rba| zm{uGdixSfb&B6x7LcCipsLE(ev7laX4Wp_XiTX}?<=wiBALc|J6m=@EtsGJe!%!dE zGq_kd{q=rhS1!a*k}Ur^6OtoIPIj6*1KrQPqkhTYWB)#X2FNW3y9Hpm7_BIq@)S0C z6Bb3jwQTYiu+YYFY0!hR8scCI*c1c1;xMhO(R_Md+~Ar5isE6ppcBY!<`E(T0lg#KZk2zB`64)EI$J43Y&Y&JBWv!LJnPm0~N$SYByZ zr{|@36dMbJVR1gG4hO67*e$ptcp-Qhi`3FvOy)8f409vV8KW4AeY)w$Lnel=?Kzo` z|B*Y3UhF=U5btp~WZuf`lv&*&kjvdqpM5897$z!enYe?AC4NUV1!(yVw-83$pwg?!hWMjIX_!*Ch|6~}2jS)Oxo!@f#jSB|YBOsizj z6N+UF7fT@1#duXYsLL324gSVZX|jtNiOc*u-R@3t5R_#}LL*wMVo{7;{rI!_<2lEl z|KE8YVr2g}d=>v_pX=V{ycC`01;fHlJ7=^TK`sj-$#;O^J#g<{>X2rG;r%>c4u)%y z<*oSFZB3~N5rSo-$W;QRDrTz)AIpPeh@#b$6@yw><$8>+44}ef zd$2b+?HBSUJEgG8g^-QxM0^@8l}$%kQ>aYGML@S123^To_SK&J$hDqq3 zJWes^I%kjDj+|j6Sc^Tm11{c=6wgAY7e+B$h2O1llwJ6Shl%mN!6yIarzI{iZU-`2RYe_xdb69#u5Rq9k+WOF9obWxTn5;ov*G znCVFHBA7TEy?7UFd=Ob)j%Onbw=tHtAHU`pd4|(O$QiIEIq+!YF_#9J%3xO$d}@GB zEi$b&!BYr^Oe_tSC5?wz6&|)mq7^ki6&!|*<^2Y3-KaDxmuNYddhFgFed_!8Gfw<- zi>i>czp7u*GaIAGD(3{l0#qCva3*`(iBLX3cBdNT?uL&u(1^lNvb+ZGP#EsTqI`-y zIbkGOwKaLr0E#`V9M`~B)CIlT(Z}lO#0qS(Dy8_7trin2pbI+$4XA$@P1W5IqzhE`erbv~ykz4gnvySy*SyzT(C z>G(4X;o>y^c3$0sU49sw^0e^|w_{O0jFaT7CO%dKTn?^Dj%%Ui>VaY%?&}*gtHI97 z$h54qthFq({OtOS*Ea?0A;AN|2f_FFmxaC2P6f1K3LawiXv`ei=6D1z9)_`N5?&*F zh3xzGJ-wr6s*jUZmN!nFLAz=jN1~Yyv8(_=YzU_DK|fMB`0=8qkLQ@FTY(g&fnc!On{)hs-t6ABo5IQ<1C~6K? z=-Z^`U<}C%!R!40wb$hQD-&XeJSi55d;zRH1N zNi=AEFzm#4h6j%Yhk~4uS;01M6PCFQ7%JjR!zOtS&mwjPv3J7v&c`Ehuk~BuEqvy3 zRMw)fS{d4~v$MkO;+G+F@FW=Cfi9efe>2HoIL~1C2%gRJ#3;M5D2K?`{J{8$Bsn{h ztjvA|cqVJy00y?e<7)0hY`fJ?wP6PByQDj*eFl(1X*$p2< z6#sap7ygdLD!fAWm$cUqKRx$gVLB6|6yR7ZFl%C(_mkf|cn`^)g+^Qeb~pIff#EC= zoePF5$Tw_c9A%%$*nCNpcaA78E13rQ))mo(^^N2<1HJaxTA|k_4#P%p)G|@_MDa{n z6RT8%uYEzVJ^AcS!P%fvq)*V+8^;)JF?5VZRy1lBYQ*?+(|M#mEBnrc+=m#2=-FS; zRnsqppW5e=4pp|~M(32F!Y03a)XNV?UIoM12E&Q;X4LW3Z+6@G4dB!|NGiSgvsRK=Dw zL3&%VwF5!TN@-6V*NMLtNOx0^Y-D6v83@^AMa|0cR0I2N%oggz4421(PlM`_w2_H^ zb+@Kd4+%*|+i1LAIx!x%cnh5xg$sQZq8RQchMVQ0eLuu72@I8Es_oQqj=7KfeW>|g zhA%h=9^MUl6a2~Wa0arx5YJ|n@eX&I{Pn*W0r<^Nha~65Gb{(QsvAjeZFFFJB(o>D zbb^nU@VqVWwqO&6HNdQLl!>LuA{Gbz`UbxqXvO=3_k*I5dJ!+!=vH7pWj-PnMF9a! z3{kTj5fd$@=bv+U?iccCWj2)D@KyL)d)BYZV^EboEgp~-WPt?;8+yH`B;%# z;o(gnY4bLZVwKlZA+igL@(~v0INDHSP`Sw&D$iQOU}&YcBgpmUuM>RKj4A0x+2_Vs zm0I9vv$C?y%4%1J!L5mx`||C%!P`MnWLRWc(Auj)#YJ{}ob=p#=pB3HLd;?p!%(ji zf3w#n+;jYuvPU3os2Kpckmd6H9dtH%#W>UbvLQ3?F;(x-FkIN7fZL$2@Ii5=)>5YJu+{Q0x{= z33divWJu&t@V+@bx$w+l(Tc?~@fd24`Zaq?jiF{H$>)+bRHR%N zG%L8J{C?=ASHS9CFuaGq>-_O}H^NY(DUbbbl@F5j`WB0l#$*i(pw(n4>Y?#kqx(9t zb%BKg3=_r2-n?#aq`4Vf6^7M7UD+@Bhb7RBHDG98=KXaKt_`*ZKL>py zhxecb$0N(Pn#eiC@I~sBGzK7T_^a`2WKqh(BYD=%VOwXmu1Ie$GykzyART8hY-3Oq zhRSV+7?z4!mAY{7a=1Gxcs@89bc|dbIqna33piak!s0M7D@QEbEU8r{Q3!_GFpORk z0NwAr6H+k^k(!HorDMnlhB>*{=ub0H{Ly{Yzk(QLBYc~0csLCVCz1J5K3jDnkHfQ~~R zu4X1=IS=4d z5SID&$g{y=ToOGk(zc(0l8z|tD)EDuLK&@KKf?K_FO4^a!59{VMWt1=)OiORy354&6!K4y2b z`@K28{b0Bh%;xhq5e#oMS!|1;>TI@ZW9eRfv72U=-ibj>c#z;hp0iL!5 z!}`W5SAu&&Px#5tu7r&DHW*F_UI~5)Mnr~2*8An$HjGdZhjO*$%WB4jjz5&E)Y6t1 z3_}i{e_bqo_7@f7Y2;gTL1gjTpcA*ctHCf2V^M2~iMwUh*&(eqk*!u99MxI)MbtubXp{*z`!ec2=ZI5>|i2r;d_#&tm zxic~{_||Q~C{IzcH@S=j%oOFJWukZ&Vwe~X+C7HX$L{lw7IjR2MjgXLWfywDz#-^CS(H)WIFx7BvUE4H z+}cFUs=$#f7r`RwDiq^hI-xNf90r%~3Jwr6&x(8-to24>nF~@qkcIp7Xdc-^jf=<5 zG<=pwM#dv|KG!vpr!nPp=x5a_<)ybrK_{D=#qW={`UJcC7zipxx!S)5779gGIouD1 zs~DZP7#=kl>%l@*Jd^}CRl28u zqUvT#a4q_Qc{j%Q_A)XxIg&q;#b4r#K_6CyhlRNENgTc8NEn8$xAZ7m zE#FY}_Sre2f*{!ziSFa}!=lusLSz%3%>po-;*Y^D-)u(pW}qEalf5dAhd*)}nZQu& zsRAm>7&bR3M8cpnx`N~nq;oOvavS43(qRi{!Pe3dwt>&RfH+uKvm8;!7znm z>VQUk#9ip$z*+4l)33T=UeU`(`Gy-{;tVnln%(oH!SF33%jy!61*@E5FsuuP9YIjO zjj$U5ib8S}+i;jAn{tVf<+k`hwc(+vMT&6+@=`6P3W=PC)8){a1Nq)a#+iQzx5 zgOMUJA%>*oZfwut_0sQ3o22JBawEsZxw`G0%bg0|DSsSU{WtIqRV_CU zIUWUuH{jjegiW~{cFHclWJc!?Q{V72m9Lq>uqb}H^4G06^RDQ_-e9QCRoWE&%Bl=T z%PFVb0iD^Bn6MTYmNYEP2M3d)7;2oqI4teY)wm>hm@2yR$e8km8kg5Rnxbgyru1LA$}5Ox>jgXUY!;C{ zoP%F;8C(>GHzUjQ!EhON`6YatHw_PeVq7^3{1JvVh|-id?207!L6$9sVxqL7{`#AE z+4|TG@Fpu6U04W9l9Rot4nC`?L@ta@?!}(k(Zgx=(dlvpJ<-c4pU z8>`sdmvFQ)7M|%J@%TIMq~2RR8)eVbeI-2{RDDQce1@vdo6fUdZMbq8gjT}Bdx`F@ zK$a&lmNE&eG94a1K+a(e;{d7>IZT%e<*+k>L2)pw28NPl!ny-zpspq=6Y)eLtRF21NVfdk&o{feSgegI1Nd@93D=J z!|)L}y4h4=OCO$uhnYENRVh?QK3juf4>;5t$rbl3io-xufBpGHHbr?GoP(Y99Slcs}wEEr!yE z;$g+#Ww{$j^#wC)m&HI?XZbdL!B-fzj7ECp`M+dXm<{aGnt3g0nU$rUHznYpy8KdDF6@(fM?PsKS zcvPOnLB%Q7rnHBJ>h&k@P+oN|v|xJJr`fs@W}7P?S(uDO3-&ma+U!rT%sC?M{V$y> zk>n~|ae0W@{(xbKU}_!qXL$VQ-b=k7`%3rnrWB`33Y5XNo=r}tB~^Y)+*$ree3V1v zw(d9H;r0GN|59x81TdUzFkEbS_%a&P^6(UOHkwN%eJEL0U4t;}&)F+plU5wb-yk;m zHa+;X9W1QRwbDGt!mv;nrbC8*LruGA?~Q##`V_BI$Qi!J|9zM&B84GY_&4I88B zB*`VfFb6(OI?U@Y*m!k;_{ofNW#XFDW}l6z0NWO1j8u*MKuzpTPHU$c*IZt<#v)@F zS{yBciN!GW-tm}>hv{!KKM1LOdj^2dkbE6ReafGuDxdc*ymU`=|XB{dSz z7Comr*vc?XIWMaVe<5o8o=pgT!M)2qD}ZBJIvTIRGFObW4sv_9QD0hx5z=JritXs8 zNX23p;uoHEA~3Z3w9Jd4DBn=o4rRUMVQW0EAXNl4={3BFin*@X#aH3l9iVs{92AD5 zL2)e7EDY~47_P%Z6o#L3=I4CbWzD7#58HyE^4NXAPa`VAa3pwYe7PsyxB3!WZCDX) zSOCo-7M_E7KY*V4(j8-@>nq0nlfYPWg@$8Q4hLl-qshbGNq>YY)C3g(Mdj=iYbn|b zJ;V^lMBr!H7w$29wD>7Tu^x6hMM=`V!vz>ir{p^rA zi9*rtC$?8zEsLWhKpu7ugJE7q%t|{Mm^H6CbKhch?}39e@eT+2J^Z02RvCjvyovYb z5$!!;>Kt}ZIrk~-`Hh;d+@=a!Z3o+Wz`!Bs!*Fa=eX=z=FG;QkW6OhiQLxX7 z?l@yuc#P4RPeJfs*edPUH{N$nP@juc$r^bm*yOEq`WOuJ|Bj($qSc4iGV6R3b&JVw zh@yQI5B19y!>mpUR;4lH6lL5rzAE{@z&lh8`wa37{lRbumU$FRyv9T-bMS6f;N5H> zVitzB+D3M{1U#%wbkQb?;Xn}VZ=_i?8h; zn9c^rsPI-NoXoKHw%|D`z}5vP=(^h<49la#^CQc)^2Uyu_})m@8(s`RpWZU zQ1z7y9?GiNalN+4a(B>EeTc?LmGhEs-PveE)!9^yMkqFKyYI>dYg3GB^=z2}<~9Q`_b6{4ey;;uqd)Z+dQ{l+Zg2 zBfUKIzi8l$a9jEf(enq%Q{Rr<-iAlh-S38%Jruh#5{-C0?3`sV)a;aPoa@J&y~R*n z2Q)U?6n!TjvAe-fm0tajXJxN7^F)~#<-95z?>aZ0VL)8+16uGiM*QDnrp5s_^@u*g z)zOT67nvVT7V&fPH{Jc>CYN2=*c8p1l3kQuO6}Psij5(9+Mk|>7>3@Vq7<8PD8^N7 zL1o%>x1V1hOZhgkIu*{{g8W|MclGBc^mKwu&dA1(5WI4U@_KP_w zL@C3Z$J}ecFbAC?9|5aLF!3_Kv)>7M9^em#iI)+rs3u#vY*le?hA)RWQ;kY#j9$@R zHME^(L$wAo+2O8i%4y4|(cFqQe5PDOS-zbcNlk-q{Ui4Hb2Q<85Zq(7ZHD8YU}@ao zBwYi0&}(ED7_RVda&I)zUJ6W9w5RNy#n4vL*|=HviMWMVBZg)8%zD_uFasFoBqLXX z2x@|}-JOm;%s_?1gIEvc42Sz2*?JPI41kHF@NRB^p>xa_^RtYeDpJwtr~?}L%!rDz z4yyZaX*8lL*;W5#>*Z9JQJ7tI7o`rNO6> z;h|QiuvK!ZgcX)yeTc@P$|A|i{wJXgPvTd9N+$AsSok{Ly|$Ogy}gch{1UtJ9b9a} z80l&%+`sS-x^t148XR+>XtWph+*RZd(~614FkT<(cPtN8ORCNTdC6Z_;My*BBHrVE zb$GD>X}uP_#-RyY`)&O$pxBG&L$N9o8C97b#qb64Uk9)S-;i%fhp(Iut5ONSMKeR2 zf}x@k$+B!o2Q;Dbt_?w1^BYrOp)w8UVDV89Jcy0o1$tZ15Sxj{*Aq+b!>)Xb*L@0) zYc%~@c5~o?cONoS1Ie*{G9*hHRk18gG=}jk44+%ZYfM%dV9oVVH?3mm=w{9nPBHHV zze?}}n(j_4@>TdUJ^kjO*a^ELCJrKAQ61t;Jn*H)yAg)!$#RN(pXNxFMAO;61C7yq z)(*Ef@~p~N&FfLNMzeVeni$XJiv0kSKR}Mf!mWmZ8$s}CSiTc(9yQ)mS#lN&nZK7O zsNgLF!Z10t9vdT^U7D#g^Je;2B#F1nkCiCe6Gs{QybIQYhfzI+_mB#5cYf-Zql>?JU z-I}ePVWDP>Hi*V}Ha>~KLW|*t>}MBt{zW{dbq2vFjop6-FHxP?3ga8iVqB#{(A|5A z3cuP>4ApTr9z%Sq#NPi(z&!EW*sz`mEp9$=g96kUW0f2E z&HOf?*cneznZzr}XHO#oc0bv%^(KcXji|K~G=sjN$vdbIi=AuM3hrs!qF|>QM*G0b}&1?~puc}xz%cl-LP79FJ{O@Mms~1L1q7&80 zr9?f$bLapo$#25KZRp{3$nYA_dxA)4rKv~J$`^;|+t!}(y=U0tLT?8)r8fG|%Cc4z z2r&%(H=T2$Zo(|wZ^%VSvM>}6Rn;sE3xi=zR;HNXz0c|g@1yH(hH)c_@Y)#+Tf)Q+ zFi{ztD~v{*K^4iP@MRO{uKqyZVwu%xE|-yJb)}Lt%TLsN@5XGxu!cdRI6XSN4(OUEaH=Bhqc~2_z-mrhU!LV*S1jPYORVr z=fq(cMkXN_Eq)<}84QNX9u{Uas)qBSGt>K+{=@HZ{F6az7~V`PR@Z3+6Ptr#N2GZW zKH@c4mD`C`G`gobDsSW2eT`-I@hr0$|E2;MRyR!4T<->067f(mvS!Vz!cgP!KI`p$ zXL2;}BC}iB*IE$NN)GCms?kGLIqsqo;4Q9DJ*qDrCED5J{RDjNYwp+b>-r7-CVopesqDk0 z(agg<&T|!BoGL~>zCGdVySOXwBzxbWh+U}!dS9Hp7Ja7!29gs zRbx%opdZu$UcIB$m+c9dsh-h$@Di&tvh@&opwGRZ(1-Q#4-4bjTAQMELLy)o<4PiV zXtjd)Ej;xb7Dr*2o=xM61=wo9#IK#lys!K`!7g$N6Tol~-b@p}hF{xou?2pjBIluK zM9pJXzxKyDNA+e=CN}JDlN;|@v*M)@)jy;TTU|CgmaIC%WTMH8_}0Io3y&hn?}5=) zJi>Kc6Unf;9;=&f?`D47ILBh@nsnd1KE!(f6UI z5Em^EH4>17T^5FQofFjBd`HG^2S-01KV}eda$~<57}kf0&5-NPSd}5Ba>T|eD~u<+ z8;$rGXZZ`ZIlak&6@_i88xeBqlvW4rSS6ZsRfH!Nx=6C(lMnF>cN<;!4A(?5tX}Zy zZ#UO4{Uxr_HuzeeafOA<6-e@Px$Vdp7c+56j;L>Ac_@t->i@){7h00=>csO<(Yi8l z!mv6=b%q(YKhX($2WN3Day$T!upt=M^y^?*n&2mP1l2*qW@!SQef|K9AwmyoYbS-RQ#A?9Wy_-VJ^;h=$}p zZGfYjxke?4x$j~ITrt0t+YxWJC_K!GMU+huBSKlWC?+1k5J@Y|ik8EAr7Te%RwGXF z-R<5fc(@&1bTx9^5B*oyuk2R`#kyFQW@yA7U_0V>tFjbNM0JSo!YB1URR72%B0lwS zwq0^n*JXX3ioB!V*vZ%vjX|A3`adJWdkd?v6~Ax|nV^TbE_1s;~)&}np>KuP>?vKwUgf6 ztUKvRl(H?^glE+J=d3(1?A|h?=c3jc+{^9cSJLpN@iwI@~CKQvjsY ztxD*rL!WvemIqf={)hJDG&cDYw7?s9H7~*EC-A5igWxO}I0O7{1x48{%_e#dw&o|= znZ}IfCVq2PV`vJ7MPOkLSg83mikRiyXtUCMu`m>>4!@#!Ib1O+I~W#X9H70E)qTf1 zL#}QMQ6vCgf* zgrK9}&Fx8sq!=hFZzl|6vK%6)^Gw7oI#dYMwhkro2}ohO2>&LcfWU*G2P7^ z`(-fh5~7r|kkR+)|cT!Bv~Kl>n-*{&v<29&gxg65jo4piz6 zDUU|gUD@$|WKVt}Tk|Cns$9*>>|-S?d;r;<4Su&WihYa8UMjbtt_N$lN=bOxH?l%a zKYxhZ2M;kGLq!ZRS+?h!2n54@ga}%Rwl+mP3^5cBQ^?zN2E%vZVb_hC={#VDK_GrF6haTXvCYM@t*qkZAT{6c}~4dRTG;X4AqsmMAVKH0YO!|=0MWz zc>HN>(C4uC4P#H9Mi)MeB+o;JZ(+|j;p3~mN7Ag?o|Wj_jKn&V*r;1^`=NO(hMFU% zz6A2H6){_u=$sRI3w?L(GiFml47EB*UNEdo%~w|UZSPkwe2KC7;V`ZP-b@7_I|z#9 zL9+%ru@%^=?m;7b)-nr2TLJbtxM-b#bZ{*R4@UF%iWw^+8&kk=Far^uEV44!hs?Oi z7PQ@y#3YOHXH*lQn&nCOr>gL=()>6GZ^c^GqZi%80PoZtU@$C(BwGw6%fe6yCf2vH z*COPheY8A`#mibJB_G74_u-McELGW&z_Brr}h7#)fP35B~zd)50*K?hesa%THaq%<9pbxM0Z*vEc{VEC*m4jE+m6hexV|$6=;a_2> zNX6zHGMXH0UV~wGCoTP7PcoN#16q9ujBAblE9DpPix?DTTWaESw?dlb-)IKkEZC#b zdSy%ZVrw52#*7JW=sC>FMxWS*p42!m4SUk!@)li1M&Bd@Ol3y#sArGx@BMh}_ zl=9fZu#(fmiQpgp1cuL(`5J^r-2(kr+|TD10>xtJ#WGly`gn;QkmeyK;+u{YP;ZCz z#1fh_r8*C-%dA?~wCKfbY?;_JcSn`#XF=j5K8;v-01RIRLFE}9B7<=Ucuj(T%359q zenZ&CVx2Yau!!sQBazBb{KH%Pd)%S)KQBhsK|Hi;{HQ8hIfsjZ;UBpu7Rsh5#?*?U zImxJNwnBI3EOE+lFnoct=?@2+fLKvKx1S7(DWF*r?N|#Y%F`amc}@UpJLBgWqoFU7n6-0hXv3m=~8@; zm+(>S+K%$IPr*0U_N(iCgqTFDN~u!#6l_vejWUMvtzSfrABTnNyRGr;s}27K5M>U; z$M1<&z6MX{F1BTOLPZ(lx;&WXKk5#T!%(uMT8Gf4gmQP`_}M*J6vJy`@r%bW92+gd zjDRk5fL-qWD~e$+I9QK3xsada=QJpai>2@v>qez{7#i^gJYri(_X3FQ1H})GWzo8h zTGdQ?(P7L)t71Cj7=D0F$MAv_;q62ht~Y+6SZJ-u7*HDk1N)#Yy08sGFHYt+?!fc; zjLc?N@;1}yIz9|(t2#(-0Y9^!9TX+eDSip;N)7ng7HJ-Y+)cy|%!7%`(Q2E?J*c0gdbMb6 z;GaR!0lPDB@(i15YQDmw`7Anm@+!Xd)BNXyQ5H@>`(4If`w(NQ#-}}7Ut?FMg7AIp zH$Nj=!@%%K^r0{;WiXUJIv+!wWuk5&55q^>V=jJt+0NBVVU(;LGw7#y?-&dxgHdg) zaCIz7dOwq&)z9YVLZXY{Csqa9mL`*^st5Jhz1L*4)z?(3O1*C)7Ol_vGssDPwa%z2 zt-r>Wd;)^9Ca=Rnt;7B>_F@jYa1yM(3>K=&tSb!cgjdi8G?m{_4}^ti+nhvu1B2VZ zaG;qbEDW`FoN8fhS4pwMo@*iz4Ed)$>t?-d=??KwvYZ2}QUtkZW2&)t84M?JCN)5- zDpH)*&**1CqNNuL5&2aD+a}oNUTA+S&FUzruADDoXSMo*`p|ui)E+kyt$7m1qk8aD z*z|#s<86HVSy*_#vB)1{F?PDGZfBDa-bpYNwJC}u;V z^CQg_U}r;E($&Px*WpX5Tg!6Br_{537ZI@PfVEEH5BQ4W-*@1tDq&UBh=qH=`sJua zw(;Hd$nglOqWXbaTlU=wrZ-@-(tIrl-@$l6CdL*n3C1zMqMOr+JV`u;w&PObF|^zZ zcDBV9bfQM6K7)%|t5EALe{bxF>VuD9aX#R4RV2NDEnk5yRCmQG{NCjt z*cT1bo=B(}TN89iZLsW)m!oQd=|n>5$cJ|bMzOX}J9wz>X@>OAfyv?^Z-3Dubhu z(yP%{w}ZodNb@@6Q){VcWvRoaI^k>BcnnU;l6(Yy2a(9Vd}kwa{G{;<=kVW?4GUGX z+!_8gGFyG*pc=o@89O}+O*s{Y2aGE;4|@2M-Dc>+{O~Xv7}_2!7t1bN46SwzSs2=7 z$+F@Ui=q64CeBUH&+ZgPIA7qL+JRpY@XG3+@_qq!&yQeTvS4G1@QG}5Yvg?pQHAQj zlzmV=;`8{48l&3J_5ob{+@SX$PP-~dU$1Vwm*D+gf3?FU8D{BpY1@Njf`6s-eyffZ!o(qZ@(1`_!SSoR*&ERSu zqMl?f>_uZ z9asx}TaBo)G}o#Q(yU760bHrq7-`VRe-*!nTMG>HbKSEW9$Me#;$Ud6hD9&lx=3$m zL`6Kbv7UUbM$Wa)^KNgB^hMa!k_b63b~uCogZG1X+Mwtm(Mi~q6!c_m7}M5htE=!! zR898)cI63BR5o!h+dfb{NDM9ceHUB24{Tq>i&TXCBnWD4s_7s&kt;R?f3%aa#Wm5x z71+vi{ffhPT`5H@6T$v8t7w!7a{G)S-6-9Z@U2Tit=DBusj3!?zNye z4QYM^c~(z9`H8Bh-VJgG!S7A*+k*_h5=C&O$@?$F7Tto^G7c{HHTJj#EUfI8gMp=y zf`UYsRd`3E+FkgKtyuYzjL4kwO1TZl9~LsXFLjmC3=v@%<{Rvp{(b3<|IZM^cn(Ti z$fktjDayg-;dn|@CqCM_&Ml1{-Gmlz%sFH+p3Rru7vA^YkKPGzJmZOpdC-aF;YcI2 zro3RypSsD|6^o))wB8DDv@-Z!*te5ERTgeUUp`B8XIFilZ9MC1;QSEy+!jmR04-Ps z@2MF3FGzHh2hX4uSE?PKUd>T`Pc2ZU;9W1B+nn(fjY;K#iK?BmbEV{8>s%9wVGK`; zW_WL9S^j|h0xDrPxX=6^x)WoUMs|M@KIJr%ry0XF=#AxQ zN^DsJ87_+cC_r492gYaPSE_QQnzN0?T7B&oq`G9OcNiAdM}rsU$hDTM>NUg(*^>}M zJxlB@H6!&~mWSy$f+X^X$<$-lrc**)_ebwA7!C)YJbnOPXT2ld``%~XQE>duJMR7J zMR+%-iCC&}eu|uJo%Surrs_Lvzujlida}~Wv2I{{fma$qUWUZq$M@J-eMNcwHW7IpR@0TJP$^mHG@hhYBk7}SdCRgF3)r3>p}BbvuO!N*dtQx!jBxC)>7If89|8*eFNKNZ0+4;ZSqi1fOu zUvy>{3&VI8S`5WQ-)ND{=#xC?mC{tT4|0BS)_ObeB8I_=oN&-}}4mfxn2r5g{ zi)+vv)>i_pqS&ykurDLtK*ayW3*cl?tV~6YW(>CK2+`hhKaY0{xy}+~>ogK%yU!^~ ziS=g@YZ5QZmWiQ06pmuP#vrm_)zlfZG+Te?eX?Kb96lU9l#R1EhXy?0?e_M2A9{yi zd}ZbYeUF;C`@5{T|d+O|!(cHmPuYl!)#TB9bL+S^;Ag z2+AkC60Y~-|JuRi+Q@NnEOBla=wZdq;nAN$Z~W>hE-DU=Wx()h{G{JN{sp%cW01Mw zVdf}?>RBdFBOb%}bH6Z+D9plJ$U^xxnz5&uqXEZ}mgC6`h6Ndwy5DK<-3Jdx;Jak< zkK-5q;Jxj=0u$e1dk^b!#QPdtf5Eb3F+PrTehacyJD>+wN22R8 z_}_6R-=Ga&_O^Tbyf!R;Idc)jh8C!yA(UCs6{hN)yKfD2dTbm z>Y}K%<)*^ZiNtoJ4GX*Qe~tP7a>kZq%K!98nIAthKYl?Sfq%jiyWE zAA@sWg5mStCT}NNaR)fQ4HN%mEXz3)vlKvA)kG)S+`v$@;?;O9Q`l7dCN|D8e>0Ke zo8h3wk4G5<+Y<}d!{U}TF-b;#?>7+qhJEXIKSKY0g)6>xB<@n~9ud@aL5#Uv5@!Bg=5DEn4qbO5iE>UGbegIphf_!_RfOOz*n zW|#N0_bg1@X=VaTdK;0=qn)XmPe*%e#XL zwNk_cInaj6zsjO059OG%vidM4cmE!X|M=6t-jkSU#{-oAN&-dA^QrALc1ltEwHyqJ z`R6zXS>m1EDsLt7yn(+L!EqNB`XHX7EXz4B1NJlpd8`H#+hLzoCp!#{cNKc?T4Z+; z_v5%G<3R9o{7z-7HD=ccO<0lGE*V)Bf~UA9lHm9G&G)!YJJ7x#!AsfbAH1^caUK|6 z?*Hia#r7pft+Lj@Qmi0f&FVs7Xf@VIYI|*{*?$s)8GD zwu#)2=958Y)TD!nO{(BQ6+;^+t~&!--x?kshW!W8AaC;f-{UF%%I}nAZwt`VE&Zq5 z8bo?Iz%T=n6T>hzzsH_+sJmik{&%0nm>52@kGj{d%0slB?z3Z=OX4%Ebq3?z%mG7X zYz}fhd%ZQ@6NZNy@p85pX+D57e}#;l@zNS;E`fhi3%eo|dm^ouvW)~Y@$pLJa5PeU zIa1yiY3~YKn`2GH!W0n9j9>i|_#FeY5BbG+xkh`?8r!*2pX2F#2R{q5x9P;*>HOht zA^e9Vq&WlLnAR*&M&4?tkb{30t^er%&c{&Q`-Gug!$%zuOK@E~G9HlK>wzW6!+CuG ze(%7$<=#^78JM`%dk!Yb<5t}ADK=CYhz$5|1>sXAnAj32wK<3(L~L{k z&ve0)Y!kI6CGjtF;nT>P{1C=wcHX)R`D zJ~eAo87Ns~YnA^a94_Q@VQBfS$IwGe%kzx9%85=Xj-Swut9 z5JP*OiIe3R>hTy_?Q1$`ujVh`;M*)tg4mu{cdy6;UKZ&&NfepJjmxW-s5+*+Ftw%2^@_U2x zJ7oMDWvDWms8%&$7DLT1RquknhnckG%AssPjY zp1l6@9Nnu<5vcg#xN-O=70Q}!?{T~AVhrr)JAk^!rAMK9x z-v8bH8iDiiIsbKt;$QFKfA1$DR`HK|e*PAZVdy9P??p{#9-?qQQr3glz0IN6`|~CE z{MY{*u@rJ4k|ByhO1cqjmbU*&INKR>(~~>bv?>o1OJ;Kh<@Jq4@tr zCjFi3ByDF=vKc`qXTK{Hg=`wWFMirz39mqGe=%&s_rk~6C;wBQosV3IUnq}an#O^` zn~+UM{wJBdMH5!1y^^(G=Q|HMOPvRuyLo;k?=_)XOTCej$Z;rxE;NLr_!?@5)L4qQ z;Z2|XPrSt+@%hJh{xQef4}1Rl4^=*81GzGc`Sy0EIa{49?m&07`?Fiw>+Oy8#(9@| zZM`Dix9(bZyj#aTOP9r&bo|#!JX-xknLq7WfA;#sOF=m0yXNN9oG!P1SEUpD#X!M-V;=gZSqc6O#+uw>`?#=;_!k zf;uSCxK7n*;fex$3ItFnR`8Fez$s-+9qR3EE# zs5Bo<>s=hDp6~?q#?}2RyczCp#*K4xg>(xo%pXy-pT>^!eEjU^y8q{H;kC8L628*! z>$!YBD_eRUq273r)5)#GXx=ocUk*?asP2zzsn488UCm_1Jg#GWe+qS9v#A-@O7?g0 zY^M24wZj_Y8^nm|1S**x1wGBxJwScyR>oTvdW+rHShXOt;h%obYDXPecw8Z3vDdM8 zVwi>R#oqa6p2dzT{5^|cEQ-kQT;r5=i+Dbja9iNg2C6ETP}!x~b{a_@VYZRHvfC)$ z8xCqiKu!GxG%7Wi_ePqL=ALN8!61H*nJc-Kny!}^f1d44a+lx%r8NhkD<~bTn~u-k zQXgsfZuG+_+Tp8z>M=Zu^N-TbzmSG<(D;p=L(Y558kk5$(;DP+E_DavK}h4Dz4`0K zIG$#}cLN){_23;n>&Ckpnb%C6wv3yqD(4aq9SXt&`TxPN@@}fOpFfdwDCCE&*R_uXP(Dn7&}HCyJBB?fO(wpPEYqruNnI6QEDJ2Q2jBC znvhOx!lwh33#}OQR5zDqjDNObQ-}O^2DdgmZ^iqKs2XnQH#XmCVJadzu#ds;RO6?k zkpE?9(k*CE$N$*f%Q$U%1AvW+tWT9%KmIvB;raaerx*L(7=o&1lK<=QSJZjYd53kT zZlyYX8nURd_O=FtHqJ^Xk@Fi}xFVl-QO(>Q6G%uj+kR~4^cfHr7Iirc{gwyi5;H+iiep=@q~Yu zSA_W}suos_m-3YV6EKX&FaEhu6oRTplI-5{PeHUFts?f8zW;;*3g z^BJO=^ud0Ar`MQ%*wToan`e(LUUtIg@#pg2^<2kd5maupuXCgOp?44dn>vitK#x_1 zd749CoGSJd{%nuasz_#iP*5j))njX=nn7sJVOXEbP4%Jr7^;f%G8iV0aj@|gcmP@Pf-QQ@T-7*nw>Kl>@c32N)x8ZIUg4`${VZPXcJ7d~3tQp?ysiKUK1 z`FYJ}mM&yAm%GK&+N{06NH(Anxa9{$=|FV}C~de|3oX|I{8WeC(=cxsSZQTaRT5v1 z7F9LzWHj6j;4k-C*n}zIzM*b2pxM8af*P;vewJ zyS1>=xykJ+_ZVi6WL|h7)l3+x)WL zJdUj>k|GqXb%`-B#PFZyqkh*SD1T60o|B2Ox;o?CG|V?!0Yijg9c)Hf&{M}C^)sys z164KL8X4?m@YDRV$^Omc8m7U!yQoB71ZEGLxwk9W&k8(e&A?q2okOa5;0wq{+{w2! zi&PzSEicEwPOX)FJy$|~qqWM-m#m7@m%LW4829HvIm zyB}ZjF?^CI$#OkQZfqTK@bh@hFYveyTWe>8D=W4HpGhm;+u5YInz=w$ciu?6GK;IQ znvBphc#0Y6t~4e%>0j=>zzpHa$ck34mW|e^jP-WJ$bb4~J${Rz>YB4*^VIF6v(t^9 z4!bztv1t6Z_#Q1_qGqyNPfj)K(tiVxxzT99iJbd%7O*#3}-m+xaK?r>hl@plpd)s4m&1oQcj6w=Xfm{dgw5(RcDV)bT@W ze`u9B;;ZyM)27yKT8Sx=DBSqnlhnT{f_nGQF5Ot+x7=#-`|H~UPlme)+D z8=pC+-}f%m%SV!+W5#nLeHv?dG-PCsOyq?F4@js+YHmh z$~8#8BwU#xWxpSRr7wcJIt(3Wt+k%P40=Rg?rk8CUK1W_ZiO9VjiDGndT=ptOubL> zUL$CB^@@zJFej2+g82BP^QgC#{PaBJw7;>rm%u;u#O?upVxZRKnuProehZB4SdJI8 z7FNAtdIG&gW?JYSp<|kQ&S`CQ^*_`~^ItP=px)T(K_@INlCL1+FT!7qA*@EmHIlHJ z>#>bJeayao1j9>MU9W5Kq4x_`r6C9w0zu93v7K$=nJC8m8EaA_@Lzcqzqi zzd-J5x+19~>@RG;qS4eDTU||0z^ZTg^cbGsCm^lWg5Tgjw}aRgbihV7NxAsEhwJbS z9a(-xmV2<8`mF)E47U=qXw+F=euQL5XJJSAV;V7brvDYJ{xjbV_pINtvTXbJWyPN4 zW2~+U9d535OVXR{ZSa|l)s@{Hj1SnC80~VNUqb|OE8g9mU@Qy#1gv?EGk=ZqmFyn* zy>sMAQ1j^Ntgg=5((-3tY3=z_=s2xfeH5;$SN6L`U+#g&ud}H;%&X{wclliFxvE!M zpWyD`kzkHL#jQsam7k-{VnzZpndm?nV{29HIfiHYpF!pCe8(cFUsdguU8gV?8m2gU z<}znF^XStB@1Wzfe)U+`I0^(cn|2)dNe9|)N)O??t%QLa8Bf}Y&Uz32eGV$eIpcGn zmxjLVb`6kRtQe4wbp!G;9;(jwS{pquy`Im&O7%DuBR>Mk1IYK=Z0{J2xu5sGM`@pA z^sge_#pef42Rr?3^f7Hi#YstIS^XmwKPzgn7~0yCM9XpvWnmX?(!N%b?K*|hhDGrj zMmxLRt5}6*KOCC|>(nLWN@RMpvB6XEJ!TsQF2M&}gZ@&-NHI`dvA;oYo#K2W(cbpC zz^!nQ!ip6|7?&@`vqJo(Fz&2%RMh)E!jO|Vd4m7d*t6|gd>A$#LSJh2fm5u!EH+m& zE4FUp+vnWN(b~oFAZ-807=}8xcrM1y{6e3d&*~V8)}qMA%FdR;R-ekEZ{6ASp?)8X z?l6)6OfzmbAu7RhLD1^Jr?CN>kvnz7{+P4>mh)9lajjR8ADpx*S7}y3D$5E!mFS;Y zjr$5bFUPw@4UT!>WF}ay*Ec4od+oAn77vkrKfpaxx* zI+%_|>d~(roQhKwt%hf2A1@TQfB*mXH>8DC`=_%jV+! z&BSutfQ7!9XR^ZgBIB~cl41EUZ^4((q7v+ZUlMX#D7yAhCF(+In|0yp5%D%O>%`UM zbqz8xLQ*U3sSY9ulp^4GmTuZ7(FNbb=i~G;*2+rhxfXf&P6O66dyrMYy96V>dzm3z z0Ut3po7O%ShSEzqFUhjy;=jk>zw&833&ZE}7}|VcE_6#VG-6A}?@D+Z{L;av$mu;u z^E~|Csqk?+h%UsBkrsT4&$Q~yZuFLw;h&MyI`D%^G}1t902Fl5$WxqO}=3kPN%JpTe4Z;9HVCCi7q8u_zyaWcVAn<8uC55w0-YDi1ylz^N%c)!W0HF08U@*Q4*ovySv;ZqFxL z$+bScUA5+33G`)N{y!_5)=Bb>4R?6Vz*Q;FHAw;e$-%PVNH9Bija6^kF@ss1`;-T> z>n+I74o9klVC-BIi>KYMWuf)3RS}&TNyyEys8>}NXRgy5@33lc6r7fjy;=hAv`Usb zvC9)&gXg=+*pc_p`ClM$CozfAfyrP|225&F-_-`BI)Yq}pdT5RzT6KC1_uMotKPiZ zIm*gLu(c{El>)_rXi=?BpP9X7;yx`aUSy-{Nt(1I9KI!ZJoqelBKXPs6r0ij47GBw z@;9NMZ6m$#EE9)ah*<2}=r{Chs=-hWPFZWoL>XkcyK|p2&O7DT3XY-emchLh_%HLY zF8A|FzTjH0Re#dAk>W4VTqm(38PHdGk+@1Qur3U2!)nt#gWe!EI2gfpDfM%sun8lu zt%KQy@JSEijxHe67&g`hrSe>XqVO%*SZuBOlLZ9xfNgE|Ssu)%2Wx_(tOK6npTR$D z3WmzZX6MMI(N)_lUnBHw5{IDOlhw8H{Eo*^vY<>-S$wXZ&Pr#NmyYxL1~2zXxVMHF zSl!~5@JfA~H^RZaSd#aV;_vYUCB@m{kYrfvEjI_j&fqryq(%gzsX!bXTtzM7wQOU_ zUtPg=IiHA?eZa8;QreicwuE9iqaDS^WP@cMbVF(O*#s_jAXf;*7`bH8D5#G z6-J6*#GA1z&3ue4{s~JW1d~}sv@9qz1dk3ucUU$Q?p+3E<5{`;I=1VBn}VBJ)olXr zj0e#xV5V5vA3k;h^=1ae%4h<+vT!l>Sb(hsL_HAjScAryN1EHu`qBVI&3=i#M7uJo(_+ufa6VMuqMOF zF|hG6zS$qC?~0^01x3lUMN#WQ$}=yGerSPaZ2*RM8VpYdOXxdSom_hnj#nO{`iQ1u z6NYxA%G%6CvC(QnyU%!umgZ2LEyrCoO`g3ihtJ=0p8Ygh0a6by%D6X4z?@S4t<&t{v&<6Y))W^j9O8#s;! zC}C$EH-L2UOsm^IC%8k3_)FAiu96z3QN; zs7{%!H}I)HC&K*+O_%{YDtW5}hnnND^u+p(0KsvfHwE_1#-;4uU46QUvoIWGEu#s$fZ(Ob=s09ml6wctTNFG@bnyUx zOM<05E;8FapeXsi6}FCri$bv%65H0;7p=Hn0}ZJ4a;tN7+T$Cw#fHxbHei8KGP@S`X^rzoiJj$D??S${-nZ80Il~Ge@(GK>mByT7XXMx-cs;UuH~gC)+zV>U zg5|*qwkLzt+zZi%g8TV=7TX=LQPz4C>>Yr_%4=-S^{c~Gk^gOdrFO7X-sYXbM*PE4 zk+#7pw)6-g>ePgXZPA#W_=TD9 z@Z+FdWK@vil_fWlgG^mU{Ooi*(~MPR9i*eg!$jWVk<)YCB;WcaWgIlBoB+YQi1bri8bkmw<1d{3}?f% z2jSduyyEAAwPst-v!@N7PxAhQpeUx!fQgg%KiQT(_&HrcOj@xX+CdoB0?W4O2)#cI z3=antBDV#tyjsrHAXl5CPvIPLzz6ZrX0xsKN)(DAf*}v>el3Qo6<4Guzd%u^Ix==8 z0=>bV;$I0ar^#q(eW-W9P`;t^tZyO7hfQ?%3pv6ZS z&--oApU;zXC?B~$xWa46DkV9cx^WmPn{6=+^2eI|RwqO(d*}(l~?w{kG^`N*MO}Q8@iitOa;g#r8Mbkaee67(9%30Lq`iO_Z zusip+@ZawRr6VhYIbI*~OS!->6M8(2k>wZT?&S^gR{iL8YkJ0NAm<|N*lgwvaMQS`8Q(Z+{UY^%^6?9Sq>-qxD^EN zL`Ijuv{h)or$OztU<(X!CSe2f%O@F|%EJ@O{RPjz@E&(uOtB{q11S zP<-WyX#G3!TJFP#d4}^{59_vq+-o4WGuX?v9W1vP*?tP^@)$YQd%62ow> zF$`7jD!c51VT5=oJ-PKPRKC`BzH#65$HA{NM8#T#RT!=z&ePcEe%||(agQ^`DyI~Sz{+ZGVT%J*`(kD z*EfF6ASC%JBytvr+=s<|0*p4m!dE!wcY`;%-wS^4@N6ec)V2wXSM$9kXhiYw1~8O= zGYF||&pw;7HQ*?OVH5OiFEAX7E!rF8jeHSQ^FMJWv*vbf^mh@^Q~yiJa)_bM&t_N> z!NeGDv1ihTE(m%^fiO%14>N&b9`xZ*C&l{;467InH{#D6z%yL^JBEjdm4AYV)+$%x zJe3g`Vq)Ye=)y%rA*0n}bvl?M4T_Ode4+fcvn^hZ*{4GLcEnn!EhZqZ!0+M0JRScith#Q2Oq*op|}SOrfOK`S;fQBYfK&Pp)M7V#r@`_0{3iS29QM-(!$tU3c#q6$GPXBQtsYg1y` zLuFvgk@;HYTdAZj6q_9|A*VuQQnOlL~07w(wB7hKWezLhQ*> zEbazlMP3hf!@t9z_Z~=o6ntV(dz%#loyXZ=gv+o)TrP3Bz~=0Kt{f}mt87NsPThnCm&-d-I&mtEGne##zd zWven|k~%pXoZGz=&Zly44&P=g(cOE*D$np|>lwtuQ;hm%1Viz#As8xOb0xAkgV^ms zwBc&xbSD)F@@Mvg-62r?04$I2>;QNQ!&i{>jWF>^<7Z1F-hj;*@&DL+4=^dJwEerf zs)n313}F~@&N*k097QssARs6rB8o~we;S&U1`DY=Z8qjYgDBDG3j&<1-4wZm@QzpDZd} z)IM)aXf!))l*hkO1zlRkpxrlucv<5MM`r)3PK>Vw?PHzaVko<;UOp*7P&>$$#VYS| zX1ImP0u*De`~=?Neq?!-!Eh@%S*>^f5?L08R+i-*c0iN$2ZPb@a602$XfWIXf_p*i z4CtNUcFsS42Sa&^>tW&&Fr3c2#_`@^oI^IHt;x~Y`RCGD71`w`Xv(H{Fw7D4gZFW0 zEE$sWXpNzlTcTBmS%C+4tM)5g$q6r_a!wwrakVJqM+$;>WMj(NSzLb%&Ix$ihTCEVEXuWKGitSdWT6R@Wf(8?Et)*JYrCr6!0)PkzQWBH+k4y@NAv{cV)b!4L>n{;ci%X z02w|9ey5E#yvXCDJi8xu?g7JIF2E#xz=(i4wp^f##!XySm?bxa;b`9s2 zbI`2=C(2MmbQ%l~8-2J89!eh`HnObFD#_4?xj|Yy>|#8d7w}LfqYdYSjC||e@KCno z9EUJG?_cD00tENN!#(&tD~x5Hg%%xa@(%rwW!V&ASQCpM@K8EeS?(@)?|Z~Cs z7@jd6w)Ei={2T2Ctn8uYM3uvqe^?5A-^5@z0Nlqh#t+ePpW@kwg?o|O6X13MCQ2KM zht{g>N0!&~x<$zHEMu2Pz++)3S++iQ6*OUKbc(WH^-T=e5lyk%Pa0Jrs;ytbeUD6? z#V~Um4{Z-TVHk`$av17N@iEkANMG1WY(?yf++*++$b*EwcU$2tm!pdH9Nys(^r3c3 z77qgqes7_GsRV{axmDutcF1yjc)rU|0*2lF2JT1HWK_T(i@_eM)t{^bR7+S4_?! zaa`t0a@G;MESoY8%{Ge|cp(;dCphf^uj8;#mRZt#hQE#&3@r~=@XGg%f2cU+8T_Me z;MCGcvSw5&V6zHwD{rz6E$~Ap!ruf@6{Gt5o!w8sup*H`PV$Hr!$2Pfx<3>d8pdhB zu;(5i8epi|lyopL7Ry|lHUHT{3BAr}_=?nAoi~x*A;ZIP#qgKp9BxrrX=A;rL}r~$ zv@E+k5ovxOb}3)86Init>|R2K&v0De(HVGn5EOUf+pI$$F2u*4hDDKQ^9-?$wJ3^~ zD0KqA}O;INovA{PIz+`xD)rf5Omuh#Hs0P~(owkJSy9efF`{j`|$S!(@hsv1sO+ z*tx=?7_SdHz6v~)EQ^OLiBnb@47GmZEc1H8FfIAR!tkXomb3>LJ{yPOT$s25o3R_+ zwHwwQGsj7=JPV2k;O7p|wFoMc{kF*;%A)kZa%c`zk#afoU@=%2gBMy1jo66ioA3!= z__zGhQRDr0-Tm0*YKDihDe@0(zYOc!Sj!w440RTZU2t!8qZemW^x3N6lV+|?q*6ST z+k1@bZ86*lX8Xx|*%bhr$v_-rUhf+sl?0~dupndAY{w(Wvf|~5*yZ<$msgP|`4 zNIi~qs0t5dm$jl!7%s;{)QrjjFx0B01n@8`m2G9n4v2@38htpP=ywj-Xhu{yhwW&- zqhKd3sO*?kaC7E+A$k}sZ{c|GGCD4&|`Ogp{ z@t^z>Q8WC{-Lqg=9p5IFNL3a2+A%Ys(NR{?G}6fY2)p=945ZqwkkX7|QV>jyXOk5x zSH~$6D&X})%U2_-a2_ji&|tV2994s@8RpZb=1Wx$S&_4n*w9w^B)y2#Yz}rN-pvyD zvKigA1;h@4p7O7U;Nl^qdmG0llYyNNYo|~%GK$#pN$k>toasKUNd-7qfOF@j20-f! za)D?qgW_N`?R949XZc&)FNyVPfMG5$OlPuBfsY+&n-aFhmWg*`sA?(A=%gY0krhd9 z;FJQx{>-vhBbvJaW)?%s!wp!KeRzmhnc4f@vk`MiUfB$vT2$3 zgRv?{!B7${?{F8-t>chYRu=mmI6ndmHLKSF7S`o_abY z*pQ}Gxb^pYb~>CU@sdCZN-u@*}||Vc6A7PbiA3>TLRBC8=#p{#mAaWQ4VK6 z(yck>4g7C0nsYjmEdTmBEXqUp&`n`m6?_`i6US1Eof)>LhmHA>@9G=_&;VEc+)?lP zhurJrZG>SSGXfc#$WA#v5u1`L}zMMHVup)gDe z54DnZCsx_6t(AA9IhBi8=G$~wNDo7l$!>yAqIxBH+2di>Y!j)hLhil*vt3vb&5g=R z??8&z$FcBZWPT2^JT`7Vsxy+@06km|NzMxgRoSjpW~%ni#j}+`S2KH;n5BQuKk5EN z&xBg!9P+@!zs4|9JPg|sU6+7~n$@#i)|I(Y9yl9Tr2^7X0N>_ubW}Ct+X%CrWE?&K z$K`0mO~%TcMI+uIQb`9>R5@2426ZD+)7fCMX2pJekpJ_{($DqJy1#;9Z7|Gj zY>Kk+wp)wM1=z7h&Zfl2QIEw$`8MLAFieR~)_!&6(9*G?%wAu3Sk3s@7Q@AN)&zV( z=3zgU`3h^Yt#?xZKcpTJ#Dmz?XUXWlhICFtN|zF=tibwi071<`2|-mlYym;dr7R-f zt;(9E$Iw-Ft<4J>!@`!k-`tmwO7tyzGPChPSOeobRE zhaDJ_MzS zKof?3bGs8e)G)HV1q8Q(-z=<(T_d%YzYkL{@;w+PLFeZ}%4&jPJ2YZHFnkGS3B`qm zZOgGAnwt`a8;wO))^HL3osT4|o_I9xc?|p|$#s!rd4@6QLgAN)HIKK==cF;wN?|gz zGw_pA;A5XdmRrEX0>-xq+Y~#)9O(YYU?>TeEL(ZDNLqbp=Tj1ap}Hbu;=7ap!`sgH zZdZe0LjNo_M12*s;>TD!+OmJdbF!S9vUROE~fQnwBII!q56lY|J0}0 z70o$ord}0viO~Idv6r>+5g))m8AQzSG7_p<;>95NA#y7ZN9gHLjmipuf!Xdte5)hQcDE%M)&#>d5ml3&5e!R%xK`UfjCIhefH6q(Ot8|r6ySjNbYW&#=rg;06K(OmSxs~st&ssHW=HQ%K&$@2Rod-dCXZd?P98fII!S$tZ3Z@A zPNXrk{L|SjhVrr%VJ72X$7L|g4~E;F)ovrMw#9HQmUum$%>>r#&4G#P<)!K+X~eHh z%}7c{rV2W(OYDqw7(_;TBx6+F%}isV?V5^@KyMKqk7DF$oc|4^Ub$<{p|mp<>#`@Y z@H!3F%%P(GC*~Z!HobDRLOM0rW=3beg~$0D_-}I0Q0b@KR}6lc)-Z>wmhQEGXn8Ik zhA~WvMM+PsNM3xKHO_}_9fP6I{PG%(b)feaeH>%-69^H@Hfx-NQ~tvUNqsmZE6#p3`Rv?C zb7kXccZFBafX+z9IvFlb1IG_wpmg9|gQT(yZ(>bU*DwG%R?bi>QflBq=Y?tN9`rk? zeTU`OZX(Cg8p@li*FkEo(^xG0&qU^n-5soYtb`rR28OA?FzDhZO&?hd1C1E?H#)mL zDg^V&=!oZpDs9quf#k!w?v*BuZX|E*xt*%c--2R$0hL++$`*f5B#|rdU-m zsw~n>^Hb==m%vbLQ%+V|afZQj4lJC+e_kPDC^>!{i!86^KCV~=6D4a`8sGmF)V{)x zzrbgx_mI}8Ccri)<2sE&zy5$f^sc*>{g=wpO;Vi@Q-Y!*u6x7707I)~^_R4vZo*Ld zP#(Urm<5UUTCoq=lWqZn;U8pZmYd!KGw6x&Cab{JpHID&R+((9@;5(VGgSdr02XOo zon7@P6h~usUxjZIk!M9D6R}g%j7Re-|9c6E57y{420>LC=EnA>hJ_(kTdE7#IpXpo zM3%>hEU$Ym|4GCvM`GE(1^u_(B~BMs7Z&HcXJZVhjAjoiVKqvPI8r|ry@1{JFMURo z;b8)UVJ0#+!mzdTlGDk}#)xYWA!yyG^xyQj9+B_BNOcxbUy-eh?>su+u9wP|!W zhsGMoq17rAV726W9L#*p;P)J|_=Jg09>7~s?q9l4o^_P5$5-K?I{F<(V<>ZOYruaa zcU}=4Fpun4M}H8#mLB4|m%^vW#t6a~#$R0#86L)Gp*^EGC=6A9C>*Um6oxU#av!G{ zO8E^iya7s!u*A#g=I|ojB40P1nx|k{79-7HFvoY)tg}$GqM4PVJgM27XAtgFJMVDpSTyct1>`y+FGgl zt5=ru&Ss{%mEY4H<@CUlD1{ahhN&5eU9o4!7fD&RnjtL7RwD)&iig6`@-RCXKICL{ zuY2Q&yMAE|9~#z8qRY~YbZ2>k?jmpFDK0Rcw!FmCMw)HSLm{-9Rx#gC)X|;UmFGEz za=eK2jzR;D;usE(v^Gy`ZB`8WfmjnjEYxpyVVc#x%`Heah`#3PvOI;2gWe) z8wP?}Q>Kn0@@X2Q3u~cA3z=+Ja&inmV@)o>!o9F>Gb~((Hr{|lA0b{*AJigfiAl_F z5f{0=SV^A)3{&ADYM)8XQCfW{hG>kDnj5v%HkOC>y!Jluv6Z(GhG|Wws+Q!F&W3h+ zFCv%U!-Nm;4L>5%d!GJt>bNW}zKt|1BfB1HK92qTo+u(Yo^~wR(`wiiaj_HH?nw^q zQYfE$D4ZOO9eJ8()Fr4FD7WO5wc)Pvg*niv9@<0J&)!pC9u8h z7395dF~e1g{(bG}b{Wk&pA;Y|41L2xyN9e86KM?NOSHw)VyIpO^070bRa70`f;#aH zq2=BX(D;_`KG#U@1p4O=LY{|WThzl@*~G>8-s%;kPC@Fgr%p>nK%|z*gSErv_JM8c zkMRunJqvnI8&n^|Zuj87?a3FaVz?~UC!V_R9)zAB3KEP9c{FwlY)!|-47uw|Jg%gOmJ!Z3ze zp8C$1&^T`(7=F$5pN$oH4}1KS>EHi?vCpr7=}htupEA;&c)*vCW_j9az(Vym>i4Bi zHxDx2hq1r?jr{iEk-D&S!5XV4P;;z_)}It46P^W&>=<1ruX>-cB&s!A3WBqZPp`Ug z^*heVJ6|9Rl+3>oDvnK2XOU#+aqUKr{580Lx{5>5EKQS6>kI8APyMyADeB`fcfKGid zYDd<&D@^PScIpG9?sGkOt_?>kbYWHGxF}MotaV~C*54UxvY+d+7E7`W4r(rJI$nU* z{T@I&AM|2)@AKqyfAh|V3gX+ON0w6{%ZcMK6f-Ojt=6~R+`oEnF%)k4u^38Q$Z}Z> zwK5_R+k%}^ad0)ND= zB*Lz!%3W(AY9Xl)fMQ4EElPg7!Ate*YY)2WA}PmZ>=ZDhTMJx#((H~h1TRNDP2YzrpCj*w z(e=v5DqgW25QSnLd>pOz>==h!M`}UT^P)BQ)#JZf6IvV%nvLj0Gf&rHp|vJoAj2Qw znJz$*rxT@X?~iq8MEN+v@CA5y##eknu1{3fO)I6e$|n!?D|M)9FX44Z8m}Tj)3C(vkhSSUclLe;#X)9YleZY#EV!}? zu4=ydYwU`HR?ANIp)mYXCaejE^|)2*-wH-{ftk%Xb1gJQC3ImvG-xI)vNEn;V|Ne3 zwe{G0S(16&=JF|T;YlqtKEN8JJS*oM%pA>5@10Ol@;2GYAEqPsrJcI$>J(-5!n!|F z9-_rgaxCr%Q{C%&CE)XH_A50WVpcrF%Gl)_PD)Svy?kwCc@*Oqha^8_x{VJ2!KcxQ z+GSxnqx=NBvI~o;j;24u#B^AQyr5VLdr{Ze7TIX6+G&J8*qYy}jk8@>3W9hBbfJxr zui$I$;}f;-@JFDhofl@3EqaZ6s}Wb?Yen<^XPKv2;=K^6%3MG$B2=w4R9{_Ni!J|J zJhY>ZEQZ#WSPbo_*vyUQ!nEo!Gv6dX)gp~qvG-VY zC7zoIS3ZU{Ytf0vVd6Jrn-jrA?frTurS4FU+je$IdLbDEjLc7&a46MmtdVVo#)Q z-1o*pKL!(rn;iotBXjDoy9o}R1I6#iP9?@hs2)sR_sZZcRzh;?f}{ASUd~$iq%OqS ziA|DWO>W^`UdB^DfJeU$pK}S;WCjRMfdAv^$vu*5HH&9I<9h8f7!GBYp_^Aa^bj+n z#qlMwVG%W(qTNx|CnZ2Jk{DV>TLgo9onLlYJha^iG&fcd*|?uwzMpY3dXU$emQ zRqzsq53=)759Ik#?6cjuY&@e~K(1S!$8qFYIeV>aPJ_><`rpD>UimsTz^?)``Q_oD zYC&RQYI-7@gv^|1PYOjRd(nkjC;btUI~nxs{^7&;e0|Dd&|LtA+2HeVGIMpkETKo4 zkt%^z$<8+yhU$N4`-)f}F3g#Kor@O3@OiDS(A830Bn&mL7!4bh$!_ZmcUrhEcEIts zBURdGYmC_wsWWz^J3Clt2e{{8;z$@e1tw~Z{Z@FSN~Z7PDmzr|rcR{Nj3vRZ46 zLmpvH5X@*0bWE=6OIUai&aZ~k+QEJ{9DEI3q5apN!yA?6)$UaDkoN5OMI(sz@}rLW zu@a;d@{@yap2oxq^3^0=vYuhK{Obtn4A!26t%}Ys43*=s7$yS46kwPY4Es0*+-q3o zg#HfV<>6?gF>tR9OzdK~sJ-vBi_|biIf)U^H#rP5d3)*)< zRqFW+2QyF&my~%!&B03-9!1khj>W-u(S_ROY&6FREZAW4DIX%qbHFegdf`Q^N)qpq z^Ca54G+I1|@o(*nlz1o%?IBr<%#Y4zS*Y{HKfn6x2VK*$pc4z?CAMMa+i~}M9q{&c zF`}WMI0_!NFngbO0>y4{QTwQ9HwNusF&CD|PgIrh1(+Cg%#Ma{veN3^o|pUF;HSR- z8Ij}Uq@KVx zEJbV}4CD7^33JenJTfQ>Ih{E`GQd$jqB^3yE%Y8>qF#c73rk z0l|()_YOlL7-pfxR5r|Fn2_t442zNupR0^BC-kz{nz0{%W7^Gf1Q<4DN3;7uu|0=& z1J$mWLy)&O827t|iQ2{DF!n+F_}I=P$*|Gtfu9u{qaA$GVzE_M=+U#}Cce-`q+j{g zElBb*SU4R7-@<|o1--$1?m+ne1Uln2(4B#$+2>`(!jD2H&TyxYwaJTJRu!}&JsT}c z!{18`10L!#Y_{BDX!i>chI#Q48sl*d2zB$CqRS4$u%RY%_$YhoHGqlQRar=CXEW`Q zq`ep?Fg`n9DtSH$GPY~6_TEW|)TRSJ?U0g!$0_j$eX=e$VCxlLdysbs!R1)pnIQNk z+V4e!-T)BnhYjn)Ut3zOq*yt z(NUc9DcJWo2=+FgGYE}1)%c4CybSPgG#a&qI|RL*2Y*&o&_T2;39b}Js8$Dk<>gQ0wzqu$NXV6z)iK3J%Jz;=yP0)t_o5hICVfZ3ma+ek%zsok$N zEeNVRXnE(T^A5ERRak4b0A6TExSs4XTEWv^tqqXrx>%U@jIcWrr`;YkQzgqHE`EV@ zsz=x)`l03{QhSdBOkG+X03Ruqaw*rX7fE{bqa&?8=n3&q@~rqsb1HUi zKrDW1b>~xRZoc!Xk;(qh*yP8M+xqOdtUX(G)P{}nxjTcc_DLSb_+?q<5wmDjpSn$| zQuZQ|%Vk)14awFXdfNN#B6-52Nbx?f*6i`ecz!cYZsWoL)n7uiB zk|%lvU-1pzmztcz6aHv#Qs~*ZXgMpITy@0Cog}y;$ASH{(ibTVtxVgqSsbM`tX0w6 zp5~=AA5e(zHo?i^ZuP1%!@Lk*tv5#x_*V)hR*yUCq7_>kZPkZy%d*JFQOxow(y5&p z_9CsP&2iRXseSXbZ|fPJ*WAc9UL{TV0dhQ4A1{!2Z?Rf4QBg$4PLodQh1{X+q7IjEBdv|9wHQJLhbJ{cpk10M0i@BWhlDfWa_18OWA? zL9ckI7k+FJw2$5UR(l~?&&F!QFjpdjpUx7-&^`*TmTZ`MwaYR~Ce-DouG7JJH#ER2 zM=Y@vFSJ>wt%X3sP_N5WNPHUu2@+>C41sam)rFc0)aR*%7!|fp0o#3Wj zFja}P6I-mgV`;+KWVyx@nY@Tk{RC%v7(Lh)^qO!qMrYLEcW2Oj4qN>e=SWH(w<>#b z{Z8lF;@D-Ie+^sZyX}d_8mXW7=L#NMKhb6$grWL$r(|9#8d*(b`kcEySE>q+JAj{T^#m|XMs7IP|BPM^O_1cgWR7K( z)rBDNY%E7Ca=}<5f#Y9a4eQ+~d!yNHVVKHM-=}I==8~Z(;`=oIDx|Cz($x_R3ws5; z;vB^|N+HoT*kQd9-bd#+c^(ShuY=-DB1Ba!Y8Ul&AhsFYv|?=&k2e^PT64z>u*TEC z z|9aGs#WTQA6(hQi+EZIK_R`f(XseruD0iK~Q2M>TSIEl)j)lE~JSxi0>Xp1|jIbH9 z*V)9+&!DvwKfi;&{4OXi!|JLcejOQwjo`T&36=~iOQ;;zbgc0!*!>}RH2pY3C(hK` zNN_b6Sc9unichJ9G>5l-zRD7GP$g+A09$X=% zLZzu*xim^6nLmVRnZNYJQND$L$HNXYF+A4rbN~BC0Tb`$p=F|#X7v|I4-eIaf1tA{)Qj1R zb70X67S=_=@_K2#3|*S zC>{n334X`N^Iv)VSLd_O={~S3(ugVfMp=-M(o~G(3SD9H^ai}?!x*Zd<@0#Sy`-R+ zlHV!8F%vwA1;r9^C^m*K%5e8Z!wJXXu<<4Qov|=d^Zc)HXs%A#t*6nQeR*vcr1*Zr z^QvAsI9CMb#d3ABf?qmVp98KdUsRVUdM+_f7Gn4b?nw|VgEka~iu0^Ru^0wO{ne<0 z|NYN@hxx4Y+CyitkA$JBIRmSrx|?E1^F?Qxdx3e8S@^z9K&cd*Nb33Al6r|bl7V7o z#+nn}lmKB#v`|#^*Bg8kMXUN)NNSdD1dntK1wmyN`hrR)qPmvIX+3Z%3!@8iopN~5 z=!*25J-L^f$9d6^)schV_(R8-3)t?pa94t0DHAIT!!Qr;w#ph`q<-v~0`!94@lgyg zwDn)nWQR*Qi=F1~7sOT{z{94DuqgaU?EUUWc?mraEE5dK#80F7AOh(-!<18CzW5Xu;etFeAD`Tu1|uEE=*R z=kAMDKjCHc-}h3uleosk9mNCjt+Ms7J!Mg7Se%(Q^J>E=1*jodPN^u>;!OS3-!b`#@CE!XW1K~ zAv6fjrVu>L1cvGiq>8V=D&I>C!{@%6i2;Uox589dx@c7+91r`@)aHqmR_1_?)yD%FkAh;uKuj%*3HZ$=x&wB_jj951AB(vG6U4_lx_Z z`-}Sn$Il??Fj^~d*}=ft=$fF|2#>N2ICbMtgrV7y?k3uJ$Ycla2ft<@C~r`{Og^+~ zR%}!nu1Ny)gahAyu4-a$T*{EM$u#38~c=6d0DXjkf zd4elsKNT=c#Akc9r*~rTUmG|#oprAArh^TJ*%@O}?`QX0_j~s{?!N=a8_2YW+@-{V zDu&4qqbh++UGQmUWZKR~KL~PNOtjGn2^M~uX{}{sxF`t4z_#>Qkfg}(ABN{Ya{eDV zdt#oKpHm%gX&$o+XT49{?_pp?bYWJ!2w|vww5?pNG@<$evCzarZXTuc>*VnJS60liqmSm{>ar8(&bkleN$ z_ZvGb{OW*oB@izH?m1z2I->=n(6l!U&%fpT-*N6Jq&q7*u@*Ld0W%3Jy+_;)$Vq7s zwAt5i)+>I_MqWoGfBwP{*TJC_EKC98esC|imqGBNdx_uI zK=Efr>hd3<7>z_1MlY5GN6lX~{Dj-KJcrBY|4%lhJ(@(>Cl3*_P=vY ze&n3rxL4gTxkfjT=9Dlo8vQVY9LAS;nCaZ{SiB-gvfXLH&H?~^GElbdmaSOx@Wjuge~7N>f4Mp38PI5TXSK- z3xT3?Izp>9tZT$k-+0Q(3RZ=IR#u3*SI?8IsX-|k87+}Cf+}k zoZ|)fygt+e4?7R~P@U1`8`_z9n{S9bI{*BP_G*RDhk;emIty8q;&hyNFq8p3z5t0V z2^-kP2@FrV$GIKjI0=eh!kDkojJJ$lOlG{|SZuiU7^{#2tYY+I9eAm%U={FFJW`BT z#DaJx!@(%D;4NhMJFdx>T$yw33C@23M*i+5;8W9M>xYp2If-U`JXD$5O?3^GHdJIF z@7C6%-tA-m^U+7<6|%~bXK6&uEvl+HEk1+h_nJG&LL1zj=%99tI-Lw3vV7b<;2w4l zx`*5&JU$I$F2Nkx7YCavPbV5*N70M?x{@5_K(Z>lEQ<{;!{5R$uaV(2=m%NiKa322 z3;!-~T~3?NIKZQCc>Z^ejOc+OX6|~An=e!ltDFT{QNB~_DVX*#-~T_wM2la5p{$DC zfguJa)^XN4kGSuWzwgG#(|WfU(RKHzyUX3nu@5W{@YhMC?t($_7koy=f@$zaqG62K zSP(3W@b1zeT9ijc;_%C6ylMG@RtsJ?5`5P1?+EAJ>F(ytS74%isc5vx3uJL;d(Y9) zITyLPXk;Z7dPgfmqzmmh0$%;Qbzxvtg6vIVBqTN7LkyOAsFT~7`Q&asQ}9%1A{@mt0#nN9*5(-;er6YZD>%c~iL0&z!P(9ObsZ9F0l-f(|H zim!4GG4LqoJ7BKUY0miz$4xf}wtYBjvIcv3+-XDr>S>daZ>g-8yc$WeHMVuR|#C+Uy$Y6T>>+8tu33m$^ZUeuajC2oDebhb0_~q&R zWF$H<7^dd+nej<-Al12GWPY&B#bcY(O2_{d56hC=1bKNfmq1YX9R|TYT%ldyd4>O7 z=d2myshWUPvzxir=v80jYW3vOE&r;yxbr1}`zOJ2`cjQOUCrhO9$CIzb$urfXRG!@wD zNJu2&dcT49&)|LyD{_vjaEz<55A^mR#oOEsocS=ge#ifRaFb%)M`4*yxSc|!jc*uD zp3Z7RSwr3Ic<#pV-v4D$4X~AMR^%@KCN*{?yUE2KcAj=sEmR2K?JB(Zmhpb!u4RlH z4UR%m`tcB~k!HNeYs8#ikm^71OqAjAuo}vj2|4}kpjmIgu?B)xW)ejeQLl$J(5PW z>-SP~m9rzy<=I;{kz0ezeGTs#OuNjeFS={o&)xMPsoMq)X-46=A0|mR%Erj&xyGx% z1JQ4cru>25*EqgK>aW7Ev*tJf`UiOD9=Nv|+@vQqbIx@fyWspq&UTJh7vwr7_7b_X zk>O&@DQ7@hw2xW1dqez@g>~b<7g?(jZH>ETY^Al!5tqIiy6kGkXdjYx36`CAm$|FW z7}vtbwcxqYjD4HYk9)!KnBnAU5I6^#7kS4;!(ofZ`8f1W!rSA>tIoR@`P~Aqr5(2M z$YQvM_g~~So46GqYxaTrdZ;7UHn)jTQX((1$zq`O&jRgqH=6h6uYiAlt3f0evUBPx zDS0VEMN7I+KQ|U0N_J0z;w5lf4~82J2fr|nwlH$p7z^Notw&`_|9|`wT(RJ{m1$j1q4?jQ|03{T1pbS_e-ZdE0{=b)WP6k?(HxAu1x8jo zW6R65wqGmEHOrNeLv~&ru~aE4&(21;vSZ3!$y-+5D9ZRL|Gts@r#{IdD6Hh4S=;V` zmi*m>AfnxLjojyVA)>)Mav{(k-m;L>J8>4^r?KB4K zJ<1CYXFC%Rzm{e7;rmX8&^w_V?mNWDZKzV($Xu{?X|}8N(y#_sd-3F91!;2s2P%tS zC7Za=jdq8H+|c{|0WfBl|?ZA$U?+2c~~>I&?z4} z5b8q)rY)7&JD9!vlKHnU!BFccj!`psl$8MoI1ZbY7TRe*edAq|NV0Uc1Zs@j59TH;2>!cd%ZR)xg1b((nK5&p-S8;2Q=bkk?_ylAe_{`<->HyBN;8 zoqb5~EUGhyGvCq?z6rZ}1~JWmR$%_1G)GyEQXFNNQ?5WZzZ^4ss+(;JlD6`pEwwl! zko};C-8657+naTvh0wTSqgG|gj_LdSBR2lCaoro#|N2w)UF{(bYR!PYe^OQ@J?}ga z8tit4e>)lTC@SLmFmGBPw$-LKq!zQs<;>h?Ve;OEs9{jGQ32-m@|s6_PHS7VVxkOq zYR0rS$Uk7@zBQFiI*Os@;j zlMPfPqIwpe=%S3X)%huu8jEm%s*PdPz_mxqNlt5Us|3@Ef{#|O7i~c2xSl z!MC>01o!tI)4M;xA{YN&2&#%ebIz&gELk-4toszUV;%hKicY%^4%Gw!t!C2NDWQ`O zKRWjv_61lKhk1p$6^DPCV=l@gJuYE>Yb|O`WV<0aYITMxXlG+#)SG0U_oC~DN)j#V z+e#Z--!$x>3O)T;B+bMB19$e@Xl_N7ed-~Umj1uvnGsmY3gq|EUG+@`O>OjBNl>s= z=&{UvYA!h+oU_>38F{g-47ZZzcUkie)ihcBO7N`Jn5q@4kB00(m9~0*4#8q;PmJ?a z@r(>TfE=g`mDX9?8I&-JcXQ!C31Ik}2t&=2s%k{Pvp7?oGohTU0Nq1ndINZOA4f&7 zlZ58>VwgpaG5F;%QmnaTX*|2WR2HNPH?3aO%IESRTE^HJ)sa{vg>pG#ajKym9)RVK z;rk4vX7gKWhZ9jf?9^#9t9%^e*b6C)Tf6LOwM;rfXBPhN$b|7 zx4=5aRRt^10QoHjb~&gEj5f7_!OAIB!pdTZds=T-0V!1_Vl8euYNPpr%B9Nux17OQ zOcdU>a$Q!YCH|v&R6Rq**(Ivjv-|VCcJ5xH%G5AXv&x#858G+`tu1yI&G?w!D=h!| zT3HOmJyqRkt~nQ;OhLCDYmb*PuF9ZM4|MW?TvqCmqN!xik%h;(;G`@?anR5T@!CkH zj{9Jitvj?nc2n$!suI;%L;AFuVW$`>yj4Lf&Da*bsP0Z9S=V#b%R|S%L*85L>zEHV zCPKHUvwRR+3rqb3V+u3!UZH2t8+_j1{uMHs+t%90lDjJEb|(s1iWamLnU%mQCn&0N zFdI^=l~U4xMc|VBzB+NZHIMV6AHZ|$Vrtmc<)RCJsk*p55_>!VX~z)Cep1wm3cN_LBZUL_-`b-}MOxZICr=nCrHv9Ud|vX61= zgCE|Hqc7)Dh5I9{8h8*)JAi_y4($vy5+9NCrnbISn21wEuFhY@UQzW^R@nH(VJK5eM zadKM|sjLTjS|imJnSU7MwBAgsQ3g?~^#XFK;{}evXgRI*c?O%&ADsGtXAjs^He(IMNZNg&8Qxj;VfC8rGnq#k^U=>{?XZlpzu>(FjzMljkJ_@ zest!zN9Zy&h0%0@VPauv`D7IHqxh+D7`@9X)w_$Js~H|w0n|PgHI;w zr9Q)x!Q^d@3Fa8j|6es|4g<}nVXZo)J!UvO5S~6wBt8KQKcm~rHo6crr4wv_znwQA zREfC+?OUD(8>1DEHmk0Y2K@U+qyC?x`S;Ie5wtaP(k?+Sf^y{aZ-icDec@)t_82-# zJgjD{hN{uUzFJsT>AnZ}ulnoRm7?l4`3lHP0I})hS7(xwco)C?edKW-b=|guz1Gst z+vEkr97#2Tg7(9&UCgLTk1NRry)|a9G=5%^%&vLhtkEp=4wjGJJ zie42WF}>H|t$Kugu>$}6aw z$bDd{UR7PO4eFRZh*6I~N?$YRO$9rxo&S(b>=HPny&^v&yZ9N8R**APhX?g$(XI;f zd8PW7su#9+sTI0o;HG+*kK?Z?oMj!p)h@J3G-5(m|74nb5kHJPu^+cAd!#sI+jloZyL>@9x znL3)xJ^I@hs zKg>qjKLFQx{BIL^oJ(XOZqvo>G4?DS>nHaTgvyZz(QYLfuq-KzR#a3FWOjtY-`8S) zh0#BI?rse2`E|X7q2`5E52@XQ3vg|_IKMi3+@y3JT@AC?jRvMYPBx%9euumctzc9n z7UrLN4QMyl1>m=WOzK9k*$r9;4S&v%4LpyRsV;P1vP$P1vsSkIm-<%iHr<2NKWIJu zZ&q?Fhr{CL60U{r*D%j@+H~Lj)hmx}o(2;~c)daen8nV_%&O|1CC#cN4{VF&Vq`Il zk8!|3T`_qDvd8Kuk^v2!7fs#HsTs=b#n8=Y6%wT#7KWqo)FnuMpE@3C&jocDf5pr* zO+nt?M|#x}Xg%_}gWQB>Sk8l&visV{`v)>vKatB)561_;EAw$Q&MGwjMo>5mdZ$6=D*YmV0I?hF@8i+MRGnd?=xFLfmEU>p4`%d!GCdK+ z&9g}Aad^8MH21(v$@x|z@!L#Z`zx|w>VsMu4BuvVy%;~8+m>1FEXYj?bhvy9#aCj5 z)ri)%L<%>{Xx#@43~;o5wJ=PApOqTVH7niH>eH2FFn#+%{w6g1^LQ(75nssu3Qu)i z8H1jD+jOs)3w}%BpY{^og{ISnm zV4Q&@M^2Ul4F3v2&GyKzNrX;Gi6m#F-lKtYG&IAj3j*31ZUi32ROD9qJlS9IPrcb^ zq5-wX-lu4>bs(s*o?*WDIM{{Ile-L%bPDZB{ z?SpiaS^MkA_?KXL309r~%~M7joPniZaYb|ulF*^H4;Wrxr@mL+vFyl|ht3#ju!Bjs zx4S&5*L|ch6mr2&d=#x`XtV3u0Vfkk=0~1e(Us+w&PoI0}z%4#=tNt9&-` zZ@RIWlHkwKU7DZLP5~Dg>rbE;1up7|Eu?Z#ACi~70t&EiR6+LqD9U|4j{H=OWH)`s z#Z2M&J687wvj2@azUFFNGLOV+?e~?D^QB<#hJNg7{3ZKVtYC#e4_2y|#HZDaOMDFN z%uD={2MGM{e#d8_&Jf^eeM4Dg+vz?t*RmjT@c>Lr!8*qljO-wiItCUlK$?dU^J*9C z>BfVR@4O5KZa{PGNBYiy-*qFuNf>E5(1|hoD-;5?l4i$`GUNa&vX_eXpV8w|W)GEI zW)F+>Fg6*?b^Tl5sQ$&)3jNF@^)BYs!`8EM&@!6=e%XayET7Y2|Ff0s)?Wv&2<{tCfFskD)?M+y?6x;GLHh3}; zEU!)7N>^erH=_epA#wpJl=X0UkNVo@V0VQ=V8@=I;8vYZ!nN6frWU{Nqr0_sNUA|E z&GP)em|>>&0@1!w>VoeZt0Rk}^Jzcp-|4TN3Og^=!Cyi+@=CpY#cfw4FVjFfBd!}PGW6r8O`)?tqSv%k$x>+Qj&Xx&+6FiZgt zHD9INljigToyZ>%__-TF;deKN@~O1TWL6ki6b!q-#Io-9-h)Wa1^lyV*o}3tZ#+^w z4+btXR%EmB0@aWGGXCB#X78TV@FXwjl|gF-JHs^tw>Ipt*TEc}*lDi=duVFExHe{| zv-+S?6RwtmkA--5Hm*WC!^mV6xz{k+~$=V8_$mBzox z0Sf{Q!?JvDd8o5XXULaI3>UR?v35!<&Nq3Go-8%o-@Hc{%@y?B9OQO0Tzm%`Ed*DC z>jtdIJ~((5FYtS8s$@7bHX=7{DP!*@Im&A2xet_Q5yl&R&lXve#Za za1=uuA?20Pkfp&tFKiSaGk|0&*e3~3%eix4k*jm|rtF42-~Y`&?^p89gJA_QRIX0F z-U3+;^kF116q*0bM*BI+qsylXW0)2UyEqNq1o#lq{@2*txuCX#49g7k;4(bvFOXvG zt8fesYNwRjXsaZQRW_suI9{#4s7ci_04~1b$JRJ4F zR1QiWhvmrKV~>x+|MaodftJ}}KFX#97|OTGhE*#;{Q02ssPm8;4P$cpKNDvyM3N6- zRkX|PIv(!?*TY!iFPSG$ERqzaWMh=Y(1LcK*7k6#6Z{(hPJ_U0IQyN90=<#^*1q(^ zz;QVDgA6Ah= z5+wHs@~j=JHzL)_#+))c09ZRP3cGsq{+Hs8QOMt$9z_}{?p6V|LLpN%K zmo@YVyQ7wNVz~ZVzo8yc0S^`L*&`AN{*^8imSGHizCmK_atb`w?DS(9=?r%Bz>++~ zV2827+L>o98gUKUaSzWNMUKBg6W+k{O^(*eg+(ootknmDcF1N=r1VK-cQkvRzhRJ@ zfnPioziuiln&6M;_J;quLGd|W_cXe<8z{CXOWg#Ct__N142mVt5BWJtpd}je-|D<( zs=opZ6Gn~qih*GreC(WHm>M3cwnh5T%5ora_ZCCPe4c8^wGvp@QhgFSIOCnsZUIJ? zm-y=l>^nddvL1V~&ahBj>n|AuwaaaKv{qguttyh(3~kk!8IS(pKg=Ivu$st@j5CNT z=5WmAc;A1IeJ*DjJm2Cq!`UluD0<^ju-E?J+S$5+vD5Ov%W-H=xzb2+WALj5%crt$ zaDu2LQM0@n^wq70m#uvs)jKVz@v((ruP&9p!MbPz!!rJgqGIhX2Z9XEpgu z?KZlESywRs_eY$aHbe zZ!xR~7pK5OFDhl!XI{5ZA7;-=kh@W@cCENlU0nir5uOpym?QBMV5h4hYiu!8Jfv7J zF_uib-fBmoLRgiS^ac9ZEsJ~=CIfR4_MHT^%_g6$D&O-+@^ur#$|C1L)=D$hhK#)f z81zRnM`L|oXRPzUZ$9XK!l;*Xtn@$U_|$N55h%V38z;lXaoFtV(4g`eyT)1Qx<;O> zaOP65TyalhSY96tCvzSL3^#fGLytL);9*|8Y>Q!Xa(D?qFkFRmFELb1X?p^xQrKdc znY>LYve;XlPuy}?%pzn`PQ_uk1*@VA%?abx{J^D7fHus8B#VVLkid>eVo#75!HkhC z@GQ{#h}%*ST<5Qbfh+mF2^5#Z$0hvdeO@CB-$b5Az+cJpLneA@icYKzZaOT6mC(1s z@Bx1kXZqbw5w(Lote-F#=DUMoQurVq-U|%Fx-ej&wf(wgnwwXrn9O{W0(kvRoxRRy zZh0^)O2m2!2|fjG+O=qBT!!H?7DX9`R74PYVWDEZkvNSWWE|Wn0MZEB*@y8#=`nEzZG(z_&a?wu1+VQnAOm7c{ zZ8`ID|F)kzYOnV)JNYz*hxw4@bjY%Hl(K7R?u9JhjbVI!s2pq}Ji}DvV6(!*vaFpr z2!`btQEB|=vq%qf` zV0hC{8gX)x6IdSFkU1c}!eza2az(D}DQ7h{a&K2!tjW}@EbBU7Q-0qPC3Tf1YOmW_;?V0%9~h|*~sZa*tVM6T2R|#P~7hC z=GX=k*ZJ#t{!?D_0k53t%TpXdJS`q}MP^%b#`<@7Sc1=}0H$rw8Lc?~GV+HBqfUAA z*t??99Sl_)D_K^>fqaet$4FxsoL3B&<`wf*Zzv2i5u@ZMhQH6b;H-_qFo}N#49~=2 zsC9Mc$yi@Ex%p%!SEC4}HnR61EbLEg{0c0biAOUZNnHcm*22GC{y~2)_q(~@>+dw2 z+yEO_V{aCK;=87}lf5{aS3wV6J{rvas6?4|kHi z5f9Iqd6U~F%F9eNSQK4VmngX_EF4I5G8WHy7To#>wycG18$fI~^T7xF{UE9Pz5Kls zHf{#tPqEG);sZ^EU9TX`PjMx>Bh78#VOH5sTN4`1 z>d4aAlt7j>KPmkr41*(5JPe=DVyM3@hPLW3*f%8)7}j+z-@!1Me-Y%A<MWoCL_roz(U!QjUctd-$&MYzqvo^9|6z(2E{G#RKDT@ z5TA-wQ4Zp1Ec?S?*cuI6#~>&k7BLvs!dgpfyuy2~;bnhIrNbDiMM@(**@zEhQ^GsZ zNS3YC-3ttbqRub-ubMse71ykoW_I$>XSI&=wX?ykgk+V+1H4R@;XJdR;^7XHxxPZq z%VrxgnS4zRW77)b|lZ`RHv5eCufIS6+|i+4EGugWtZjKh=<=2=_SLn zk#|@MtE{ZlLyZ0zFnkkj_#X1P3`yOB#O^fwI|+(H@QiZ>}QUIpy(WpdW4J=$)38|^kP3~y1nk(#_= zJ}|7#c<+Zj{n4N!4Tg%57oiPzqWkuN*kNir&KUg8gW)lr-No~Jh*{R)r!0bpQ+S`n zP`O0u!-n7`9%^<#c|={SW;~}ksYU)}eC!xMF@3VNXL%v4q57KI^&ml<64=kkVQATB z^`bqyouij$BOazTJj_W>vl@Lt4!AYZ@RbY?_k-I-IJklQwK5Ku$vE8f5*ZBhV=-%? zqvRbv33Eror#aY(CGhKWG~pib+K&!A%VAMG3x@m6^LyZ_tjc0zQ^uR9`B{^X6%QL? zgR6mI1vpp$t5OO?n<3wg!EgyYOdVCs&&v9<=3rQmtFIMQDU3d}vTWtd`gM^((Q3fJ zvII!#bA+LGYt>qS7>=4~L^pKQt&3b%#co_7+jWU!=bfyL=0a~#k(!DqrLfUcjq!XR z!72~M%bNg(AA-xL_<-_hj)2)oX5r2mE}jR=BAeQdLyIp537M;d*X|y@N8bjzRrS&%GGQJLD`XW;CGqZ zDUOT&74zr_|FimVB|MaW_?F?}Ab2SMu(iqCSPYAxbqkaKs))`|t;vgE__ZG$)!J|F zeuzyePj!Z5ISov(dm>pLO6zN!k@*QYXj!OpTJ2~%g-9Q2uZq+L!x{#|)2{rmTG-yN z(Scva$?_iNl&xJ>Wr!+k^1_m8==ru_I1no|3M^*htt`e?Y%8&$4cGb;7r;c#^5cYe;Ohy91$Pm$&F z#-?OKj)FbuaUEeva6|?}@!4|v<)(lz*u1_LAkCaTuO)TjFip$DG~wWUj4E*$_7q zaEZ*#kDhCIs93KmGS(K4WB_r7Y|30%sEpSpW0R#3&!NAr`WHCFMS0nZ{dV&TXg@H;;?s+Zr#T?G%dH);$#6o!gZ z?0%0{(sUoWp8!XTqBy8S@v>}+c3Fsz;aT^7@&HZnqJJQBZDm=pp6cW-!NZ@yP%{C! zv6z*OEGyP~4x2e14|^f{a5*0IUbNm}G~!v;EHWK+aLWo|wKLuIl@p--R0zIPxlZeS$2 zGzb=A_FC1O1@RE;;@i{*!xjE@KUdUaet&m8vRuVrC?3i`ly@Upwis%(k@>M4)HMkX zVJH@=;!*ygd~ESBmN=ynJ-p7i4`3@>^6h_SMoKl(%42UgHP`20qUNL$V6}5F%1Vq) zd7H=4W+UL!EEA=yG&wKDbo-IrGaT}4B+K$`#KIl$a0PPzVO;J@lKcc1w&!g1jXfy| zg88XfiRPwx`dT1c7Y}NA9EML&#Tg$%+2z2uvC*D*6PX{2o^Xqgq4jMf%Zk;lf0&af zrHXUgIR=I^VOKU*UCSd3c(@-6qZK>f;AP9IWMh=&@PV4dVW@a{Dwur8IM?C-?J!(C z2zs(3N5N4Z_93Kv8yJ6v)PI0yGaC&s3T&)rV{08$kyr$Kl9yb0Mm|S%jWzKB>fk{M z!`P_4$nqwt$EzA1%BIA(%U04N$wP}>d=&MJm>B58G;uz5Rld{D&VKaa9CUjQDsE20 zL*>5KF)y_bPe(JQ*YFTMBEe`zS;lxbw!Yzcyu8W0Q@O9z21B8^5AK}?J>?uuA;mlS zkB;?7{D;_}Y4`xckYriq2Ti`A7V$|5Lk!=g3QYXY+2XchY#p$OH{qSS8SVnZ zEpZs0#j02gv*7=h#4>Bn?GZB0)7#-Es@5cUSt@y?T_}-Zf zhGDxL$a19gVOWL({Dh#@h*~i!47K}F47sn$XvE9TDz_;z)|sn++sLkDd5wvf55vPV zSQQ(oD1%tcSmyS4h>v4cl(~^K&j%mPq-?`>9E5|%jW@H`cr`mgcO3{S&YO#0^A0vy zEbPxY+G3NHeU&cEjT~f!g(;}&QFUHfj%wt`mf@*qih96r=dLvviid%36U58%I_$V3 zsSg7d28c?MCCfn$R=KZSSd<#{yl?@g{8!?{)LscFoWM(fG3@hPt z+)sQV3`fDkDc~{}+q?#@Z85TZ1P1Qokf5i@;3*$-wKMmu`?DyMauKBC(0eZ$UC3FH|#=`R}Ze0NBiZ*ny8*Q zxmj_Q2}Dc5&dLVEul%%8tkQ6ogQ0d**WRrbL#=@gM=2U{WPSn+0|r_Qtt=}}Q4Usn zsAmVmip&M_g1_ z5*SVvV{B}=@Sn>eU!%?jA0iks>9^0FwZ^82Z8-peyS)cW{HZzV9ER4SsG2R%hmrDa!Waf9TCU5g#K$lj z7*=3TYLhe2Ee?-*#$hPGTKmJTrsiQ6H8+~wyGFgp@ATq|M%R~N+^V$Bg%3~*uD&yyO=hHp5g2^9df(0pHLtvHf6^7zpAkTp`2l`Mv%*LEldFDpf zI9=S_THrWP~5ZkUr(IHK+0%8u1NKoQy_X z27spPB6Cu#nj#0=hU%7JK1DfL`Pd1JuF?o2FUtWR103(peOV04f#EWzftwKwALl#& zjx1|6_YP``Re`7#wpt-7jrfPBH5ED0h!yZhw3gygBHowqRK~-^Ih;WluEdvK3;Wh` ztmn7#{|nI+3($q{Ao;3SQk>isd(wnxt^!)GK%6~JLehMlBg>JoCpt@jTKwO4%W_g?7R5vDbz6^q{#o7I-ZI!xkQLeihU%`c7A8uX z55Y{$)O*IN6h!*!B8TmWG9D*p8;2}U#J^GP>qkhfYB*OLMy>$Wr93MZszPKOUX8r# z{@4?F*Y%87lNY}*J#w4~YZ4;b{GPc7`B7R0T^ybJ0rAd{L{O)ww`t8xKtWiT85XLl zT%D_|ENirp`w28dfT2*e(XuLTtbeHW9a+(brLif~ohbJ!ZwX$6#ZVP<+gKN{6w9n` z7wXm^tD;W!%4X-mVroX94f6OHKJW-4zPIpMG(-9!aw`m_164_ED-RbK-+BVCvo&+d zytYG<72_2~VzY2<5}OsIzw;^Iz(k)6M>L;R1f4sNi1Axy%eIF4gJFJnm=O$&oJ1H^VAC$QYl)-+KvLF&PxKqGCQ=T*^&yy@Z=;h^G*pyao%O2lL0V z=xw0&YI8)??b#c$?`E|_c-?WBUbLI_s{24 z-l$uO@;N`E5mTe5l)b58^4Z|a*S7w8GXGKDdIXFeh<z1eW72s-JQk=!k&x0(6?Ehk=CPt2zk0;;@`jA@mM z>R}t8)gD4Np8}niu)12GH-&g^Hda{G>of3qCgOu@z1~Z_<|$bF2>BuT*44=<;D=;$co6AT?=AJRvAw*c5l@=2DQeCP zKUB?82dyUmW&nQC3-~Q>V4oHDO-G_98Off6B@qW-gypL8>c#8tH*#Eo=p;85ITcv_ zfgZG-Mb6@l9^h(S$9}21Pz>*w#{9xM{|T>ks0#HqImsVpxMP=XtQS}mjWAL_0Tb{3 ztxr(IYb;7@l`uRMUK?2hKFxHMlb^WUthCTt$ZxRCK1|F4 z0(NGlJ<|LLn*T|x^Q+7_kKsI8DLM{3$MM(8a8N742f@M~c$DqP)zrWyiG`VsHTfND zaux2KhUtfhjJAWMcK%V%mTbJ|9r8dQ`L$R#T7`81xxrAordbSa)mMO_#vPfT_!tH{ zF+PTh)8*wWVk&}{{g@LSiuGQEMJ3_k5$y0TgW)9lil{g1Vo+QSJ7ur0nEZ{MYgSE6 zT{3_jL8gbX&%>}S!>}EST*kr6k#KMX{>)%(_mjkTor$trnCPS=7CAe7mK@*28lMBV zW5~dMK6wk@;UX4BU2mevO-v*QIomJbWy7Z2M@4&HzPH^;*v`;fNwy4(9EKJ}@jKwS zWujK>X{D6KFq&^t7>(H5`NN6w2BXVL;-zVih3()tn+{7e=u0>c6c^$tZbZr?&0=B{ z9!?HWP>%FIJni=XUwdaB=X16H@$cuHQO3~BFm_`Z%Z#yvLZwJZ5h{DO#OxJ3A=2!%@Yd%n(@Pv7sz?RL$#-#@>Z^Z3qpKi|)J zo!5Ig?{m)kfTH%%OJeUv^F4?qo&bVlC|Y$r1f6-k8=G`ubaD|a)XJoCjPEo){)YLQ zZQ!SjiJHgJKK{?ph~MyAW%1%4B?|OJLW(UCv|(hL1ry8RA!?O_BCy^&6i2-5#fyi& zwh%+r9$FbM%fhf2SCYoKsTSEfwnes@>$%pm91oR^busznrXtPHfugwh8n$@_EKz=? z!`MvC@%elNt&q*m4#gW`n{qge0=);h-9s4zrb9t+AQ&rZt+ngtqj$9;qy(PzF)Z*t zS`Q~2HixjvEwRcO=TxEV!CXlHiA{!4c-Nb9spX(=x_(Pje?QGc&z_IP@1p@XlqB=x%i=qTA#sQ-|w&{ zU&6W#)UAAo%iP#VjMoaj&%v-S_${F~Hkz$OV4IrjENv0)wr!l6sH8EJ|A>bw}e2c?$C{nC7*WI00sQf_nu_hIqE(Cy|u95p915E?$Ut_e>2A5Z9As8y_;nOZ_pKQx~+PB<|rL{8lJ7%cT zU}8<=zZsV0G7#zs+Xi4c27#aEoCbsF&Dfg$aI_omE8?n^NwxT18CV#wGG#Ap+YakK zfPqWVAa4-sn2#K+!AftYlz_csi2w9Sc$^iSy&0P-k4HM%P$ya_uYzP*Ubb&%yrlV; z82VSFA2oX;YbFdeCm{?w;vrsW($L%I5~W)Y6JCdPIb`vhNZz?89gefmhzn_@)-i3T z<%eA?Q8SoziC;B`i|vu;KCq?_NDhXJgNPOk0@FY6nmE`F>(hc?sE;*C2miu&h5M1% zZRoyrAh;ME@&(zaTp?n)<%1XquDb6*evlp{#WOuOMPC>UF;=2DvS%3_WA%EYacsP%NV~)!K ztJwkVgw$B<7#@!D&iS477lC3s>~m*$(+4j`^4kkb+y_jR$42{4l!2v=i!!9b!sCp2 zcEdL1gk1(|%1%6wNXJZWZ=y9ex)Dn<48D`-4>J&^S7ZPq{05+>TxB}rN4fiiVQh&H zhQ5}(_4;RGq1Z1Bl@Yc$zE(wYb+n1R5zMobu(5}*k3Q;^<1)c2|G!SZd=icLcgkC| zPE6bjhSsfoP&85dNt*EOw#ZyZn9~#dlxeOjh<4>Q&q3v-Qrtg{h^ryy|CO`HYv9}> z5d1rua4z=iX{1@6)Q9}iN@io*p~So zAH`Jq&||1oH_CKhB~qDL`pY8Y1J0U&OGn`0OW5Ig@bDh8nLbD!#EC8=tj-OcL+iAU zPkSM?15LYD%ECnDm}*EQrnQsl_KZbX#SiX z56sK(d^TkUJ@zcm-iGf>&>Ux=MQBuvykR#4Bj^`ki`LYA zZDc@l;II4ABS$wJjpD%z%_A;A{&A*8l+G0B=s1fOh{;3Ok<>8@b0z%&=Y;CB=^3TQ4 zl6VxQ2la~X<v?DXF@EEji`vVt%ucZ0%8}U`&z<1;iF=F*lqSdkHAcODIjW6AJ*KfdC0crpjh zXdRB`T)x7?Ip$_qO2IE>0rUBDg`{#s3eWSgB#j+{)sf>;j1f;`P0zQCO}0|UN;JdU zU^N>Ar&DtHmGS7_zu@I8MHb$-Y1A>0Xvj^reNdMf`ifkyW>d5tI=XHyZk71?0AV@)Fo>$!GC4rh{W{8sFW&N#Ub6erL&#UNgx z)yo_ccQ`*$W9L+~qE=m=>5B3apNALIfSa;})J6|Vise@)I?wtm^ywz*ScWxulNuGP zlpi_`NxmO1U=}v!9gtmaOEKp#h|zZ|TPbMlB45>zo|5kBg`wAmAy;Bso?k3|ovJ0u z$fzuP47JiiBb98V`Kri_$g`#en&&j;AYXwIFEGX%$w><%u`KsE6dy#+o`f$i;m7%P z#)|T3<~{-WOvP4I!s=$i#ya@X;-0ec)dkh+;9G(BQ?TO(ZLtr}dJBA44xQIw;fws1 z&c2ox^#H6_er`q2|I8;!!{i~1mP^?K#2YSTZ0h$&C3E%O!}c0K)c*NJF8cjwzl3#r zt*B?peV{XCW!K6f5w-c>i#Y6!rVi19q=bz~>r+lUjU?N>&WX}#M|Z;yoz^oAbYFIo zrc4F9VB$B(vrsG!&#J(;>fk6wD!X_!P}SI2tKy0?OJ}gh2k;8F;X$v3g>PY_UIeQg zxHpA9ngoKQdGrj(F5ntg*)nJZ<-yry7Q0*ywdwZ?Tzd+V9D3Mc8>2E8MdH*S3Bzd3 zzHU9&zBVt*%HLPW>E=3kIK9c&(aEqg zh()^w_A&Ak7q!kvnT5+Us;B~f>F}>Kw-RVft)w}I4LShopCZS~x$zcSV>X z?ICV=Q68cXb)HHNNTQ_&qW^up zQ4DpOtmRrxLpv7*!&LNP#mM~NEjs~iwhf!{H2Uco#(LMevwLpi{FK|kax^GD?0g)J zS>(N~CE}tzLHnGA4v^MjjD(AWWLYFzd9RhJS*wGOyZPgNXwWTu>z`QMx2Z$tA?2V2 zCvcwBSZ-tJtvkWA49P$Dz_qKYUd=!L|GP{ zgs%14I~>nF#+uR-uL=M^fe-2!|l&3ipXjSpb?M{yhB zPU;)Sum26bn}gm=$G?1%deTgz$ko{6dR(zadeTH8hM@->wle?tg*AB`^()HK9(Ru^=d)dM_grMG9B(oh9gkzY|-S0a{a$-TTnJchg71 z;l2F&sm@kU#}7}(Q_msm&ZaNH2`#F2Q4k08(T!X?j3;Np6? zrTwMK{rm%?=pS+8b&}6dXvw3L!+1b@cwe?;EwwL&!LRZC8T8a-EXbeG5+l$Rcf!I! z-0tPq=g?0x`Bpk{46^F7Ua2Z;zLfP+^Ww_-rjee;3n7N_wkRQ9C%f0E&&R*C`m7Y! zrIkxf@v_%3r_!Gh^9tl>5;}?18Jx)FPqZAsy+74ar>acOmFL&cYRY)3eSJF3Vjni+ zAl{Bnr8$D7(TN^fhqMP7*4*bu#4DCyOXj2P{)`k0!3U7yyXYU;t(&l7L*V07c>XxL zaxeO@GozHz_K!hh;!DbEq--AArKIRnAy5o43?(jnLRZ}WI$f;o@a{c+kkhqg^Z_YJUD|9m2-PK_VxxF{y-RFFR-aj(KDoyVZHvYM~j z!zfN!)((N(-rTYW25ttwb=Z>SpgtdMr;&-IcszFGPArzrdmRGX2e?z!CZSEH@R|Mi zHhq|*ZfEN-FWL+QE7Rx7IOq4=N|HT>9O1-P$~dNV&6!xb zj$~_ZWs+EzRU}~**h~bcad59EOuPmp`*FtE0JPO@aAuVAx1T`TbSi_iqCXo>JH|hA z7UoMBxd%(L3oEh*er^U~oxG(vr}_Bkf5qxQjy1j?-8a%9sFQ|o z7MCL?ep^DCJ;(_q%F0rXqSfr-im$L;(fjZeZz0@-t2_6W7eH{mTxbt9#!a>Cz=E1{sW(eyhjAK5k8I6D_|6pP&qxmrrM7F~Ur&0f2R+scw%-E2l7ctDuoCMgY{G@+`rK7>K6WS` zss-M&3E}!ZF^r-Zve9Qx(;7PM9IONzo1WuguoDOvt@@js(Z0V5t2mkJ;J5#l*do!$Ua~luJRQgal_D;r7$^saTHTw5BuX2e;(}+)FrtStkj1roBCo=z*|ED2_!cAI0a^ z>ZZTqb?XElfT z^)AELyny-Dj-uHAdQY!PpUAsWR?uqn za4S|j)HZwUHN;om0TrEeJ{S#kv2D$*6}OhIjh)?D%-6zEd5d?$sR_(n{ml3<;*th1FTfuKglchLhk;4NK){*$J-%=M6dqXW+e(SJkG$SdGs8^$Tc zY+BHfsF&6qmPYr;v(eT2F-nM$$1vLR@by!@kF6>8vFZyk^gJvK50gPK9Sl3MPi2<9 zim~pyXt`k^cq4eV0=tXsg_LHT2`E}3j zCEvdU4vOIo?b)`ztxahNrk7J^S89A749haRe3`u^=*ApCmb1z!&MM2V@iCitqV(%^ zSyYjVrxaXlE>$f4NAvGFQ5 zdQrq%-j3$4G#5Au

dGY;_p#NP@2eZ`l!XunoP?999dxGwGphTis?-b2FzWI?^Yz z7&TU5PUg6&8C*#ZXCf;l(cR(Pym*+mUB8Y&{NL5Dt0%oxNsj2eweuNI%@0P}Cagqw z5e$1fZ?GBY)u4sd+^q(!twX!T#f!k94Q%WVE`5;D>(O>Yuoy$(q-L^)x<^C7?*{Bd z|6Ffcc0^k60wnh=>Z)n8xK*>2Y*o-~N{#KncnBD7v{}rh&NaJ;9M(okl%Gl~3;a6v zyq-<`5jt6oQF)H?P;tc5oY$&sCry!UVN!2&Qzxu)rmaG$>~585Z5C`i z3j~@t3nRa{6Wo$+8~|EgvTuP&gD7I(AK;&KocCnffnakd!wo<%i~6ckD%!H3Scli; zkG7*1ULgKcC*ece$IRl&lv}B^8>46)yLw*{yzuY$;A3o^!p=VrU-hqOrrGaRQ9Q0Z z7&hig(}ItUcAvb?c&m?NL~~eJ!IgCPzmm;>k+qQNv%&Fve4n;3tAn%DijT>vDw3sf z+_etBK8)YGgYuQscPSVN!L#99ZPz;$!LOn%W7DWT8-B~rxeT4~7ov|@2{UYxxr@kQ z7Ncd2QMC5dW2orU$^K6M>d8L&-@JCRT7{!*v%J+}SmrWl8|j>XlI!MEbosm3g+6Hg zwzQ~*ElnvyDQ!!^m-03pPG&ioZUi0|!5n?^aU5x& zVc{qFEtgwwiampxv(U8Xqb1v-3+52TE}78Hz7kx=l||3titf-mS-bulT=M(b&@%gd zq<%+u7IBA%u`09KX}B7jdNpm9=C5v3DJ5)i{-6C@gtBGP5N}9Ko`Nih9eP6#W6lIyckC=)yXE3|hFY;>8pVd~I z@lLL$)fZ5z+hVqeElMduOOwD+h?YSDGhkC4tZ73$6pb=22CGY4l%fODE#|eR2)$fLs!DIByU_L_mDXhbE8B2AcA zlA{=g{`POf;Z#25d(Qu2h@r;sS_4!LjZ=>d89jm&*8G3Lc;gz{ehwO{I=EST8l@2T zg&dy6XmLp-u_BDB2`*kcUH~6eWU*Vp%;r39=GZ4M@GQqbFTo{Ybu#sy?&`JnmhEB#;l2jsE-_;3+m?}u@``x zG@G6^qLzB#oXzK|U^&Xb;$--j2=`QH#Qtc$GsjIaq&t(()It8o5ViRc|FVRc#;#2H z*Cpv^?W&P~?PrE!TOQlW*w^FpIB)G9K`+a~P-7MC_&FnYtWquVPNbJv!g_^SSXK56 z)B5UYtE05>nE9TzerJB*QH1uN4v&(-LtaT$K2;N6S!2Tb+!`Ub-hVz5Olmm`k^yU_ z0VTnS@GpRUM_oB=4su_FcQZh_Hd=HXbEya5@PgnryoDM+YeR2Q@?sdaCO&wbtd`~O}<}|LeCZ6p0$l+il5tCQ(nOo4FZ15>;h23}NTk{R&Td@2wmyd}*vr*cy z29`A(&D7(&HNj5Is>XZiFt#*^d;BcyJ8BM7*CB3So4w|UIURf}peeJlMHB2cR+bz! zJ%dho3)Rqu#d9r+ALDtg6m84z`zVT`=V0uw1rj8kC`V-7lzh<8`$w5=DNi*~QtPd~Ac`b@>QM8@0 zEsXc$6EW0tSvc)jDe8R0bmZpzNS$D#*~qNiFziAW@|TY8I|x(unXhQ;7qoaE?cNWj zhmh-^j76%Y9aG>@8Q4}H%qmgJgIqa^{AbB;5@;7fqaFo+N$*~>hoTyHQx4J}$$YXT zJnTvIaTB(=Inh^ntvbg{S%|fJ-CLCC7$u6*{}nf5drOaXMe?-$+M9BCVi}PSBO{1B zY-YsV8Vu!IC)oq$e`xP+v(uG5;P@37h>_ysaoT?x2nk2|J;G9Qt7o#mMbQSb>C%AT zf&4e-0N>tiKBw&Dww?c9Q;YO#D%$fJyu&qU!>z#tbfNrfKT?)&=yjpzLu_lJSjPL& ziGAaLJM^MF4<*ggh$Ya7igZ2|NinxG!#)8IFWYe_5gvYSc2GVu+sqd8FYZ4FgD*gG zf394II2M7ENqEgdPLGnn&%(ZAx$?Uo7VqVIyQ$+}l&##h@c1kGLj7bKMv?Ypr5JcjbMmAzdWu__o2j64@)Gq*k+A0rzc$%{J( zc53lvv&kUo=2IST1Jj+bbDz_XN6>J>(YS(x3~~nMiG9+3N5ES0E6Lr-H@8y9C$4NX zAA{l^wiAMJy`4mJ+*a>S!Gt5 z_ss`pt;6you-xJpxf_HIVqN9?9D-BQdcFudS&lv6w2i*_gx>gwn*KpqMNKQI`4h0+ z$2Y$>$#?^g(0|p)8TlA;;eA6tryQ#deJlL+hb_+gEPORy_rH!|D9u{)kO7L*BGzYt26$g2He)D2hem=0|Y! zW0)kl-ofoN{_mp5-m1t7Ns9_MvF`)Nz4y%u$LM9|J?=N~oi8c>gBEB){QFXSohcbK z1VhCQl|w|UUSs{j5G#KlWnlPv{xLhTh7iLLvlH(n&06KG(MlOc`KiHVb2CxtiuMP1 zq_P9N-goFNr`1cj2?gQs567jout`YDLVN<(w!p1Tx!6gEuFcg0D;;vn`L-VE`zz=R z@$f6Sya_g*i7kWs!CPqO@?4Mn8b40Xi%)DDPHb0tPHA@_d@A7{s&z8YUA^HJmh-V&E>AQb8jF$76rF>_R zS!mwj|2y0-cHexD?=IoC3BK;67Z$?iG`#a1Gb*@&tC4rD=(X(0iE~i@u6@63@u|L& zx6fj0iN!E{>_;owDWH6{rN}<! zu0mBkzNgkJJ5;o#vuRo#UkY&%c|rp;jegUR;~=Kk}Ma# qTyS52Kmh^;2oxYtfItBP1qc)%P=G)I0tE;ZAW(on0RsQ85cpr{OnTJ- literal 0 HcmV?d00001 diff --git a/heudiconv/tests/test_dicoms.py b/heudiconv/tests/test_dicoms.py index 3700297a..10aa9ef5 100644 --- a/heudiconv/tests/test_dicoms.py +++ b/heudiconv/tests/test_dicoms.py @@ -12,6 +12,8 @@ embed_dicom_and_nifti_metadata, group_dicoms_into_seqinfos, parse_private_csa_header, + get_datetime_from_dcm, + get_timestamp_from_series ) from .utils import ( assert_cwd_unchanged, @@ -85,3 +87,51 @@ def test_group_dicoms_into_seqinfos(tmpdir): assert type(seqinfo) is OrderedDict assert len(seqinfo) == len(dcmfiles) assert [s.series_description for s in seqinfo] == ['AAHead_Scout_32ch-head-coil', 'PhoenixZIPReport'] + + + +def test_get_datetime_from_dcm(): + import datetime + typical_dcm = dcm.dcmread(op.join(TESTS_DATA_PATH, 'phantom.dcm'), stop_before_pixels=True) + XA30_enhanced_dcm = dcm.dcmread(op.join(TESTS_DATA_PATH, 'MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm'), stop_before_pixels=True) + + # do we try to grab from AcquisitionDate/AcquisitionTime first when available? + assert type(get_datetime_from_dcm(typical_dcm)) is datetime.datetime + assert get_datetime_from_dcm(typical_dcm) == datetime.datetime.strptime( + typical_dcm.get("AcquisitionDate") + typical_dcm.get("AcquisitionTime"), + "%Y%m%d%H%M%S.%f" + ) + + # can we rely on AcquisitionDateTime if AcquisitionDate and AcquisitionTime not there? + assert type(get_datetime_from_dcm(XA30_enhanced_dcm)) is datetime.datetime + + # if these aren't available, can we rely on AcquisitionDateTime? + del XA30_enhanced_dcm.SeriesDate + del XA30_enhanced_dcm.SeriesTime + assert type(get_datetime_from_dcm(XA30_enhanced_dcm)) is datetime.datetime + + # and if there's no known source (e.g., after anonymization), are we still able to proceed? + del XA30_enhanced_dcm.AcquisitionDateTime + assert get_datetime_from_dcm(XA30_enhanced_dcm) is None + + +def test_get_timestamp_from_series(tmpdir): + dcmfile = op.join(TESTS_DATA_PATH, 'phantom.dcm') + + assert type(get_timestamp_from_series([dcmfile])) is int + + # can this function return an int when we don't have any useable dates? + typical_dcm = dcm.dcmread(op.join(TESTS_DATA_PATH, 'phantom.dcm'), stop_before_pixels=True) + del typical_dcm.InstanceCreationDate + del typical_dcm.StudyDate + del typical_dcm.SeriesDate + del typical_dcm.AcquisitionDate + del typical_dcm.ContentDate + del typical_dcm.PerformedProcedureStepStartDate + tmp_dcmfile = tmpdir / "phantom.dcm" + dcm.dcmwrite(tmp_dcmfile, typical_dcm) + + assert type(get_timestamp_from_series([tmp_dcmfile])) is int + + + From 58708e288edbb82e9adb829d2d2e0faacfd818f4 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 19 Dec 2022 16:44:40 -0500 Subject: [PATCH 045/176] Add a bash anon-cmd to be used to incrementally anonymize sids A simple bash script which would keep local ../.git/anon_sid_map.csv (assuming the script goes e.g. into code/ or some other subfolder like .heudiconv/), and could be used as anonymization script --- utils/anon-cmd | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 utils/anon-cmd diff --git a/utils/anon-cmd b/utils/anon-cmd new file mode 100755 index 00000000..43be849e --- /dev/null +++ b/utils/anon-cmd @@ -0,0 +1,42 @@ +#!/bin/bash +# Generic anonymization script which would anonymize sid based on what it had +# seen in the past or simply what the translation dict already has. + +set -eu + +debug() { + : echo "DEBUG: $*" >&2 +} + +# Translation file location +# Store under .git by default to guarantee that it is not committed or locked by git-annex etc +# But it might not fit some usecases where there is no .git +anon_file_default=$(dirname "$0")/../.git/anon_sid_map.csv +anon_file="${AC_ANON_FILE:-$anon_file_default}" +anon_fmt="${AC_ANON_FMT:-%03d}" + +sid="$1" + +# harmonize since elderly awk on rolando seems to have no clue about IGNORECASE +sid=$(echo "$sid" | tr '[:lower:]' '[:upper:]') + +debug "Using $anon_file to map $sid" + +if [ ! -e "$anon_file" ]; then + touch "$anon_file" # initiate it +fi + +res=$(grep "^$sid," "$anon_file" | head -n 1) +if [ -n "$res" ]; then + ann="${res##*,}" + debug "Found $ann in '$res'" +else + # need to take the latest one + largest=$(sed -e 's/.*,//g' "$anon_file" | sort -n | tail -n1) + next=$((largest+1)) + # shellcheck disable=SC2059 + ann=$(printf "$anon_fmt" $next) + debug "Found $largest and $next to get $ann, storing" + echo "$sid,$ann" >> "$anon_file" +fi +echo "$ann" From c76737fc3c24f77debcc911efbfc80cfe38b63bf Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 19 Dec 2022 17:19:56 -0500 Subject: [PATCH 046/176] remove trailing 0s or bash would consider 009 base 8 and thus illegal --- utils/anon-cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/anon-cmd b/utils/anon-cmd index 43be849e..b49e48f5 100755 --- a/utils/anon-cmd +++ b/utils/anon-cmd @@ -32,7 +32,7 @@ if [ -n "$res" ]; then debug "Found $ann in '$res'" else # need to take the latest one - largest=$(sed -e 's/.*,//g' "$anon_file" | sort -n | tail -n1) + largest=$(sed -e 's/.*,//g' "$anon_file" | sort -n | tail -n1 | sed -e 's,^0*,,g') next=$((largest+1)) # shellcheck disable=SC2059 ann=$(printf "$anon_fmt" $next) From 59922c3f1ba70e64d629be84c005c1a114c1a1a3 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 4 Jan 2023 09:35:57 -0500 Subject: [PATCH 047/176] fix minor typo --- docs/usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.rst b/docs/usage.rst index 05a73676..fc485e43 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -47,7 +47,7 @@ If using bids, the ``notop`` bids option suppresses creation of top-level files in the bids directory (e.g., ``dataset_description.json``) to avoid possible race conditions. These files may be generated later with ``populate_templates.sh`` -below (except for ``participants.tsv``, which must be create +below (except for ``participants.tsv``, which must be created manually). .. code:: shell From ccc2eacdf09c3bff8b51541deb71d5c914d95bbc Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 4 Jan 2023 09:39:19 -0500 Subject: [PATCH 048/176] minor fix -- Fix use of code:: directive --- docs/usage.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/usage.rst b/docs/usage.rst index fc485e43..b7a7175d 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -92,7 +92,8 @@ The second script processes a DICOM directory with ``heudiconv`` using the built This script creates the top-level bids files (e.g., ``dataset_description.json``) -..code:: shell +.. code:: shell + #!/bin/bash set -eu From aea555b22527650094e6751cea39fdff454070e0 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 23 Jan 2023 12:32:45 -0500 Subject: [PATCH 049/176] BF: Use .get in group_dicoms_into_seqinfos to not puke if SeriesDescription is missing happened on some hyperscanning data I was given my hands to touch at Dartmouth --- heudiconv/dicoms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/dicoms.py b/heudiconv/dicoms.py index 00692fe1..1a5c2b95 100644 --- a/heudiconv/dicoms.py +++ b/heudiconv/dicoms.py @@ -267,7 +267,7 @@ def group_dicoms_into_seqinfos(files, grouping, file_filter=None, if mw.image_shape is None: # this whole thing has no image data (maybe just PSg DICOMs) # If this is a Siemens PhoenixZipReport or PhysioLog, keep it: - if mw.dcm_data.SeriesDescription == 'PhoenixZIPReport': + if mw.dcm_data.get('SeriesDescription') == 'PhoenixZIPReport': # give it a dummy shape, so that we can continue: mw.image_shape = (0, 0, 0) else: From dbd69c93609927b0456be1d6184f25115fa0caf1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 24 Jan 2023 14:34:58 -0500 Subject: [PATCH 050/176] codespell config for what to ignore --- .codespellrc | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .codespellrc diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 00000000..4757ec97 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,4 @@ +[codespell] +skip = .git,.venv,venvs,*.svg,_build +# te -- TE as codespell is case insensitive +ignore-words-list = te From 9a0d4ac9875b3eff6a2fb1ff2784e4a8466f3a73 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 25 Jan 2023 11:17:47 -0500 Subject: [PATCH 051/176] [DATALAD RUNCMD] Run codespell -w === Do not change lines below === { "chain": [], "cmd": "codespell -w", "exit": 0, "extra_inputs": [], "inputs": [], "outputs": [], "pwd": "." } ^^^ Do not change lines above ^^^ --- CHANGELOG.md | 6 +++--- utils/test-compare-two-versions.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d59ff6c..03a3d4d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -120,7 +120,7 @@ - try a simple fix for wrongly ordered files in tar file [#535](https://github.com/nipy/heudiconv/pull/535) ([@bpinsard](https://github.com/bpinsard)) - BF: Fix the order of the 'echo' entity in the filename [#542](https://github.com/nipy/heudiconv/pull/542) ([@pvelasco](https://github.com/pvelasco)) - ENH: add HeudiconvVersion to sidecar .json files [#529](https://github.com/nipy/heudiconv/pull/529) ([@yarikoptic](https://github.com/yarikoptic)) -- BF (TST): make anonymize_script actually output anything and map determinstically [#511](https://github.com/nipy/heudiconv/pull/511) ([@yarikoptic](https://github.com/yarikoptic)) +- BF (TST): make anonymize_script actually output anything and map deterministically [#511](https://github.com/nipy/heudiconv/pull/511) ([@yarikoptic](https://github.com/yarikoptic)) - Rename DICOMCONVERT_README.md to README.md [#4](https://github.com/nipy/heudiconv/pull/4) ([@satra](https://github.com/satra)) #### ⚠️ Pushed to `master` @@ -165,7 +165,7 @@ Various improvements and compatibility/support (dcm2niix, datalad) changes. - Python 3.5 EOLed, supported (tested) versions now: 3.6 - 3.9 - In reprorin heuristic, allow for having multiple accessions since now there is - `-g all` groupping ([#508][]) + `-g all` grouping ([#508][]) - For BIDS, produce a singular `scans.json` at the top level, and not one per sub/ses (generates too many identical files) ([#507][]) @@ -486,7 +486,7 @@ A usable release to support [DBIC][] use-case A somewhat working release on the way to support [DBIC][] use-case ## Added - more tests -- groupping of dicoms by series if provided +- grouping of dicoms by series if provided - many more features and fixes # [0.2] - 2016-10-20 diff --git a/utils/test-compare-two-versions.sh b/utils/test-compare-two-versions.sh index d063b18e..9dd7e77a 100755 --- a/utils/test-compare-two-versions.sh +++ b/utils/test-compare-two-versions.sh @@ -26,7 +26,7 @@ function run() { shift source $heudiconvdir/venvs/dev3/bin/activate whichheudiconv=$(which heudiconv) - # to get "reproducible" dataset UUIDs (might be detremental if we had multiple datalad calls + # to get "reproducible" dataset UUIDs (might be detrimental if we had multiple datalad calls # but since we use python API for datalad, should be Ok) export DATALAD_SEED=1 From c492ee9e0e6f4175b5cb032a794db90295c1c8d7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 25 Jan 2023 11:19:21 -0500 Subject: [PATCH 052/176] add codespell workflow --- .github/workflows/codespell.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/codespell.yml diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml new file mode 100644 index 00000000..5768d7c6 --- /dev/null +++ b/.github/workflows/codespell.yml @@ -0,0 +1,19 @@ +--- +name: Codespell + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Codespell + uses: codespell-project/actions-codespell@v1 From d262c55d88ca2d1588eb740af228925bde7c5dc9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 25 Jan 2023 12:52:43 -0500 Subject: [PATCH 053/176] Do not issue warning if cannot parse _task entity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also adjusted docstring to note that now _task- can be used in many other modalities ❯ git grep -l -B4 -E 'task: (r|o)' src/schema/README.md src/schema/rules/files/raw/anat.yaml src/schema/rules/files/raw/beh.yaml src/schema/rules/files/raw/channels.yaml src/schema/rules/files/raw/eeg.yaml src/schema/rules/files/raw/func.yaml src/schema/rules/files/raw/ieeg.yaml src/schema/rules/files/raw/meg.yaml src/schema/rules/files/raw/nirs.yaml src/schema/rules/files/raw/pet.yaml src/schema/rules/files/raw/task.yaml and overall -- even if required for some modality (e.g. func) IMHO we should not redo bids-validator functionality here. It should be validator to report oddities (may be we should bolt on integration with basic validation present in bidsschematools). --- heudiconv/convert.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index 362c0544..4e5e2115 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -860,7 +860,10 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam def add_taskname_to_infofile(infofiles): - """Add the "TaskName" field to json files corresponding to func images. + """Add the "TaskName" field to json files with _task- entity in the name. + + Note: _task- entity could be present not only in functional data + but in many other modalities now. Parameters ---------- @@ -879,7 +882,8 @@ def add_taskname_to_infofile(infofiles): op.basename(infofile)) .group(0).split('_')[0]) except AttributeError: - lgr.warning("Failed to find task field in {0}.".format(infofile)) + # leave it to bids-validator to validate/inform about presence + # of required entities/fields. continue # write to outfile From df3811f9933962bcd44deee44b151a9a4197b1f7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 25 Jan 2023 13:06:59 -0500 Subject: [PATCH 054/176] do not remap if already matches anonymized -- workaround around oddity of heudiconv --- utils/anon-cmd | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/utils/anon-cmd b/utils/anon-cmd index b49e48f5..c655a3e2 100755 --- a/utils/anon-cmd +++ b/utils/anon-cmd @@ -26,11 +26,23 @@ if [ ! -e "$anon_file" ]; then touch "$anon_file" # initiate it fi +# apparently heudiconv passes even those we provided in `-s` CLI option +# to anonymization script. So, we will have to match those by our format +# and then give back if matches. That would forbid plain remapping though if +# original ids are in the same format, so some folks might want to disable that! +sid_input_fmted=$(echo "$sid" | sed -e 's,^0*,,g' | xargs printf "$anon_fmt" 2>&1 || :) +if [ "$sid" = "$sid_input_fmted" ]; then + debug already in the anonymized format + echo "$sid" + exit 0 +fi + res=$(grep "^$sid," "$anon_file" | head -n 1) if [ -n "$res" ]; then ann="${res##*,}" debug "Found $ann in '$res'" else + echo "We have all sids mapped already! Will not create a new one for $sid" >&2; exit 1 # need to take the latest one largest=$(sed -e 's/.*,//g' "$anon_file" | sort -n | tail -n1 | sed -e 's,^0*,,g') next=$((largest+1)) From 8886daa8ca404b1bb4869d3240b5d8dd33e72278 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 25 Jan 2023 13:16:20 -0500 Subject: [PATCH 055/176] Duecredit dcm2niix Closes #132 as we already have duecredit added, and just forgot to credit dcm2niix for all its hard work --- heudiconv/convert.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index 362c0544..4c57b788 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -505,7 +505,12 @@ def convert(items, converter, scaninfo_suffix, custom_callable, with_prov, elif outtype in ['nii', 'nii.gz']: assert converter == 'dcm2niix', ('Invalid converter ' '{}'.format(converter)) - + due.cite( + Doi('10.1016/j.jneumeth.2016.03.001'), + path='dcm2niix', + description="DICOM to NIfTI + .json sidecar conversion utility", + tags=["implementation"] + ) outname, scaninfo = (prefix + '.' + outtype, prefix + scaninfo_suffix) From 5b6ddefe4ec07593eac0c898db5cbb855d7c88e1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 25 Jan 2023 13:50:06 -0500 Subject: [PATCH 056/176] Add HOWTO 101 section, with references to ReproIn to README.rst I think at large this is Closes #115 --- README.rst | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 0ac8ab7f..a8c1c110 100644 --- a/README.rst +++ b/README.rst @@ -33,10 +33,37 @@ into structured directory layouts. - it allows flexible directory layouts and naming schemes through customizable heuristics implementations - it only converts the necessary DICOMs, not everything in a directory - you can keep links to DICOM files in the participant layout -- using dcm2niix under the hood, it's fast +- using `dcm2niix `_ under the hood, it's fast - it can track the provenance of the conversion from DICOM to NIfTI in W3C PROV format -- it provides assistance in converting to `BIDS `_. -- it integrates with `DataLad `_ to place converted and original data under git/git-annex version control, while automatically annotating files with sensitive information (e.g., non-defaced anatomicals, etc) +- it provides assistance in converting to `BIDS `_ +- it integrates with `DataLad `_ to place converted and original data under git/git-annex + version control, while automatically annotating files with sensitive information (e.g., non-defaced anatomicals, etc). + +HOWTO 101 +--------- + +In a nutshell -- ``heudiconv`` operates using a heuristic which, given metadata from DICOMs, would decide how to name +resultant (from conversion using `dcm2niix`_) files. Heuristic `convertall `_ could also be used to actually have no real +heuristic and simply establish your own conversion mapping by editing produced mapping files. +In most use-cases of retrospecive study data conversion you would need to create your custom heuristic following +`existing heuristics as examples `_ and/or +referring to `"Heuristic" section `_ in the documentation. +**Note** that `ReproIn heuristic `_ is +generic and powerful enough to be adopted virtually for *any* study: For prospective studies you would just need +to name your sequences following the `ReproIn convention `_ and for +retrospective conversions, you often would be able to create a new very versatile heuristic by simply providing +remappings into ReproIn as shown in `this issue (documentation is coming) `_. + +Having decided on heuristic you could use command line:: + + heudiconv -f HEURISTIC-FILE-OR-NAME -o OUTPUT-PATH --files INPUT-PATHs + +with various additional options (see ``heudiconv --help`` or +`"Usage" in documentation `__) to tune its behavior, to +convert your data. See e.g. `ReproIn conversion invocation examples `_. + +Please also see `user tutorials `_ in documentation. How to cite ----------- @@ -52,9 +79,7 @@ How to contribute HeuDiConv sources are managed with Git on `GitHub `_. Please file issues and suggest changes via Pull Requests. -HeuDiConv requires installation of -`dcm2niix `_ and optionally -`DataLad`_. +HeuDiConv requires installation of `dcm2niix`_ and optionally `DataLad`_. For development you will need a non-shallow clone (so there is a recent released tag) of the aforementioned repository. You can then From b93fdf2eecd2be62be0bad3c5b19c6b06090d656 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 25 Jan 2023 14:17:33 -0500 Subject: [PATCH 057/176] ENH: description how to map IDs on docker and reference ///repronim/containers Closes #241 --- docs/installation.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index ee15c6df..adced6a2 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -28,6 +28,13 @@ to view available releases. To pull the latest release, run:: $ docker pull nipy/heudiconv:latest +Note that when using via ``docker run`` you might need to provide your user and group IDs so they map correspondingly +within container, i.e. like:: + + $ docker run --user=$(id -u):$(id -g) -e "UID=$(id -u)" -e "GID=$(id -g)" --rm -t -v $PWD:$PWD nipy/heudiconv:latest [OPTIONS TO FOLLOW] + +`ReproIn heuristic project `_ provides its own Docker images from +Docker Hub `repronim/reproin` which bundle its `reproin` helper. Singularity =========== @@ -36,3 +43,17 @@ you can use it to pull and convert our Docker images! For example, to pull and build the latest release, you can run:: $ singularity pull docker://nipy/heudiconv:latest + +Singularity YODA style using ///repronim/containers +=================================================== + +ReproNim project provides `///repronim/containers `_ +(git clone present also on `GitHub `__) `DataLad +`_ dataset with Singularity containers for many popular neuroimaging tools, e.g. all BIDS-Apps. +It also contains converted from Docker singularity images for stock heudiconv images (as `nipy-heudiconv +`__) and reproin images (as `repronim-reproin +`__). Please see `"A typical workflow" +`_ section for a prototypical example of using +`datalad-container `_ extension with this dataset, while fulfilling +`YODA principles `_. **Note** that it should also work on +OSX with ``///repronim/containers`` automagically taking care about running those Singularity containers via Docker. \ No newline at end of file From b57665d966e727d2ee627996be417fa254611ad4 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 25 Jan 2023 13:27:09 -0500 Subject: [PATCH 058/176] Add .duecredit.p to both .bidsignore and .gitignore --- heudiconv/bids.py | 3 +++ heudiconv/tests/test_main.py | 1 + 2 files changed, 4 insertions(+) diff --git a/heudiconv/bids.py b/heudiconv/bids.py index 56339fc2..12bad0fb 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -154,6 +154,9 @@ def populate_bids_templates(path, defaults={}): create_file_if_missing(op.join(path, 'scans.json'), json_dumps(SCANS_FILE_FIELDS, sort_keys=False) ) + create_file_if_missing(op.join(path, '.bidsignore'), ".duecredit.p") + if op.lexists(op.join(path, '.git')): + create_file_if_missing(op.join(path, '.gitignore'), ".duecredit.p") populate_aggregated_jsons(path) diff --git a/heudiconv/tests/test_main.py b/heudiconv/tests/test_main.py index a845086e..c67ff899 100644 --- a/heudiconv/tests/test_main.py +++ b/heudiconv/tests/test_main.py @@ -131,6 +131,7 @@ def test_prepare_for_datalad(tmpdir): # the last one should have been the study target_files = { + '.bidsignore', '.gitattributes', '.datalad/config', '.datalad/.gitattributes', 'dataset_description.json', From a83a6bdde3f652022a05a371c4e0a5e1721c2340 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 31 Jan 2023 15:11:57 -0500 Subject: [PATCH 059/176] Tune up .mailmap to harmonize Pablo, Dae and Mathias --- .mailmap | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.mailmap b/.mailmap index 732eb4d7..9c444f27 100644 --- a/.mailmap +++ b/.mailmap @@ -3,3 +3,9 @@ Matteo Visconti di Oleggio Castello Matteo Visconti di Oleggio Castello Matteo Visconti di Oleggio Castello Chris Filo Gorgolewski Chris Gorgolewski +Pablo Velasco +Pablo Velasco +Dae Houlihan +Dae Houlihan +Mathias Goncalves +Mathias Goncalves From 8eb9ccec9648ebc477e2a1349c5b47c026caf5de Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 31 Jan 2023 15:55:27 -0500 Subject: [PATCH 060/176] DOC: add clarification on where docs/requirements.txt should be "installed" from --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 83948e26..81964c3a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ +# should be "installed" from top directory since that one refers to .[all] sphinx-argparse sphinxcontrib-napoleon -r ../dev-requirements.txt From 76ee6d9a1ddc1622bba73c49e0378ac6814f545a Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Tue, 31 Jan 2023 16:13:13 -0500 Subject: [PATCH 061/176] dcm2niix explicitly noted as a dependency Closes: https://github.com/nipy/heudiconv/issues/627 --- heudiconv/info.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/heudiconv/info.py b/heudiconv/info.py index 3f3b4a63..e60769ce 100644 --- a/heudiconv/info.py +++ b/heudiconv/info.py @@ -22,12 +22,13 @@ PYTHON_REQUIRES = ">=3.7" REQUIRES = [ - 'nibabel', - 'pydicom', - 'nipype >=1.2.3', + 'dcm2niix', 'dcmstack>=0.8', 'etelemetry', 'filelock>=3.0.12', + 'nibabel', + 'nipype >=1.2.3', + 'pydicom', ] TESTS_REQUIRES = [ From a02261103fd8eff7c0057566817c444805f984a3 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 31 Jan 2023 13:17:41 -0800 Subject: [PATCH 062/176] Capitalize sentences and end sentences with period Capitalize sentences in the intro paragraph and end them with a period. Additionally, a bit of rewording in the second point about converting DICOMs in a directory --- README.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index a8c1c110..52ee22ac 100644 --- a/README.rst +++ b/README.rst @@ -30,14 +30,14 @@ About ``heudiconv`` is a flexible DICOM converter for organizing brain imaging data into structured directory layouts. -- it allows flexible directory layouts and naming schemes through customizable heuristics implementations -- it only converts the necessary DICOMs, not everything in a directory -- you can keep links to DICOM files in the participant layout -- using `dcm2niix `_ under the hood, it's fast -- it can track the provenance of the conversion from DICOM to NIfTI in W3C PROV format -- it provides assistance in converting to `BIDS `_ -- it integrates with `DataLad `_ to place converted and original data under git/git-annex - version control, while automatically annotating files with sensitive information (e.g., non-defaced anatomicals, etc). +- It allows flexible directory layouts and naming schemes through customizable heuristics implementations. +- It only converts the necessary DICOMs and ignores everything else in a directory. +- You can keep links to DICOM files in the participant layout. +- Using `dcm2niix `_ under the hood, it's fast. +- It can track the provenance of the conversion from DICOM to NIfTI in W3C PROV format. +- It provides assistance in converting to `BIDS `_. +- It integrates with `DataLad `_ to place converted and original data under git/git-annex + version control while automatically annotating files with sensitive information (e.g., non-defaced anatomicals, etc). HOWTO 101 --------- From 7a0c8e487d84b887db0b24c754386e2660b63390 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 31 Jan 2023 21:19:51 -0500 Subject: [PATCH 063/176] RF: do not apt-get install or demand manual installation of dcm2niix in the docs --- .github/workflows/test.yml | 2 +- docs/installation.rst | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 72883a62..8598b31f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ jobs: # The ultimate one-liner setup for NeuroDebian repository bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) sudo apt-get update -qq - sudo apt-get install git-annex-standalone dcm2niix + sudo apt-get install git-annex-standalone - name: Install dependencies run: | diff --git a/docs/installation.rst b/docs/installation.rst index adced6a2..ccc4bb30 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -13,9 +13,6 @@ If installing through ``PyPI``, eg:: pip install heudiconv[all] -Manual installation of `dcm2niix `_ -is required. - On Debian-based systems we recommend using `NeuroDebian `_ which provides the `heudiconv package `_. @@ -56,4 +53,4 @@ It also contains converted from Docker singularity images for stock heudiconv im `_ section for a prototypical example of using `datalad-container `_ extension with this dataset, while fulfilling `YODA principles `_. **Note** that it should also work on -OSX with ``///repronim/containers`` automagically taking care about running those Singularity containers via Docker. \ No newline at end of file +OSX with ``///repronim/containers`` automagically taking care about running those Singularity containers via Docker. From 6c2cba81820191d85ba05da3a72ae3ecaea085fb Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 31 Jan 2023 21:21:06 -0500 Subject: [PATCH 064/176] Do not bother with separate installation of dcm2niix in neurodocker --- utils/gen-docker-image.sh | 1 - 1 file changed, 1 deletion(-) mode change 100644 => 100755 utils/gen-docker-image.sh diff --git a/utils/gen-docker-image.sh b/utils/gen-docker-image.sh old mode 100644 new mode 100755 index 70d6ea65..7b87c1f2 --- a/utils/gen-docker-image.sh +++ b/utils/gen-docker-image.sh @@ -8,7 +8,6 @@ VER=$(grep -Po '(?<=^__version__ = ).*' $thisd/../heudiconv/info.py | sed 's/"// image="kaczmarj/neurodocker:0.9.1" docker run --rm $image generate docker -b neurodebian:bullseye -p apt \ - --dcm2niix version=v1.0.20220720 method=source \ --install git gcc pigz liblzma-dev libc-dev git-annex-standalone netbase \ --copy . /src/heudiconv \ --miniconda version="py39_4.12.0" conda_install="python=3.9 traits>=4.6.0 scipy numpy nomkl pandas" \ From 95d9d20258f9fec120db6bce711c97b8fb9d2709 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 31 Jan 2023 21:21:15 -0500 Subject: [PATCH 065/176] [DATALAD RUNCMD] produce updated dockerfile === Do not change lines below === { "chain": [ "e179ee94e55f524620bea09c4d29fe56a7908fff" ], "cmd": "utils/gen-docker-image.sh", "exit": 0, "extra_inputs": [], "inputs": [], "outputs": [], "pwd": "." } ^^^ Do not change lines above ^^^ --- Dockerfile | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5ec60639..c65ba1ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,6 @@ # Generated by Neurodocker and Reproenv. FROM neurodebian:bullseye -ENV PATH="/opt/dcm2niix-v1.0.20220720/bin:$PATH" -RUN apt-get update -qq \ - && apt-get install -y -q --no-install-recommends \ - ca-certificates \ - cmake \ - g++ \ - gcc \ - git \ - make \ - pigz \ - zlib1g-dev \ - && rm -rf /var/lib/apt/lists/* \ - && git clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix \ - && cd /tmp/dcm2niix \ - && git fetch --tags \ - && git checkout v1.0.20220720 \ - && mkdir /tmp/dcm2niix/build \ - && cd /tmp/dcm2niix/build \ - && cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20220720 .. \ - && make -j1 \ - && make install \ - && rm -rf /tmp/dcm2niix RUN apt-get update -qq \ && apt-get install -y -q --no-install-recommends \ gcc \ @@ -89,18 +67,6 @@ RUN printf '{ \ "base_image": "neurodebian:bullseye" \ } \ }, \ - { \ - "name": "env", \ - "kwds": { \ - "PATH": "/opt/dcm2niix-v1.0.20220720/bin:$PATH" \ - } \ - }, \ - { \ - "name": "run", \ - "kwds": { \ - "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n ca-certificates \\\\\\n cmake \\\\\\n g++ \\\\\\n gcc \\\\\\n git \\\\\\n make \\\\\\n pigz \\\\\\n zlib1g-dev\\nrm -rf /var/lib/apt/lists/*\\ngit clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix\\ncd /tmp/dcm2niix\\ngit fetch --tags\\ngit checkout v1.0.20220720\\nmkdir /tmp/dcm2niix/build\\ncd /tmp/dcm2niix/build\\ncmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20220720 ..\\nmake -j1\\nmake install\\nrm -rf /tmp/dcm2niix" \ - } \ - }, \ { \ "name": "install", \ "kwds": { \ From 0c30781248e0c1f2d437e0dfd923a74d7e647411 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 1 Feb 2023 21:34:58 -0500 Subject: [PATCH 066/176] Setting git author and email in test environment --- heudiconv/tests/conftest.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 heudiconv/tests/conftest.py diff --git a/heudiconv/tests/conftest.py b/heudiconv/tests/conftest.py new file mode 100644 index 00000000..f4f47d16 --- /dev/null +++ b/heudiconv/tests/conftest.py @@ -0,0 +1,3 @@ +import os +os.environ["GIT_AUTHOR_EMAIL"] = "maxm@example.com" +os.environ["GIT_AUTHOR_NAME"] = "Max Mustermann" From 5bb6f0b8dfb70f5eaebb68b5873ef5f343e9ddb6 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 1 Feb 2023 22:12:08 -0500 Subject: [PATCH 067/176] Added distribution badges --- README.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.rst b/README.rst index 52ee22ac..3b355e9d 100644 --- a/README.rst +++ b/README.rst @@ -24,6 +24,18 @@ :target: https://doi.org/10.5281/zenodo.1012598 :alt: Zenodo (latest) +.. image :: https://repology.org/badge/version-for-repo/debian_unstable/heudiconv.svg?header=Debian%20Unstable + :target: https://repology.org/project/heudiconv/versions + :alt: Debian Unstable package + +.. image:: https://repology.org/badge/version-for-repo/gentoo_ovl_science/python:heudiconv.svg?header=Gentoo%20%28%3A%3Ascience%29 + :target: https://repology.org/project/python:heudiconv/versions + :alt: Gentoo (::science) + +.. image:: https://repology.org/badge/version-for-repo/pypi/python:heudiconv.svg + :target: https://repology.org/project/python:heudiconv/versions + :alt: PyPI package + About ----- From fceb910fa25cc97a391d4231f20f76775e7161c5 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 1 Feb 2023 21:53:36 -0500 Subject: [PATCH 068/176] committer name and email --- heudiconv/tests/conftest.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/heudiconv/tests/conftest.py b/heudiconv/tests/conftest.py index f4f47d16..ab83fdfa 100644 --- a/heudiconv/tests/conftest.py +++ b/heudiconv/tests/conftest.py @@ -1,3 +1,8 @@ import os -os.environ["GIT_AUTHOR_EMAIL"] = "maxm@example.com" -os.environ["GIT_AUTHOR_NAME"] = "Max Mustermann" +import pytest +@pytest.fixture(autouse=True, scope="session") +def git_env(): + os.environ["GIT_AUTHOR_EMAIL"] = "maxm@example.com" + os.environ["GIT_AUTHOR_NAME"] = "Max Mustermann" + os.environ["GIT_COMMITTER_EMAIL"] = "maxm@example.com" + os.environ["GIT_COMMITTER_NAME"] = "Max Mustermann" From 3d61b9be39e5c1c26ab73ffe6bbf28746220b291 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Thu, 2 Feb 2023 14:02:02 -0500 Subject: [PATCH 069/176] Typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3b355e9d..bc1bd63b 100644 --- a/README.rst +++ b/README.rst @@ -24,7 +24,7 @@ :target: https://doi.org/10.5281/zenodo.1012598 :alt: Zenodo (latest) -.. image :: https://repology.org/badge/version-for-repo/debian_unstable/heudiconv.svg?header=Debian%20Unstable +.. image:: https://repology.org/badge/version-for-repo/debian_unstable/heudiconv.svg?header=Debian%20Unstable :target: https://repology.org/project/heudiconv/versions :alt: Debian Unstable package From 4f32b422621f04b24abc563ed3c98d109cde8e43 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Thu, 2 Feb 2023 15:32:42 -0500 Subject: [PATCH 070/176] =?UTF-8?q?Removed=20reptetitions=20of=20the=20wor?= =?UTF-8?q?d=20=E2=80=9Cpackage=E2=80=9D=20all=20over=20the=20place?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index bc1bd63b..438108e3 100644 --- a/README.rst +++ b/README.rst @@ -26,15 +26,15 @@ .. image:: https://repology.org/badge/version-for-repo/debian_unstable/heudiconv.svg?header=Debian%20Unstable :target: https://repology.org/project/heudiconv/versions - :alt: Debian Unstable package + :alt: Debian Unstable .. image:: https://repology.org/badge/version-for-repo/gentoo_ovl_science/python:heudiconv.svg?header=Gentoo%20%28%3A%3Ascience%29 :target: https://repology.org/project/python:heudiconv/versions :alt: Gentoo (::science) -.. image:: https://repology.org/badge/version-for-repo/pypi/python:heudiconv.svg +.. image:: https://repology.org/badge/version-for-repo/pypi/python:heudiconv.svg?header=PyPI :target: https://repology.org/project/python:heudiconv/versions - :alt: PyPI package + :alt: PyPI About ----- From dea6dac720d97a3a75440640e7ecb5a7782653be Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 8 Feb 2023 11:39:19 -0500 Subject: [PATCH 071/176] Add testing against 3.11 --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8598b31f..4446f84c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,6 +18,7 @@ jobs: - '3.8' - '3.9' - '3.10' + - '3.11' steps: - name: Check out repository uses: actions/checkout@v3 From 67a654c64a48dacda97b89fc124629c917599eed Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 8 Feb 2023 11:40:22 -0500 Subject: [PATCH 072/176] Do state that we support 3.10 and also 3.11 --- heudiconv/info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/heudiconv/info.py b/heudiconv/info.py index e60769ce..0ae24df0 100644 --- a/heudiconv/info.py +++ b/heudiconv/info.py @@ -14,8 +14,8 @@ 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', - # needs fixing - # 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Topic :: Scientific/Engineering' ] From 1b1bf911a7c4bddf6d6ff7363038047e37d06595 Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 15 Feb 2023 13:06:51 -0500 Subject: [PATCH 073/176] add install link to README --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 438108e3..fbdf1831 100644 --- a/README.rst +++ b/README.rst @@ -51,6 +51,12 @@ into structured directory layouts. - It integrates with `DataLad `_ to place converted and original data under git/git-annex version control while automatically annotating files with sensitive information (e.g., non-defaced anatomicals, etc). +Installation +------------ + +See our `installation page `_ +on heudiconv.readthedocs.io + HOWTO 101 --------- From 8d0b69ff5aca90f7e21a3108fab5f90db72037b6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 15 Feb 2023 13:12:43 -0500 Subject: [PATCH 074/176] Remove "v" prefix from the tag for docker hub And also tag as unstable as that is the one we promoted a little in CHANGELOG in the times of 0.5 releases. We remove "v" since not semver and IMHO not needed, and that is how we set it up on docker hub originally so prior releases are all without "v" and only the ones we uploaded from actions were with "v". --- .github/workflows/docker.yml | 6 +++++- .github/workflows/release.yml | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8c66c2ef..1049b43b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -20,7 +20,11 @@ jobs: working-directory: utils - name: Build Docker image - run: docker build -t nipy/heudiconv:master . + run: | + docker build \ + -t nipy/heudiconv:master \ + -t nipy/heudiconv:unstable \ + . - name: Push Docker image run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 05957516..ae73a885 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,8 +67,9 @@ jobs: run: | docker build \ -t nipy/heudiconv:master \ + -t nipy/heudiconv:unstable \ -t nipy/heudiconv:latest \ - -t nipy/heudiconv:"$(git describe)" \ + -t nipy/heudiconv:"$(git describe | sed -e 's,^v,,g')" \ . - name: Push Docker images From 0efe7b7dd9eff14be2bb83399eca3d310e33ad49 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 15 Feb 2023 13:17:13 -0500 Subject: [PATCH 075/176] Avoid duplication of building/pushing docker images for non-releases --- .github/workflows/docker.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1049b43b..ab6b21f8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,15 +21,20 @@ jobs: - name: Build Docker image run: | - docker build \ - -t nipy/heudiconv:master \ - -t nipy/heudiconv:unstable \ - . + # build only if not release tag, i.e. has some "-" in describe + # so we do not duplicate work with release workflow. + git describe --match 'v[0-9]*' | grep -q -e - && \ + docker build \ + -t nipy/heudiconv:master \ + -t nipy/heudiconv:unstable \ + . - name: Push Docker image run: | - docker login -u "$DOCKER_LOGIN" --password-stdin <<<"$DOCKER_TOKEN" - docker push nipy/heudiconv:master + git describe --match 'v[0-9]*' | grep -q -e - && ( + docker login -u "$DOCKER_LOGIN" --password-stdin <<<"$DOCKER_TOKEN" + docker push nipy/heudiconv:master + ) env: DOCKER_LOGIN: ${{ secrets.DOCKER_LOGIN }} DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} From bdcafa61a79009144eaac0681f75eb13bf03c36c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 15 Feb 2023 13:18:54 -0500 Subject: [PATCH 076/176] Also push unstable tag --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ab6b21f8..83e75a77 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -34,6 +34,7 @@ jobs: git describe --match 'v[0-9]*' | grep -q -e - && ( docker login -u "$DOCKER_LOGIN" --password-stdin <<<"$DOCKER_TOKEN" docker push nipy/heudiconv:master + docker push nipy/heudiconv:unstable ) env: DOCKER_LOGIN: ${{ secrets.DOCKER_LOGIN }} From f7fc632ccd87f57b913057c548c34f83dfc96f9f Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 15 Feb 2023 13:29:02 -0500 Subject: [PATCH 077/176] Update README.rst Co-authored-by: Yaroslav Halchenko --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index fbdf1831..6533b676 100644 --- a/README.rst +++ b/README.rst @@ -55,7 +55,7 @@ Installation ------------ See our `installation page `_ -on heudiconv.readthedocs.io +on heudiconv.readthedocs.io . HOWTO 101 --------- From 90175c3f2c758994df444fa15501bbeecb04559a Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 15 Feb 2023 10:04:28 -0800 Subject: [PATCH 078/176] Reword number of intended ideas in README.rst --- README.rst | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/README.rst b/README.rst index 438108e3..bcca258d 100644 --- a/README.rst +++ b/README.rst @@ -56,26 +56,28 @@ HOWTO 101 In a nutshell -- ``heudiconv`` operates using a heuristic which, given metadata from DICOMs, would decide how to name resultant (from conversion using `dcm2niix`_) files. Heuristic `convertall `_ could also be used to actually have no real -heuristic and simply establish your own conversion mapping by editing produced mapping files. -In most use-cases of retrospecive study data conversion you would need to create your custom heuristic following +.com/nipy/heudiconv/blob/master/heudiconv/heuristics/convertall.py>`_ could actually be used with no real +heuristic and by simply establish your own conversion mapping through editing produced mapping files. +In most use-cases of retrospective study data conversion, you would need to create your custom heuristic following `existing heuristics as examples `_ and/or referring to `"Heuristic" section `_ in the documentation. **Note** that `ReproIn heuristic `_ is -generic and powerful enough to be adopted virtually for *any* study: For prospective studies you would just need -to name your sequences following the `ReproIn convention `_ and for -retrospective conversions, you often would be able to create a new very versatile heuristic by simply providing +generic and powerful enough to be adopted virtually for *any* study: For prospective studies, you would just need +to name your sequences following the `ReproIn convention `_, and for +retrospective conversions, you often would be able to create a new versatile heuristic by simply providing remappings into ReproIn as shown in `this issue (documentation is coming) `_. -Having decided on heuristic you could use command line:: +Having decided on a heuristic, you could use the command line:: heudiconv -f HEURISTIC-FILE-OR-NAME -o OUTPUT-PATH --files INPUT-PATHs with various additional options (see ``heudiconv --help`` or -`"Usage" in documentation `__) to tune its behavior, to -convert your data. See e.g. `ReproIn conversion invocation examples `_. +`"Usage" in documentation `__) to tune its behavior to +convert your data. + +For detailed examples and guides, please check out `ReproIn conversion invocation examples `_ +and the `user tutorials `_ in the documentation. -Please also see `user tutorials `_ in documentation. How to cite ----------- @@ -93,9 +95,9 @@ Please file issues and suggest changes via Pull Requests. HeuDiConv requires installation of `dcm2niix`_ and optionally `DataLad`_. -For development you will need a non-shallow clone (so there is a -recent released tag) of the aforementioned repository. You can then -install all necessary development requirements using ``pip install -r +For development, you will need a non-shallow clone (so there is a +recent released tag) of the aforementioned repository. Once you have cloned the repository, +you can then install all the necessary development requirements using ``pip install -r dev-requirements.txt``. Testing is done using `pytest `_. Releases are packaged using Intuit auto. Workflow for releases and preparation of Docker images is in From 7e522107063a486e6a586706de4c19cd06f917a6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 15 Feb 2023 16:48:27 -0500 Subject: [PATCH 079/176] Clarify the infotodict function --- docs/heuristics.rst | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/heuristics.rst b/docs/heuristics.rst index ec9f0e76..ae460237 100644 --- a/docs/heuristics.rst +++ b/docs/heuristics.rst @@ -21,21 +21,28 @@ Components The only required function for a heuristic, `infotodict` is used to both define the conversion outputs and specify the criteria for scan to output association. -Conversion outputs are defined as keys, a `tuple` consisting of a template path -used for the basis of outputs, as well as a `tuple` of output types. Valid types -include `nii`, `nii.gz`, and `dicom`. +Conversion outputs are defined as keys, a `tuple` consisting of three elements: -.. note:: An example conversion key +- a template path used for the basis of outputs +- `tuple` of output types. Valid types include `nii`, `nii.gz`, and `dicom`. +- `None` - a historical artifact (corresponds to some notion of + ``annotation_class`` no living human is aware about) - ``('sub-{subject}/func/sub-{subject}_task-test_run-{item}_bold', ('nii.gz', 'dicom'))`` +.. note:: An example conversion key + ``('sub-{subject}/func/sub-{subject}_task-test_run-{item}_bold', ('nii.gz', 'dicom'), None)`` The ``seqinfos`` parameter is a list of namedtuples which serves as a grouped and stacked record of the DICOMs passed in. Each item in `seqinfo` contains DICOM metadata that can be used to isolate the series, and assign it to a conversion key. -A dictionary of {``conversion key``: ``seqinfo``} is returned. +A function ``create_key`` is commonly defined by heuristics (internally) +to assist in creating the key, and to be used inside ``infotodict``. + +A dictionary of {``conversion key``: ``series_id``} is returned, where +``series_id`` is the 3rd (indexes as ``[2]`` or accessed as ``.series_id`` from +``seqinfo``). --------------------------------- ``create_key(template, outtype)`` From cda2a7e5b5fd6bab5e267976e5b65999772524a6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 15 Feb 2023 16:58:51 -0500 Subject: [PATCH 080/176] Use http instead of https to avoid certificate issue --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index ccc4bb30..8ca96838 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -30,7 +30,7 @@ within container, i.e. like:: $ docker run --user=$(id -u):$(id -g) -e "UID=$(id -u)" -e "GID=$(id -g)" --rm -t -v $PWD:$PWD nipy/heudiconv:latest [OPTIONS TO FOLLOW] -`ReproIn heuristic project `_ provides its own Docker images from +`ReproIn heuristic project `_ provides its own Docker images from Docker Hub `repronim/reproin` which bundle its `reproin` helper. Singularity From e0ff32dedb40bb6b644878afafa9ac8761362e05 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Fri, 17 Feb 2023 06:13:36 -0800 Subject: [PATCH 081/176] Reword and correct punctuation on usage.rst (#644) * Reword and correct punctuation on usage.rst --------- Co-authored-by: Yaroslav Halchenko --- docs/usage.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index b7a7175d..8befdc99 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -32,8 +32,9 @@ http://neurostars.org/tags/heudiconv/ Batch jobs ========== -``heudiconv`` can natively handle multi-subject, multi-session conversions, -although it will process these linearly. To speed this up, multiple ``heudiconv`` +``heudiconv`` can natively handle multi-subject, multi-session conversions +although it will do these conversions in a linear manner, i.e. one subject and one session at a time. +To speed up these conversions, multiple ``heudiconv`` processes can be spawned concurrently, each converting a different subject and/or session. From cda541a84edf4dd886685bf2966493126efc8f98 Mon Sep 17 00:00:00 2001 From: Keith Callenberg Date: Fri, 14 Jan 2022 10:56:25 -0500 Subject: [PATCH 082/176] strip non-alphanumeric from session ids too --- heudiconv/convert.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index 54b03513..e1ff5362 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -104,6 +104,9 @@ def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid, "BIDS requires alphanumeric subject ID. Got an empty value") if not sid.isalnum(): # alphanumeric only sid, old_sid = convert_sid_bids(sid) + + if ses and not ses.isalnum(): # alphanumeric only + ses, old_ses = convert_sid_bids(ses) if not anon_sid: anon_sid = sid From f15354e119b7d28a9537d9af476a45d36d66d7c9 Mon Sep 17 00:00:00 2001 From: Keith Callenberg Date: Thu, 10 Feb 2022 16:18:06 -0500 Subject: [PATCH 083/176] refactor convert_sid_bids to sanitize_label --- heudiconv/bids.py | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/heudiconv/bids.py b/heudiconv/bids.py index 12bad0fb..ddf31035 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -500,9 +500,8 @@ def get_formatted_scans_key_row(dcm_fn): row = ['n/a' if not str(e) else e for e in row] return row - def convert_sid_bids(subject_id): - """Strips any non-BIDS compliant characters within subject_id + """Shim for stripping any non-BIDS compliant characters within subject_id Parameters ---------- @@ -515,14 +514,10 @@ def convert_sid_bids(subject_id): subject_id : string Original subject ID """ - cleaner = lambda y: ''.join([x for x in y if x.isalnum()]) - sid = cleaner(subject_id) - if not sid: - raise ValueError( - "Subject ID became empty after cleanup. Please provide manually " - "a suitable alphanumeric subject ID") - lgr.warning('{0} contained nonalphanumeric character(s), subject ' - 'ID was cleaned to be {1}'.format(subject_id, sid)) + sid, subject_id = sanitize_label(subject_id) + lgr.warning('Deprecation warning: convert_sid_bids() is deprecated, ' + 'please use sanitize_label() instead.') + return sid, subject_id @@ -1028,3 +1023,29 @@ def suffix(self): @property def extension(self): return self._extension + + +def sanitize_label(label): + """Strips any non-BIDS compliant characters within label + + Parameters + ---------- + label : string + + Returns + ------- + clean_label : string + New, sanitized label + label : string + Original label + """ + cleaner = lambda y: ''.join([x for x in y if x.isalnum()]) + clean_label = cleaner(label) + if not clean_label: + raise ValueError( + "Label became empty after cleanup. Please provide manually " + "a suitable alphanumeric label.") + if clean_label != label: + lgr.warning('{0} contained nonalphanumeric character(s), label ' + 'was cleaned to be {1}'.format(label, clean_label)) + return clean_label, label From 4be7a48ad99416b892da8e693485486e09e4f4ff Mon Sep 17 00:00:00 2001 From: Keith Callenberg Date: Thu, 10 Feb 2022 16:20:12 -0500 Subject: [PATCH 084/176] update prep_conversion() to use sanitize_label --- heudiconv/convert.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index e1ff5362..f021c8b5 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -26,7 +26,7 @@ file_md5sum ) from .bids import ( - convert_sid_bids, + sanitize_label, populate_bids_templates, populate_intended_for, save_scans_key, @@ -103,10 +103,10 @@ def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid, raise ValueError( "BIDS requires alphanumeric subject ID. Got an empty value") if not sid.isalnum(): # alphanumeric only - sid, old_sid = convert_sid_bids(sid) + sid, old_sid = sanitize_label(sid) if ses and not ses.isalnum(): # alphanumeric only - ses, old_ses = convert_sid_bids(ses) + ses, old_ses = sanitize_label(ses) if not anon_sid: anon_sid = sid From 27372c81b8f01b604e7606bb87cbe2918fdcdbdc Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 17 Feb 2023 10:51:45 -0500 Subject: [PATCH 085/176] Formatting and issuing warning tuneup --- heudiconv/bids.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/heudiconv/bids.py b/heudiconv/bids.py index ddf31035..2df2dcca 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -14,6 +14,7 @@ from random import sample from glob import glob import errno +import warnings from .external.pydicom import dcm @@ -500,6 +501,7 @@ def get_formatted_scans_key_row(dcm_fn): row = ['n/a' if not str(e) else e for e in row] return row + def convert_sid_bids(subject_id): """Shim for stripping any non-BIDS compliant characters within subject_id @@ -514,11 +516,10 @@ def convert_sid_bids(subject_id): subject_id : string Original subject ID """ - sid, subject_id = sanitize_label(subject_id) - lgr.warning('Deprecation warning: convert_sid_bids() is deprecated, ' - 'please use sanitize_label() instead.') - - return sid, subject_id + warnings.warn('convert_sid_bids() is deprecated, ' + 'please use sanitize_label() instead.', + DeprecationWarning) + return sanitize_label(subject_id) def get_shim_setting(json_file): @@ -1046,6 +1047,6 @@ def sanitize_label(label): "Label became empty after cleanup. Please provide manually " "a suitable alphanumeric label.") if clean_label != label: - lgr.warning('{0} contained nonalphanumeric character(s), label ' - 'was cleaned to be {1}'.format(label, clean_label)) + lgr.warning('%r label contained non-alphanumeric character(s), it ' + 'was cleaned to be %r', label, clean_label) return clean_label, label From b366e9c6df521eb58925f7215e885329f132fc6f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 17 Feb 2023 10:59:12 -0500 Subject: [PATCH 086/176] RF: simplify - do not bother returning orig value, no internal function needed orig value was useful only for debugging, but then code could be instrumented if needed. removing helper function actually makes code more readable. do not duplicate "check" (isalnum) - always call sanitize --- heudiconv/bids.py | 7 ++----- heudiconv/convert.py | 8 +++----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/heudiconv/bids.py b/heudiconv/bids.py index 2df2dcca..3c14358d 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -1037,11 +1037,8 @@ def sanitize_label(label): ------- clean_label : string New, sanitized label - label : string - Original label """ - cleaner = lambda y: ''.join([x for x in y if x.isalnum()]) - clean_label = cleaner(label) + clean_label = ''.join(x for x in label if x.isalnum()) if not clean_label: raise ValueError( "Label became empty after cleanup. Please provide manually " @@ -1049,4 +1046,4 @@ def sanitize_label(label): if clean_label != label: lgr.warning('%r label contained non-alphanumeric character(s), it ' 'was cleaned to be %r', label, clean_label) - return clean_label, label + return clean_label diff --git a/heudiconv/convert.py b/heudiconv/convert.py index f021c8b5..0f3cdbf6 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -102,11 +102,9 @@ def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid, if not sid: raise ValueError( "BIDS requires alphanumeric subject ID. Got an empty value") - if not sid.isalnum(): # alphanumeric only - sid, old_sid = sanitize_label(sid) - - if ses and not ses.isalnum(): # alphanumeric only - ses, old_ses = sanitize_label(ses) + sid = sanitize_label(sid) + if ses: + ses = sanitize_label(ses) if not anon_sid: anon_sid = sid From 3d668a6e34c2a9ea52d869b231703cfef00b1e12 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 17 Feb 2023 16:08:03 -0500 Subject: [PATCH 087/176] TST: add rudimentary test for sanitize_label --- heudiconv/tests/test_bids.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/heudiconv/tests/test_bids.py b/heudiconv/tests/test_bids.py index 83fdf3d5..745397a7 100644 --- a/heudiconv/tests/test_bids.py +++ b/heudiconv/tests/test_bids.py @@ -42,6 +42,7 @@ AllowedCriteriaForFmapAssignment, KeyInfoForForce, BIDSFile, + sanitize_label, ) from heudiconv.cli.run import main as runner @@ -1155,3 +1156,8 @@ def test_ME_mag_phase_conversion(tmpdir, subID='MEGRE', heuristic='bids_ME.py'): % (subID, subID, e, part, ext) ) + +def test_sanitize_label(): + assert sanitize_label('az XZ-@09') == 'azXZ09' + with pytest.raises(ValueError): + sanitize_label(' @ ') \ No newline at end of file From 94911c7d1605c076f6f62c3943dc8a5ad74ba8c6 Mon Sep 17 00:00:00 2001 From: auto Date: Tue, 21 Feb 2023 19:34:36 +0000 Subject: [PATCH 088/176] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03a3d4d8..f9fa32ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,56 @@ +# v0.12.0 (Tue Feb 21 2023) + +#### 🚀 Enhancement + +- strip non-alphanumeric from session ids too [#647](https://github.com/nipy/heudiconv/pull/647) ([@keithcallenberg](https://github.com/keithcallenberg) [@yarikoptic](https://github.com/yarikoptic)) + +#### 🐛 Bug Fix + +- Docker images: tag also as "unstable", strip "v" prefix, and avoid building in non-release workflow for releases. [#642](https://github.com/nipy/heudiconv/pull/642) ([@yarikoptic](https://github.com/yarikoptic)) +- add install link to README [#640](https://github.com/nipy/heudiconv/pull/640) ([@asmacdo](https://github.com/asmacdo)) +- Setting git author and email in test environment [#631](https://github.com/nipy/heudiconv/pull/631) ([@TheChymera](https://github.com/TheChymera)) +- Duecredit dcm2niix [#622](https://github.com/nipy/heudiconv/pull/622) ([@yarikoptic](https://github.com/yarikoptic)) +- Do not issue warning if cannot parse _task entity [#621](https://github.com/nipy/heudiconv/pull/621) ([@yarikoptic](https://github.com/yarikoptic)) +- Provide codespell config and workflow [#619](https://github.com/nipy/heudiconv/pull/619) ([@yarikoptic](https://github.com/yarikoptic)) +- BF: Use .get in group_dicoms_into_seqinfos to not puke if SeriesDescription is missing [#622](https://github.com/nipy/heudiconv/pull/622) ([@yarikoptic](https://github.com/yarikoptic)) +- DOC: do provide short version for sphinx [#609](https://github.com/nipy/heudiconv/pull/609) ([@yarikoptic](https://github.com/yarikoptic)) + +#### ⚠️ Pushed to `master` + +- DOC: add clarification on where docs/requirements.txt should be "installed" from ([@yarikoptic](https://github.com/yarikoptic)) +- fix minor typo ([@yarikoptic](https://github.com/yarikoptic)) +- DOC: fixed the comment. Original was copy/pasted from DataLad ([@yarikoptic](https://github.com/yarikoptic)) + +#### 🏠 Internal + +- dcm2niix explicitly noted as a (PyPI) dependency and removed from being installed via apt-get etc [#628](https://github.com/nipy/heudiconv/pull/628) ([@TheChymera](https://github.com/TheChymera) [@yarikoptic](https://github.com/yarikoptic)) + +#### 📝 Documentation + +- Reword number of intended ideas in README.rst [#639](https://github.com/nipy/heudiconv/pull/639) ([@candleindark](https://github.com/candleindark)) +- Add a bash anon-cmd to be used to incrementally anonymize sids [#615](https://github.com/nipy/heudiconv/pull/615) ([@yarikoptic](https://github.com/yarikoptic)) +- Reword and correct punctuation on usage.rst [#644](https://github.com/nipy/heudiconv/pull/644) ([@candleindark](https://github.com/candleindark) [@yarikoptic](https://github.com/yarikoptic)) +- Clarify the infotodict function [#645](https://github.com/nipy/heudiconv/pull/645) ([@yarikoptic](https://github.com/yarikoptic)) +- Added distribution badges [#632](https://github.com/nipy/heudiconv/pull/632) ([@TheChymera](https://github.com/TheChymera)) +- Capitalize sentences and end sentences with period [#629](https://github.com/nipy/heudiconv/pull/629) ([@candleindark](https://github.com/candleindark)) +- Tune up .mailmap to harmonize Pablo, Dae and Mathias [#629](https://github.com/nipy/heudiconv/pull/629) ([@yarikoptic](https://github.com/yarikoptic)) +- Add HOWTO 101 section, with references to ReproIn to README.rst [#623](https://github.com/nipy/heudiconv/pull/623) ([@yarikoptic](https://github.com/yarikoptic)) +- minor fix -- Fix use of code:: directive [#623](https://github.com/nipy/heudiconv/pull/623) ([@yarikoptic](https://github.com/yarikoptic)) + +#### 🧪 Tests + +- Add 3.11 to be tested etc [#635](https://github.com/nipy/heudiconv/pull/635) ([@yarikoptic](https://github.com/yarikoptic)) + +#### Authors: 5 + +- Austin Macdonald ([@asmacdo](https://github.com/asmacdo)) +- Horea Christian ([@TheChymera](https://github.com/TheChymera)) +- Isaac To ([@candleindark](https://github.com/candleindark)) +- Keith Callenberg ([@keithcallenberg](https://github.com/keithcallenberg)) +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + # v0.11.6 (Thu Nov 03 2022) #### 🏠 Internal From 76c26af7d3c0f177270414f85344645a4985c7b5 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 15 Feb 2023 13:16:40 -0500 Subject: [PATCH 089/176] Added dependency section --- README.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.rst b/README.rst index 5352991d..8cd1bd2b 100644 --- a/README.rst +++ b/README.rst @@ -108,3 +108,22 @@ dev-requirements.txt``. Testing is done using `pytest `_. Releases are packaged using Intuit auto. Workflow for releases and preparation of Docker images is in ``.github/workflows/release.yml``. + + +Dependencies +------------ + +If an up-to-date HeuDiConv version is installed via a package manager (e.g. Debian's +`apt-get`, Gentoo's `emerge`, or PIP), the following should be handled automatically. + +Mandatory dependencies: +* [dcm2niix](https://github.com/rordenlab/dcm2niix) +* [dcmstack](https://dcmstack.readthedocs.org/en/latest/) >=0.8 +* [etelemetry](https://github.com/sensein/etelemetry-client) +* [filelock](https://github.com/tox-dev/py-filelock/) >=3.0.12 +* [nibabel](https://nipy.org/nibabel/) +* [nipype](http://nipy.sourceforge.net/nipype/) >=1.2.3 +* [pydicom](http://www.pydicom.org/) + +Optional dependencies: +* [datalad](https://github.com/datalad/datalad) From 5b62d63337916c501d2f9be5f8e3b2ab62c419fe Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 17 Feb 2023 09:15:48 -0500 Subject: [PATCH 090/176] minor rewording --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 8ca96838..0f43b3a7 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -47,7 +47,7 @@ Singularity YODA style using ///repronim/containers ReproNim project provides `///repronim/containers `_ (git clone present also on `GitHub `__) `DataLad `_ dataset with Singularity containers for many popular neuroimaging tools, e.g. all BIDS-Apps. -It also contains converted from Docker singularity images for stock heudiconv images (as `nipy-heudiconv +It contains converted from Docker singularity images for stock heudiconv images (as `nipy-heudiconv `__) and reproin images (as `repronim-reproin `__). Please see `"A typical workflow" `_ section for a prototypical example of using From be0e4641ce65ddfe99c33329a2508979321a2647 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 15 Feb 2023 14:13:48 -0500 Subject: [PATCH 091/176] Replaced contributing text with the upcoming contribution document --- README.rst | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 8cd1bd2b..705a88dd 100644 --- a/README.rst +++ b/README.rst @@ -96,18 +96,10 @@ all relevant citations via `DueCredit `_. How to contribute ----------------- -HeuDiConv sources are managed with Git on `GitHub `_. -Please file issues and suggest changes via Pull Requests. - -HeuDiConv requires installation of `dcm2niix`_ and optionally `DataLad`_. - -For development, you will need a non-shallow clone (so there is a -recent released tag) of the aforementioned repository. Once you have cloned the repository, -you can then install all the necessary development requirements using ``pip install -r -dev-requirements.txt``. Testing is done using `pytest -`_. Releases are packaged using Intuit -auto. Workflow for releases and preparation of Docker images is in -``.github/workflows/release.yml``. +For a detailed into, see our contributing guide. + +Our releases are packaged using Intuit auto, with the corresponding workflow including +Docker image preparation being found in ``.github/workflows/release.yml``. Dependencies From 483ecb9d66c01fbe3deb0a1e12133a96b04aecc6 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 15 Feb 2023 14:14:46 -0500 Subject: [PATCH 092/176] Removed manual dependency listing --- README.rst | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/README.rst b/README.rst index 705a88dd..620eac71 100644 --- a/README.rst +++ b/README.rst @@ -100,22 +100,3 @@ For a detailed into, see our contributing guide. Our releases are packaged using Intuit auto, with the corresponding workflow including Docker image preparation being found in ``.github/workflows/release.yml``. - - -Dependencies ------------- - -If an up-to-date HeuDiConv version is installed via a package manager (e.g. Debian's -`apt-get`, Gentoo's `emerge`, or PIP), the following should be handled automatically. - -Mandatory dependencies: -* [dcm2niix](https://github.com/rordenlab/dcm2niix) -* [dcmstack](https://dcmstack.readthedocs.org/en/latest/) >=0.8 -* [etelemetry](https://github.com/sensein/etelemetry-client) -* [filelock](https://github.com/tox-dev/py-filelock/) >=3.0.12 -* [nibabel](https://nipy.org/nibabel/) -* [nipype](http://nipy.sourceforge.net/nipype/) >=1.2.3 -* [pydicom](http://www.pydicom.org/) - -Optional dependencies: -* [datalad](https://github.com/datalad/datalad) From 0f7656d1e827e167212860264d88b53f9efaff27 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 15 Feb 2023 14:14:59 -0500 Subject: [PATCH 093/176] Dedicated contributing document --- CONTRIBUTING.rst | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 CONTRIBUTING.rst diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 00000000..628acb5d --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,93 @@ +========================= +Contributing to HeuDiConv +========================= + +Files organization +------------------ + +- [heudiconv/](./heudiconv) is the main Python module where major development is happening, + with major submodules being: + - `cli/` - wrappers and argument parsers bringing the HeuDiConv functionality to the command + line. + - `external/` - general compatibility layers for external functions HeuDiConv depends on. + - `heuristics/` - heuristic evaluators for workflows, pull requests here are particularly + welcome. +- [docs/](./docs) - documentation directory. +- [utils/](./utils) - helper utilities used during development, testing, and distribution of + HeuDiConv. + +How to contribute +----------------- + +The preferred way to contribute to the HeuDiConv code base is +to fork the `main repository `_ on GitHub. + +If you are unsure what that means, here is a set-up workflow you may wish to follow: + +0. Fork the `project repository `_ on GitHub, by clicking + on the “Fork” button near the top of the page — this will create a copy of the repository + writeable by your GitHub user. + +1. Set up a clone of the repository on your local machine and connect it to both the “official” + and your copy of the repository on GitHub: +.. code-block:: sh + git clone git://github.com/nipy/heudiconv + cd heudiconv + git remote rename origin official + git remote add origin git://github.com/YOUR_GITHUB_USERNAME/heudiconv + +2. When you wish to start a new contribution, create a new branch: +.. code-block:: sh + git checkout -b topic_of_your_contribution + +3. When you are done making the changes you wish to contribute, record them in Git: +.. code-block:: sh + git add the/paths/to/files/you/modified can/be/more/than/one + git commit + +3. Push the changes to your copy of the code on GitHub, following which Git will + provide you with a link which you can click to initiate a pull request +.. code-block:: sh + git push -u origin topic_of_your_contribution + + +(If any of the above seems overwhelming, you can look up the `Git documentation +`_ on the web.) + + +Development environment +----------------------- + +We support Python 3 only (>= 3.7). + +Dependencies which you will need are `listed in the repository `_. +Note that you will likely have these will already be available on your system if you used a +package manger (e.g. Debian's ``apt-get``, Gentoo's ``emerge``, or simply PIP) to install the +software. + +Development work might require live access to the copy of HeuDiConv which is being developed. +If a system-wide release of HeuDiConv is already installed, or likely to be, it is best to keep +development work sandboxed inside a dedicated virtual environment. +This is best accomplished via: + +.. code-block:: sh + cd /path/to/your/clone/of/heudiconv + mkdir -p venvs/dev + python -m venv venvs/dev + source venvs/dev/bin/activate + + +Additional Hints +---------------- + +It is recommended to check that your contribution complies with the following +rules before submitting a pull request: + +* All public functions (i.e. functions whose name does not start with an underscore) should have + informative docstrings with sample usage presented as doctests when appropriate. +.. code-block:: sh + cd /path/to/your/clone/of/heudiconv + pytest -vvs . + +* All other tests pass: +* New code should be accompanied by new tests. From d7a902a78ec395be510d19886aaf59a9d42cd6aa Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 15 Feb 2023 14:18:15 -0500 Subject: [PATCH 094/176] reST syntax --- CONTRIBUTING.rst | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 628acb5d..afcd927a 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -5,15 +5,17 @@ Contributing to HeuDiConv Files organization ------------------ -- [heudiconv/](./heudiconv) is the main Python module where major development is happening, - with major submodules being: - - `cli/` - wrappers and argument parsers bringing the HeuDiConv functionality to the command +* `heudiconv/ <./heudiconv>`_ is the main Python module where major development is happening, with + major submodules being: + + * ``cli/`` - wrappers and argument parsers bringing the HeuDiConv functionality to the command line. - - `external/` - general compatibility layers for external functions HeuDiConv depends on. - - `heuristics/` - heuristic evaluators for workflows, pull requests here are particularly + * ``external/`` - general compatibility layers for external functions HeuDiConv depends on. + * ``heuristics/`` - heuristic evaluators for workflows, pull requests here are particularly welcome. -- [docs/](./docs) - documentation directory. -- [utils/](./utils) - helper utilities used during development, testing, and distribution of + +* `docs/ <./docs>`_ - documentation directory. +* `utils/ <./utils>`_ - helper utilities used during development, testing, and distribution of HeuDiConv. How to contribute @@ -27,28 +29,31 @@ If you are unsure what that means, here is a set-up workflow you may wish to fol 0. Fork the `project repository `_ on GitHub, by clicking on the “Fork” button near the top of the page — this will create a copy of the repository writeable by your GitHub user. - 1. Set up a clone of the repository on your local machine and connect it to both the “official” - and your copy of the repository on GitHub: -.. code-block:: sh - git clone git://github.com/nipy/heudiconv - cd heudiconv - git remote rename origin official - git remote add origin git://github.com/YOUR_GITHUB_USERNAME/heudiconv + and your copy of the repository on GitHub + + .. code-block:: sh + git clone git://github.com/nipy/heudiconv + cd heudiconv + git remote rename origin official + git remote add origin git://github.com/YOUR_GITHUB_USERNAME/heudiconv 2. When you wish to start a new contribution, create a new branch: + .. code-block:: sh - git checkout -b topic_of_your_contribution + git checkout -b topic_of_your_contribution 3. When you are done making the changes you wish to contribute, record them in Git: + .. code-block:: sh - git add the/paths/to/files/you/modified can/be/more/than/one - git commit + git add the/paths/to/files/you/modified can/be/more/than/one + git commit 3. Push the changes to your copy of the code on GitHub, following which Git will provide you with a link which you can click to initiate a pull request + .. code-block:: sh - git push -u origin topic_of_your_contribution + git push -u origin topic_of_your_contribution (If any of the above seems overwhelming, you can look up the `Git documentation From 06529e5db2dd931cdb03ca977f97b1c5d09f409b Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 15 Feb 2023 14:36:19 -0500 Subject: [PATCH 095/176] Linking to contributing guide. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 620eac71..1d4c1cad 100644 --- a/README.rst +++ b/README.rst @@ -96,7 +96,7 @@ all relevant citations via `DueCredit `_. How to contribute ----------------- -For a detailed into, see our contributing guide. +For a detailed into, see our `contributing guide `_. Our releases are packaged using Intuit auto, with the corresponding workflow including Docker image preparation being found in ``.github/workflows/release.yml``. From b96445ec34c371446944327f6ab1d76fcc48c4a7 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 15 Feb 2023 14:48:10 -0500 Subject: [PATCH 096/176] rest fix --- CONTRIBUTING.rst | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index afcd927a..840a6172 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -8,10 +8,10 @@ Files organization * `heudiconv/ <./heudiconv>`_ is the main Python module where major development is happening, with major submodules being: - * ``cli/`` - wrappers and argument parsers bringing the HeuDiConv functionality to the command + - ``cli/`` - wrappers and argument parsers bringing the HeuDiConv functionality to the command line. - * ``external/`` - general compatibility layers for external functions HeuDiConv depends on. - * ``heuristics/`` - heuristic evaluators for workflows, pull requests here are particularly + - ``external/`` - general compatibility layers for external functions HeuDiConv depends on. + - ``heuristics/`` - heuristic evaluators for workflows, pull requests here are particularly welcome. * `docs/ <./docs>`_ - documentation directory. @@ -30,30 +30,26 @@ If you are unsure what that means, here is a set-up workflow you may wish to fol on the “Fork” button near the top of the page — this will create a copy of the repository writeable by your GitHub user. 1. Set up a clone of the repository on your local machine and connect it to both the “official” - and your copy of the repository on GitHub + and your copy of the repository on GitHub:: - .. code-block:: sh - git clone git://github.com/nipy/heudiconv - cd heudiconv - git remote rename origin official - git remote add origin git://github.com/YOUR_GITHUB_USERNAME/heudiconv + git clone git://github.com/nipy/heudiconv + cd heudiconv + git remote rename origin official + git remote add origin git://github.com/YOUR_GITHUB_USERNAME/heudiconv -2. When you wish to start a new contribution, create a new branch: +2. When you wish to start a new contribution, create a new branch:: -.. code-block:: sh - git checkout -b topic_of_your_contribution + git checkout -b topic_of_your_contribution -3. When you are done making the changes you wish to contribute, record them in Git: +3. When you are done making the changes you wish to contribute, record them in Git:: -.. code-block:: sh - git add the/paths/to/files/you/modified can/be/more/than/one - git commit + git add the/paths/to/files/you/modified can/be/more/than/one + git commit 3. Push the changes to your copy of the code on GitHub, following which Git will - provide you with a link which you can click to initiate a pull request + provide you with a link which you can click to initiate a pull request:: -.. code-block:: sh - git push -u origin topic_of_your_contribution + git push -u origin topic_of_your_contribution (If any of the above seems overwhelming, you can look up the `Git documentation From 3f832471a87cca3d65a2217e8a840fc4efae2d74 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 15 Feb 2023 15:46:52 -0500 Subject: [PATCH 097/176] typo --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 840a6172..dd78d44b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -63,7 +63,7 @@ We support Python 3 only (>= 3.7). Dependencies which you will need are `listed in the repository `_. Note that you will likely have these will already be available on your system if you used a -package manger (e.g. Debian's ``apt-get``, Gentoo's ``emerge``, or simply PIP) to install the +package manager (e.g. Debian's ``apt-get``, Gentoo's ``emerge``, or simply PIP) to install the software. Development work might require live access to the copy of HeuDiConv which is being developed. From 3979f1e5b14b8a123b4f9a6cf96114c2d2a95e08 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 15 Feb 2023 15:48:27 -0500 Subject: [PATCH 098/176] PIP install command --- CONTRIBUTING.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index dd78d44b..aabfde17 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -69,13 +69,13 @@ software. Development work might require live access to the copy of HeuDiConv which is being developed. If a system-wide release of HeuDiConv is already installed, or likely to be, it is best to keep development work sandboxed inside a dedicated virtual environment. -This is best accomplished via: +This is best accomplished via:: -.. code-block:: sh cd /path/to/your/clone/of/heudiconv mkdir -p venvs/dev python -m venv venvs/dev source venvs/dev/bin/activate + pip install -e .[all] Additional Hints From 3e94145fefea593762c52107eb005e2aa74c57a6 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 15 Feb 2023 17:44:42 -0500 Subject: [PATCH 099/176] Added contribution guide to documentation --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 621bd2b2..5565625b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,6 +4,7 @@ contain the root `toctree` directive. .. include:: ../README.rst +.. include:: ../CONTRIBUTING.rst Contents -------- From 5bf16a3cbdbafd65fc377fde5409af3408d562c5 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 15 Feb 2023 17:55:31 -0500 Subject: [PATCH 100/176] Style hints for contribution --- CONTRIBUTING.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index aabfde17..88bef0bc 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -86,9 +86,11 @@ rules before submitting a pull request: * All public functions (i.e. functions whose name does not start with an underscore) should have informative docstrings with sample usage presented as doctests when appropriate. -.. code-block:: sh - cd /path/to/your/clone/of/heudiconv - pytest -vvs . +* Docstrings are formatted in `NumPy style `_. +* Lines are no longer than 120 characters. +* All tests still pass:: + + cd /path/to/your/clone/of/heudiconv + pytest -vvs . -* All other tests pass: * New code should be accompanied by new tests. From f424bc57a398442be615fa2a581e7c509e5e0dc3 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Fri, 24 Feb 2023 12:21:34 -0800 Subject: [PATCH 101/176] Some rewording on the installation page --- docs/installation.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 0f43b3a7..0ad8f2f8 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -13,7 +13,7 @@ If installing through ``PyPI``, eg:: pip install heudiconv[all] -On Debian-based systems we recommend using `NeuroDebian `_ +On Debian-based systems, we recommend using `NeuroDebian `_, which provides the `heudiconv package `_. @@ -25,13 +25,13 @@ to view available releases. To pull the latest release, run:: $ docker pull nipy/heudiconv:latest -Note that when using via ``docker run`` you might need to provide your user and group IDs so they map correspondingly -within container, i.e. like:: +Note that when using HeuDiConv via ``docker run``, you might need to provide your user and group IDs so they map correspondingly +within the container, i.e.:: $ docker run --user=$(id -u):$(id -g) -e "UID=$(id -u)" -e "GID=$(id -g)" --rm -t -v $PWD:$PWD nipy/heudiconv:latest [OPTIONS TO FOLLOW] -`ReproIn heuristic project `_ provides its own Docker images from -Docker Hub `repronim/reproin` which bundle its `reproin` helper. +Additionally, HeuDiConv is available through the Docker image at `repronim/reproin `_ provided by +`ReproIn heuristic project `_, which develops the ``reproin`` heuristic. Singularity =========== @@ -51,6 +51,6 @@ It contains converted from Docker singularity images for stock heudiconv images `__) and reproin images (as `repronim-reproin `__). Please see `"A typical workflow" `_ section for a prototypical example of using -`datalad-container `_ extension with this dataset, while fulfilling +`datalad-container `_ extension with this dataset while fulfilling `YODA principles `_. **Note** that it should also work on OSX with ``///repronim/containers`` automagically taking care about running those Singularity containers via Docker. From c8f71b2ca8c89de3b2c0b2ff7271faffeddf0c5e Mon Sep 17 00:00:00 2001 From: Isaac To Date: Fri, 3 Mar 2023 10:13:27 -0800 Subject: [PATCH 102/176] Reword the last paragraph --- docs/installation.rst | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 0ad8f2f8..0dfca47e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -41,16 +41,22 @@ build the latest release, you can run:: $ singularity pull docker://nipy/heudiconv:latest + Singularity YODA style using ///repronim/containers =================================================== +`ReproNim `_ provides a large collection of Singularity container images of popular +neuroimaging tools, e.g. all the BIDS-Apps. This collection also includes the forementioned container +images for `HeuDiConv `_ and +`ReproIn `_ in the Singularity image format. This collection is available as a +`DataLad `_ dataset at `///repronim/containers `_ +on `Datalad.org `_ and as `a GitHub repo `_. +The HeuDiConv and ReproIn container images are named ``nipy-heudiconv`` and ``repronim-reproin``, respectively, in this collection. +To use them, you can install the DataLad dataset and then use the ``datalad containers-run`` command to run. +For a more detailed example of using images from this collection while fulfilling +the `YODA Principles `_, please check out +`A typical YODA workflow `_ in +the documentation of this collection. + +**Note:** With the ``datalad containers-run`` command, the images in this collection work on macOS (OSX) +as well for ``datalad containers-run`` automagically takes care of running the Singularity containers via Docker. -ReproNim project provides `///repronim/containers `_ -(git clone present also on `GitHub `__) `DataLad -`_ dataset with Singularity containers for many popular neuroimaging tools, e.g. all BIDS-Apps. -It contains converted from Docker singularity images for stock heudiconv images (as `nipy-heudiconv -`__) and reproin images (as `repronim-reproin -`__). Please see `"A typical workflow" -`_ section for a prototypical example of using -`datalad-container `_ extension with this dataset while fulfilling -`YODA principles `_. **Note** that it should also work on -OSX with ``///repronim/containers`` automagically taking care about running those Singularity containers via Docker. From cfde934d0a354ec7b5cd12bc5eadab86c42cc86e Mon Sep 17 00:00:00 2001 From: Isaac To Date: Fri, 3 Mar 2023 12:48:09 -0800 Subject: [PATCH 103/176] Specify datasets.datalad.org as images host sites Co-authored-by: Yaroslav Halchenko --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 0dfca47e..4f8996bd 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -49,7 +49,7 @@ neuroimaging tools, e.g. all the BIDS-Apps. This collection also includes the fo images for `HeuDiConv `_ and `ReproIn `_ in the Singularity image format. This collection is available as a `DataLad `_ dataset at `///repronim/containers `_ -on `Datalad.org `_ and as `a GitHub repo `_. +on `datasets.datalad.org `_ and as `a GitHub repo `_. The HeuDiConv and ReproIn container images are named ``nipy-heudiconv`` and ``repronim-reproin``, respectively, in this collection. To use them, you can install the DataLad dataset and then use the ``datalad containers-run`` command to run. For a more detailed example of using images from this collection while fulfilling From b2c38272a3aa6b37069278e5edbefbdea9131064 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Fri, 3 Mar 2023 12:50:47 -0800 Subject: [PATCH 104/176] Clarify why singularity images work on MacOS Co-authored-by: Yaroslav Halchenko --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 4f8996bd..eaaaed34 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -58,5 +58,5 @@ the `YODA Principles the documentation of this collection. **Note:** With the ``datalad containers-run`` command, the images in this collection work on macOS (OSX) -as well for ``datalad containers-run`` automagically takes care of running the Singularity containers via Docker. +as well for ``repronim/containers`` helpers automagically take care of running the Singularity containers via Docker. From de5b2d1d14c836d18853739a315f5d86215b8d51 Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Thu, 9 Mar 2023 14:11:27 -0500 Subject: [PATCH 105/176] Ensure that the extracted reproducible int corresponds to something like a time --- heudiconv/bids.py | 3 +- heudiconv/dicoms.py | 112 +++++++++++++++++++++------------ heudiconv/tests/test_dicoms.py | 87 +++++++++++++++++-------- 3 files changed, 137 insertions(+), 65 deletions(-) diff --git a/heudiconv/bids.py b/heudiconv/bids.py index cd28bd7a..82c77c1d 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -14,6 +14,7 @@ from random import sample from glob import glob import errno +import typing import warnings from .external.pydicom import dcm @@ -462,7 +463,7 @@ def add_rows_to_scans_keys_file(fn, newrows): writer.writerows([header] + data_rows_sorted) -def get_formatted_scans_key_row(dcm_fn) -> list[str]: +def get_formatted_scans_key_row(dcm_fn) -> typing.List[str]: """ Parameters ---------- diff --git a/heudiconv/dicoms.py b/heudiconv/dicoms.py index 2a17f388..0a49dc95 100644 --- a/heudiconv/dicoms.py +++ b/heudiconv/dicoms.py @@ -6,6 +6,8 @@ from collections import OrderedDict import tarfile +from typing import List, Optional + from .external.pydicom import dcm from .utils import ( get_typed_attr, @@ -315,55 +317,87 @@ def group_dicoms_into_seqinfos(files, grouping, file_filter=None, return seqinfos -def get_timestamp_from_series(dicom_list: list[str]) -> int: - """try to return a timestamp indicating when the first dicom in dicom_list was created, or if that - info isn't present then a reproducible integer. This is used in setting mtimes reproducibly +def get_reproducible_int(dicom_list: List[str]) -> int: + """Get integer that can be used to reproducibly sort input DICOMs, which is based on when they were acquired. + + Parameters + ---------- + dicom_list : List[str] + Paths to existing DICOM files + + Returns + ------- + int + An integer relating to when the DICOM was acquired + + Raises + ------ + AssertionError - Args: - dicom_list (list[str]): list of strings pointing to existing dicom files + Notes + ----- + + 1. When date and time for can be read (see :func:`get_datetime_from_dcm`), return + that value as time in seconds since epoch (i.e., Jan 1 1970). + 2. In cases where a date/time/datetime is not available (e.g., anonymization stripped this info), return + epoch + AcquisitionNumber (in seconds), which is AcquisitionNumber as an integer + 3. If 1 and 2 are not possible, then raise AssertionError and provide message about missing information - Returns: - int: either an int representing when the first dicom was created, - in number of seconds since epoch, or if no datetime info is found then a hash of the - SeriesInstanceUID (meaningless value, but reproducible) + Cases are based on only the first element of the dicom_list. """ import calendar dicom = dcm.read_file(dicom_list[0], stop_before_pixels=True, force=True) - if (dicom_datetime := get_datetime_from_dcm(dicom)): + dicom_datetime = get_datetime_from_dcm(dicom) + if dicom_datetime: return calendar.timegm(dicom_datetime.timetuple()) - else: - logging.warning("unable to get real timestamp from series. returning arbitrary time based on hash of SeriesInstanceUID") - return hash(dicom.SeriesInstanceUID) + + acquisition_number = dicom.get('AcquisitionNumber') + if acquisition_number: + return int(acquisition_number) + + raise AssertionError( + "No metadata found that can be used to sort DICOMs reproducibly. Was header information erased?" + ) -def get_datetime_from_dcm(dcm_data: dcm.FileDataset) -> datetime.datetime | None: - """try to extract datetime from filedataset +def get_datetime_from_dcm(dcm_data: dcm.FileDataset) -> Optional[datetime.datetime]: + """Extract datetime from filedataset, or return None is no datetime information found. - Args: - dcm_data (pydicom.FileDataset): dicom with header, e.g., as ready by pydicom.dcmread + Parameters + ---------- + dcm_data : dcm.FileDataset + DICOM with header, e.g., as ready by pydicom.dcmread. + Objects with __getitem__ and have those keys with values properly formatted may also work + + Returns + ------- + Optional[datetime.datetime] + One of several datetimes that are related to when the scan occurred, or None if no datetime can be found + + Notes + ------ + The following fields are checked in order + + 1. AcquisitionDate & AcquisitionTime (0008,0022); (0008,0032) + 2. AcquisitionDateTime (0008,002A); + 3. SeriesDate & SeriesTime (0008,0021); (0008,0031) - Returns: - datetime.datetime | None: one of several datetimes that are related to when the scan occurred. - """ - if ( - ( - (acq_date := dcm_data.get("AcquisitionDate")) - and (acq_time := dcm_data.get("AcquisitionTime")) - ) - or ( - (acq_date := dcm_data.get("SeriesDate")) - and (acq_time := dcm_data.get("SeriesTime")) - ) - ): - acq_datetime = datetime.datetime.strptime(acq_date+acq_time, "%Y%m%d%H%M%S.%f") - elif (acq_dt := dcm_data.get("AcquisitionDateTime")): - acq_datetime = datetime.datetime.strptime(acq_dt, "%Y%m%d%H%M%S.%f") - else: - acq_datetime = None - return acq_datetime + acq_date = dcm_data.get("AcquisitionDate") + acq_time = dcm_data.get("AcquisitionTime") + if not (acq_date is None or acq_time is None): + return datetime.datetime.strptime(acq_date+acq_time, "%Y%m%d%H%M%S.%f") + + acq_dt = dcm_data.get("AcquisitionDateTime") + if not acq_dt is None: + return datetime.datetime.strptime(acq_dt, "%Y%m%d%H%M%S.%f") + + series_date = dcm_data.get("SeriesDate") + series_time = dcm_data.get("SeriesTime") + if not (series_date is None or series_time is None): + return datetime.datetime.strptime(series_date+series_time, "%Y%m%d%H%M%S.%f") def compress_dicoms(dicom_list, out_prefix, tempdirs, overwrite): @@ -401,11 +435,11 @@ def compress_dicoms(dicom_list, out_prefix, tempdirs, overwrite): # Solution from DataLad although ugly enough: dicom_list = sorted(dicom_list) - dcm_time = get_timestamp_from_series(dicom_list) + dcm_time = get_reproducible_int(dicom_list) def _assign_dicom_time(ti: tarfile.TarInfo) -> tarfile.TarInfo: - # Try to reset the date to match the one of the last commit, not from the - # filesystem since git doesn't track those at all + # Reset the date to match the one from the dicom, not from the + # filesystem so we could sort reproducibly ti.mtime = dcm_time return ti diff --git a/heudiconv/tests/test_dicoms.py b/heudiconv/tests/test_dicoms.py index 10aa9ef5..16ac3cf3 100644 --- a/heudiconv/tests/test_dicoms.py +++ b/heudiconv/tests/test_dicoms.py @@ -1,3 +1,4 @@ +import datetime import os.path as op import json from glob import glob @@ -13,7 +14,7 @@ group_dicoms_into_seqinfos, parse_private_csa_header, get_datetime_from_dcm, - get_timestamp_from_series + get_reproducible_int ) from .utils import ( assert_cwd_unchanged, @@ -89,49 +90,85 @@ def test_group_dicoms_into_seqinfos(tmpdir): assert [s.series_description for s in seqinfo] == ['AAHead_Scout_32ch-head-coil', 'PhoenixZIPReport'] - -def test_get_datetime_from_dcm(): - import datetime +def test_get_datetime_from_dcm_from_acq_date_time(): typical_dcm = dcm.dcmread(op.join(TESTS_DATA_PATH, 'phantom.dcm'), stop_before_pixels=True) - XA30_enhanced_dcm = dcm.dcmread(op.join(TESTS_DATA_PATH, 'MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm'), stop_before_pixels=True) # do we try to grab from AcquisitionDate/AcquisitionTime first when available? - assert type(get_datetime_from_dcm(typical_dcm)) is datetime.datetime - assert get_datetime_from_dcm(typical_dcm) == datetime.datetime.strptime( - typical_dcm.get("AcquisitionDate") + typical_dcm.get("AcquisitionTime"), - "%Y%m%d%H%M%S.%f" + dt = get_datetime_from_dcm(typical_dcm) + assert ( + dt == datetime.datetime.strptime( + typical_dcm.get("AcquisitionDate") + typical_dcm.get("AcquisitionTime"), + "%Y%m%d%H%M%S.%f" ) + ) - # can we rely on AcquisitionDateTime if AcquisitionDate and AcquisitionTime not there? - assert type(get_datetime_from_dcm(XA30_enhanced_dcm)) is datetime.datetime - # if these aren't available, can we rely on AcquisitionDateTime? - del XA30_enhanced_dcm.SeriesDate - del XA30_enhanced_dcm.SeriesTime - assert type(get_datetime_from_dcm(XA30_enhanced_dcm)) is datetime.datetime +def test_get_datetime_from_dcm_from_acq_datetime(): + # if AcquisitionDate and AcquisitionTime not there, can we rely on AcquisitionDateTime? + XA30_enhanced_dcm = dcm.dcmread( + op.join(TESTS_DATA_PATH, 'MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm'), + stop_before_pixels=True + ) + dt = get_datetime_from_dcm(XA30_enhanced_dcm) + assert (dt == datetime.datetime.strptime(XA30_enhanced_dcm.get("AcquisitionDateTime"), "%Y%m%d%H%M%S.%f")) + - # and if there's no known source (e.g., after anonymization), are we still able to proceed? +def test_get_datetime_from_dcm_from_only_series_date_time(): + # if acquisition date/time/datetime not available, can we rely on SeriesDate & SeriesTime? + XA30_enhanced_dcm = dcm.dcmread( + op.join(TESTS_DATA_PATH, 'MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm'), + stop_before_pixels=True + ) del XA30_enhanced_dcm.AcquisitionDateTime + dt = get_datetime_from_dcm(XA30_enhanced_dcm) + assert ( + dt == datetime.datetime.strptime( + XA30_enhanced_dcm.get("SeriesDate") + XA30_enhanced_dcm.get("SeriesTime"), + "%Y%m%d%H%M%S.%f" + ) + ) + +def test_get_datetime_from_dcm_wo_dt(): + # if there's no known source (e.g., after anonymization), are we still able to proceed? + XA30_enhanced_dcm = dcm.dcmread( + op.join(TESTS_DATA_PATH, 'MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm'), + stop_before_pixels=True + ) + del XA30_enhanced_dcm.AcquisitionDateTime + del XA30_enhanced_dcm.SeriesDate + del XA30_enhanced_dcm.SeriesTime assert get_datetime_from_dcm(XA30_enhanced_dcm) is None -def test_get_timestamp_from_series(tmpdir): +def test_get_reproducible_int(): dcmfile = op.join(TESTS_DATA_PATH, 'phantom.dcm') - assert type(get_timestamp_from_series([dcmfile])) is int + assert type(get_reproducible_int([dcmfile])) is int + +@pytest.fixture +def test_get_reproducible_int_wo_dt(tmp_path): # can this function return an int when we don't have any useable dates? typical_dcm = dcm.dcmread(op.join(TESTS_DATA_PATH, 'phantom.dcm'), stop_before_pixels=True) - del typical_dcm.InstanceCreationDate - del typical_dcm.StudyDate del typical_dcm.SeriesDate del typical_dcm.AcquisitionDate - del typical_dcm.ContentDate - del typical_dcm.PerformedProcedureStepStartDate - tmp_dcmfile = tmpdir / "phantom.dcm" - dcm.dcmwrite(tmp_dcmfile, typical_dcm) + dcm.dcmwrite(tmp_path, typical_dcm) + + assert type(get_reproducible_int([tmp_path])) is int - assert type(get_timestamp_from_series([tmp_dcmfile])) is int +@pytest.fixture +def test_get_reproducible_int_raises_assertion_wo_dt(tmp_path): + # if there's no known source (e.g., after anonymization), is AssertionError Raised? + XA30_enhanced_dcm = dcm.dcmread( + op.join(TESTS_DATA_PATH, 'MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm'), + stop_before_pixels=True + ) + del XA30_enhanced_dcm.AcquisitionDateTime + del XA30_enhanced_dcm.SeriesDate + del XA30_enhanced_dcm.SeriesTime + dcm.dcmwrite(tmp_path, dataset=XA30_enhanced_dcm) + with pytest.raises(AssertionError): + get_reproducible_int([tmp_path]) From 74e3fcc3f21c54a410f90cd25d174cecd58546a6 Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Thu, 9 Mar 2023 15:21:00 -0500 Subject: [PATCH 106/176] codespell --- heudiconv/dicoms.py | 2 +- heudiconv/tests/test_dicoms.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/heudiconv/dicoms.py b/heudiconv/dicoms.py index 0a49dc95..7e0fb0ee 100644 --- a/heudiconv/dicoms.py +++ b/heudiconv/dicoms.py @@ -172,7 +172,7 @@ def group_dicoms_into_seqinfos(files, grouping, file_filter=None, `seqinfo` is a list of info entries per each sequence (some entry there defines a key for `filegrp`) filegrp : dict - `filegrp` is a dictionary with files groupped per each sequence + `filegrp` is a dictionary with files grouped per each sequence """ allowed_groupings = ['studyUID', 'accession_number', 'all', 'custom'] if grouping not in allowed_groupings: diff --git a/heudiconv/tests/test_dicoms.py b/heudiconv/tests/test_dicoms.py index 16ac3cf3..1497cbfe 100644 --- a/heudiconv/tests/test_dicoms.py +++ b/heudiconv/tests/test_dicoms.py @@ -148,7 +148,7 @@ def test_get_reproducible_int(): @pytest.fixture def test_get_reproducible_int_wo_dt(tmp_path): - # can this function return an int when we don't have any useable dates? + # can this function return an int when we don't have any usable dates? typical_dcm = dcm.dcmread(op.join(TESTS_DATA_PATH, 'phantom.dcm'), stop_before_pixels=True) del typical_dcm.SeriesDate del typical_dcm.AcquisitionDate From 34c4c90890439c91c68f1b5048b14bb4b86bcd81 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 13 Mar 2023 23:03:22 -0400 Subject: [PATCH 107/176] Revert "Do not bother with separate installation of dcm2niix in neurodocker" This reverts commit 6c2cba81820191d85ba05da3a72ae3ecaea085fb. only we re-add adding executable permissions to the script --- utils/gen-docker-image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/gen-docker-image.sh b/utils/gen-docker-image.sh index 7b87c1f2..70d6ea65 100755 --- a/utils/gen-docker-image.sh +++ b/utils/gen-docker-image.sh @@ -8,6 +8,7 @@ VER=$(grep -Po '(?<=^__version__ = ).*' $thisd/../heudiconv/info.py | sed 's/"// image="kaczmarj/neurodocker:0.9.1" docker run --rm $image generate docker -b neurodebian:bullseye -p apt \ + --dcm2niix version=v1.0.20220720 method=source \ --install git gcc pigz liblzma-dev libc-dev git-annex-standalone netbase \ --copy . /src/heudiconv \ --miniconda version="py39_4.12.0" conda_install="python=3.9 traits>=4.6.0 scipy numpy nomkl pandas" \ From 4e3817b1bae3cfa86ff77dba250ce60c3c112847 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 13 Mar 2023 23:07:47 -0400 Subject: [PATCH 108/176] Re-adding instructions to install dcm2niix manually --- .github/workflows/test.yml | 2 +- docs/installation.rst | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4446f84c..4973aef7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: # The ultimate one-liner setup for NeuroDebian repository bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) sudo apt-get update -qq - sudo apt-get install git-annex-standalone + sudo apt-get install git-annex-standalone dcm2niix - name: Install dependencies run: | diff --git a/docs/installation.rst b/docs/installation.rst index eaaaed34..921233ea 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -13,6 +13,11 @@ If installing through ``PyPI``, eg:: pip install heudiconv[all] +Manual installation of `dcm2niix `_ +is required. You can also benefit from an installer/downloader helper ``dcm2niix`` package +on ``PyPI``, so you can simply ``pip install dcm2niix`` if you are installing in user space so +subsequently it would be able to download and install dcm2niix binary. + On Debian-based systems, we recommend using `NeuroDebian `_, which provides the `heudiconv package `_. From 84d8151f376f0d65d49011f8387b0f721f413e6f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 13 Mar 2023 23:09:10 -0400 Subject: [PATCH 109/176] Disable dependency on dcm2niix in our requires --- heudiconv/info.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/heudiconv/info.py b/heudiconv/info.py index 0ae24df0..8e354823 100644 --- a/heudiconv/info.py +++ b/heudiconv/info.py @@ -22,7 +22,8 @@ PYTHON_REQUIRES = ">=3.7" REQUIRES = [ - 'dcm2niix', + # not usable in some use cases since might be just a downloader, not binary + # 'dcm2niix', 'dcmstack>=0.8', 'etelemetry', 'filelock>=3.0.12', From b530ddc6700e0dc9aa5d94dc54b53a0defae2a92 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 13 Mar 2023 23:16:46 -0400 Subject: [PATCH 110/176] Fix new typo identified with codespell -- groupped --- heudiconv/dicoms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/dicoms.py b/heudiconv/dicoms.py index 1a5c2b95..3062e757 100644 --- a/heudiconv/dicoms.py +++ b/heudiconv/dicoms.py @@ -169,7 +169,7 @@ def group_dicoms_into_seqinfos(files, grouping, file_filter=None, `seqinfo` is a list of info entries per each sequence (some entry there defines a key for `filegrp`) filegrp : dict - `filegrp` is a dictionary with files groupped per each sequence + `filegrp` is a dictionary with files grouped per each sequence """ allowed_groupings = ['studyUID', 'accession_number', 'all', 'custom'] if grouping not in allowed_groupings: From 4bb1f15e26367fdcf6574f368a79dcd2a717129a Mon Sep 17 00:00:00 2001 From: auto Date: Tue, 14 Mar 2023 11:52:40 +0000 Subject: [PATCH 111/176] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9fa32ef..c475bab2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +# v0.12.1 (Tue Mar 14 2023) + +#### 🐛 Bug Fix + +- Re-add explicit instructions to install dcm2niix "manually" and remove it from install_requires [#651](https://github.com/nipy/heudiconv/pull/651) ([@yarikoptic](https://github.com/yarikoptic)) + +#### 📝 Documentation + +- Contributing guide. [#641](https://github.com/nipy/heudiconv/pull/641) ([@TheChymera](https://github.com/TheChymera)) +- Reword and correct punctuation on installation.rst [#643](https://github.com/nipy/heudiconv/pull/643) ([@yarikoptic](https://github.com/yarikoptic) [@candleindark](https://github.com/candleindark)) + +#### Authors: 3 + +- Horea Christian ([@TheChymera](https://github.com/TheChymera)) +- Isaac To ([@candleindark](https://github.com/candleindark)) +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + # v0.12.0 (Tue Feb 21 2023) #### 🚀 Enhancement From efef1d385a4f331b03c0562d6d77ce327ef7ecff Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 14 Mar 2023 11:59:19 -0400 Subject: [PATCH 112/176] [DATALAD RUNCMD] produce updated dockerfile === Do not change lines below === { "chain": [ "e179ee94e55f524620bea09c4d29fe56a7908fff" ], "cmd": "utils/gen-docker-image.sh", "exit": 0, "extra_inputs": [], "inputs": [], "outputs": [], "pwd": "." } ^^^ Do not change lines above ^^^ --- Dockerfile | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Dockerfile b/Dockerfile index c65ba1ea..5ec60639 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,28 @@ # Generated by Neurodocker and Reproenv. FROM neurodebian:bullseye +ENV PATH="/opt/dcm2niix-v1.0.20220720/bin:$PATH" +RUN apt-get update -qq \ + && apt-get install -y -q --no-install-recommends \ + ca-certificates \ + cmake \ + g++ \ + gcc \ + git \ + make \ + pigz \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* \ + && git clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix \ + && cd /tmp/dcm2niix \ + && git fetch --tags \ + && git checkout v1.0.20220720 \ + && mkdir /tmp/dcm2niix/build \ + && cd /tmp/dcm2niix/build \ + && cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20220720 .. \ + && make -j1 \ + && make install \ + && rm -rf /tmp/dcm2niix RUN apt-get update -qq \ && apt-get install -y -q --no-install-recommends \ gcc \ @@ -67,6 +89,18 @@ RUN printf '{ \ "base_image": "neurodebian:bullseye" \ } \ }, \ + { \ + "name": "env", \ + "kwds": { \ + "PATH": "/opt/dcm2niix-v1.0.20220720/bin:$PATH" \ + } \ + }, \ + { \ + "name": "run", \ + "kwds": { \ + "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n ca-certificates \\\\\\n cmake \\\\\\n g++ \\\\\\n gcc \\\\\\n git \\\\\\n make \\\\\\n pigz \\\\\\n zlib1g-dev\\nrm -rf /var/lib/apt/lists/*\\ngit clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix\\ncd /tmp/dcm2niix\\ngit fetch --tags\\ngit checkout v1.0.20220720\\nmkdir /tmp/dcm2niix/build\\ncd /tmp/dcm2niix/build\\ncmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20220720 ..\\nmake -j1\\nmake install\\nrm -rf /tmp/dcm2niix" \ + } \ + }, \ { \ "name": "install", \ "kwds": { \ From 71d87c0fe9562503e89e51fe5f82e65a692e2226 Mon Sep 17 00:00:00 2001 From: auto Date: Tue, 14 Mar 2023 16:03:03 +0000 Subject: [PATCH 113/176] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c475bab2..ff02dad0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# v0.12.2 (Tue Mar 14 2023) + +#### 🏠 Internal + +- [DATALAD RUNCMD] produce updated dockerfile [#652](https://github.com/nipy/heudiconv/pull/652) ([@yarikoptic](https://github.com/yarikoptic)) + +#### Authors: 1 + +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + # v0.12.1 (Tue Mar 14 2023) #### 🐛 Bug Fix From 6a1b4499acdd53d4eb0bb20a7251c105ba9ab765 Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Thu, 23 Mar 2023 11:57:35 -0400 Subject: [PATCH 114/176] allow input of zipped dicoms --- heudiconv/parser.py | 127 +++++++++++++++++-------------- heudiconv/tests/test_archives.py | 74 ++++++++++++++++++ 2 files changed, 142 insertions(+), 59 deletions(-) create mode 100644 heudiconv/tests/test_archives.py diff --git a/heudiconv/parser.py b/heudiconv/parser.py index fa37d0f6..d47bc099 100644 --- a/heudiconv/parser.py +++ b/heudiconv/parser.py @@ -7,9 +7,10 @@ from collections import defaultdict -import tarfile -import zipfile -from tempfile import mkdtemp +import shutil +import typing +from itertools import chain +from pathlib import Path from .dicoms import group_dicoms_into_seqinfos from .utils import ( @@ -25,6 +26,7 @@ _VCS_REGEX = r'%s\.(?:git|gitattributes|svn|bzr|hg)(?:%s|$)' % (op.sep, op.sep) +_UNPACK_FORMATS = tuple(chain.from_iterable([x[1] for x in shutil.get_unpack_formats()])) @docstring_parameter(_VCS_REGEX) def find_files(regex, topdir=op.curdir, exclude=None, @@ -61,75 +63,82 @@ def find_files(regex, topdir=op.curdir, exclude=None, yield path -def get_extracted_dicoms(fl): - """Given a list of files, possibly extract some from tarballs. +def _get_files_in_dir(src: str) -> typing.List[str]: + return [str(f.resolve()) for f in Path(src).rglob("*") if f.is_file()] - For 'classical' heudiconv, if multiple tarballs are provided, they correspond - to different sessions, so here we would group into sessions and return - pairs `sessionid`, `files` with `sessionid` being None if no "sessions" - detected for that file or there was just a single tarball in the list + +def get_extracted_dicoms( + fl: typing.Collection[str] + ) -> typing.ItemsView[typing.Optional[int], typing.List[str]]: + """Given a collection of files and/or directories, possibly extract + some from tarballs. + + Parameters + ---------- + fl : Collection[str] + A collection (e.g., list or tuple) of files that will be extracted. + + Returns + ------- + ItemsView[int, list[str]] + A tuple of integer keys and list of strs representing absolute paths + of extracted files. + + Notes + ----- + For 'classical' heudiconv, if multiple tarballs are provided, they + correspond to different sessions, so here we would group into sessions + and return pairs `sessionid`, `files` with `sessionid` being None if no + "sessions" detected for that file or there was just a single tarball in the + list. + + When contents of fl are directories whose names do not have a suffix + that is recognized as a common archive format (e.g., .tgz), then the + corresponding item will contain a list of the files in that directory. + + When contents of fl appeart to be an unpackable archive, the contents are + extracted into utils.TempDirs(f'heudiconvDCM-{ses}') and the mode of all + extracted files is set to 700. """ # TODO: bring check back? # if any(not tarfile.is_tarfile(i) for i in fl): # raise ValueError("some but not all input files are tar files") - - # tarfiles already know what they contain, and often the filenames - # are unique, or at least in a unique subdir per session - # strategy: extract everything in a temp dir and assemble a list - # of all files in all tarballs - - # cannot use TempDirs since will trigger cleanup with __del__ - tmpdir = tempdirs('heudiconvDCM') - - sessions = defaultdict(list) - session = 0 + sessions: typing.DefaultDict[ + typing.Optional[int], + typing.List[str] + ] = defaultdict(list) if not isinstance(fl, (list, tuple)): fl = list(fl) + # keep track of session manually to ensure that the variable is bound + # when it is used after the loop (e.g., consider situation with + # fl being empty) + session = 0 # needs sorting to keep the generated "session" label deterministic - for i, t in enumerate(sorted(fl)): - # "classical" heudiconv has that heuristic to handle multiple - # tarballs as providing different sessions per each tarball or zipfile - - if tarfile.is_tarfile(t): - # cannot use TempDirs since will trigger cleanup with __del__ - tmpdir = mkdtemp(prefix='heudiconvDCM') - # load file - tf = tarfile.open(t) + for _, t in enumerate(sorted(fl)): + # Each file extracted must be associated with the proper session, + # but the high-level shutil does not have a way to list the files + # contained within each archive. So, files are temporarily + # extracted into unique tempdirs + # cannot use TempDirs since will trigger cleanup with __del__ + tmpdir = tempdirs(prefix="heudiconvDCM") + + if t.endswith(_UNPACK_FORMATS): + shutil.unpack_archive(t, extract_dir=tmpdir) + tf_content = _get_files_in_dir(tmpdir) # check content and sanitize permission bits - tmembers = tf.getmembers() - for tm in tmembers: - tm.mode = 0o700 - # get all files, assemble full path in tmp dir - tf_content = [m.name for m in tmembers if m.isfile()] - # store full paths to each file, so we don't need to drag along - # tmpdir as some basedir - sessions[session] = [op.join(tmpdir, f) for f in tf_content] - session += 1 - # extract into tmp dir - tf.extractall(path=tmpdir, members=tmembers) - - elif zipfile.is_zipfile(t): - # cannot use TempDirs since will trigger cleanup with __del__ - tmpdir = mkdtemp(prefix='heudiconvDCM') - # load file - zf = zipfile.ZipFile(t) - # check content - zmembers = zf.infolist() - # get all files, assemble full path in tmp dir - zf_content = [m.filename for m in zmembers if not m.is_dir()] - # store full paths to each file, so we don't need to drag along - # tmpdir as some basedir - sessions[session] = [op.join(tmpdir, f) for f in zf_content] - session += 1 - # extract into tmp dir - zf.extractall(path=tmpdir) - + for f in tf_content: + os.chmod(f, mode=0o700) else: - sessions[None].append(t) + tf_content = _get_files_in_dir(t) + + # store full paths to each file, so we don't need to drag along + # tmpdir as some basedir + sessions[session] = tf_content + session += 1 if session == 1: - # we had only 1 session, so no really multiple sessions according + # we had only 1 session, so not really multiple sessions according # to classical 'heudiconv' assumptions, thus just move them all into # None sessions[None] += sessions.pop(0) diff --git a/heudiconv/tests/test_archives.py b/heudiconv/tests/test_archives.py new file mode 100644 index 00000000..3d921d61 --- /dev/null +++ b/heudiconv/tests/test_archives.py @@ -0,0 +1,74 @@ +import os +import os.path as op +from pathlib import Path +import stat +import typing + + +import pytest +import shutil +from .utils import TESTS_DATA_PATH +from ..parser import get_extracted_dicoms + + +@pytest.fixture +def get_dicoms_gztar(tmpdir: Path) -> typing.List[str]: + tmp_file = tmpdir / "dicom" + archive = shutil.make_archive( + str(tmp_file), + format="gztar", + root_dir=TESTS_DATA_PATH, + base_dir="01-anat-scout") + return [archive] + + +@pytest.fixture +def get_dicoms_zip(tmpdir: Path) -> typing.List[str]: + tmp_file = tmpdir / "dicom" + archive = shutil.make_archive( + str(tmp_file), + format="zip", + root_dir=TESTS_DATA_PATH, + base_dir="01-anat-scout") + return [archive] + + +@pytest.fixture +def get_dicoms_folder() -> typing.List[str]: + return [op.join(TESTS_DATA_PATH, "01-anat-scout")] + + +def test_get_extracted_dicoms_single_session_is_none(get_dicoms_gztar: typing.List[str]): + for session_, _ in get_extracted_dicoms(get_dicoms_gztar): + assert session_ is None + + +def test_get_extracted_dicoms_multple_session_integers(get_dicoms_gztar: typing.List[str]): + sessions = [] + for session, _ in get_extracted_dicoms(get_dicoms_gztar + get_dicoms_gztar): + sessions.append(session) + + assert sessions == [0, 1] + + +def test_get_extracted_dicoms_from_tgz(get_dicoms_gztar: typing.List[str]): + for _, files in get_extracted_dicoms(get_dicoms_gztar): + # check that the only file is the one called "0001.dcm" + assert all(file.endswith("0001.dcm") for file in files) + + +def test_get_extracted_dicoms_from_zip(get_dicoms_zip: typing.List[str]): + for _, files in get_extracted_dicoms(get_dicoms_zip): + # check that the only file is the one called "0001.dcm" + assert all(file.endswith("0001.dcm") for file in files) + + +def test_get_extracted_dicoms_from_folder(get_dicoms_folder: typing.List[str]): + for _, files in get_extracted_dicoms(get_dicoms_folder): + # check that the only file is the one called "0001.dcm" + assert all(file.endswith("0001.dcm") for file in files) + + +def test_get_extracted_have_correct_permissions(get_dicoms_gztar: typing.List[str]): + for _, files in get_extracted_dicoms(get_dicoms_gztar): + assert all(stat.S_IMODE(os.stat(file).st_mode) == 448 for file in files) From d508eb034366ccba131995b3475113102ab05962 Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Thu, 23 Mar 2023 13:53:23 -0400 Subject: [PATCH 115/176] ensure no tests fail --- heudiconv/parser.py | 103 ++++++++++++++++--------------- heudiconv/tests/test_archives.py | 24 +++++-- 2 files changed, 70 insertions(+), 57 deletions(-) diff --git a/heudiconv/parser.py b/heudiconv/parser.py index d47bc099..d53d3dad 100644 --- a/heudiconv/parser.py +++ b/heudiconv/parser.py @@ -10,7 +10,6 @@ import shutil import typing from itertools import chain -from pathlib import Path from .dicoms import group_dicoms_into_seqinfos from .utils import ( @@ -63,85 +62,87 @@ def find_files(regex, topdir=op.curdir, exclude=None, yield path -def _get_files_in_dir(src: str) -> typing.List[str]: - return [str(f.resolve()) for f in Path(src).rglob("*") if f.is_file()] - - def get_extracted_dicoms( - fl: typing.Collection[str] + fl: typing.Iterable[str] ) -> typing.ItemsView[typing.Optional[int], typing.List[str]]: """Given a collection of files and/or directories, possibly extract some from tarballs. Parameters ---------- - fl : Collection[str] - A collection (e.g., list or tuple) of files that will be extracted. + fl : Iterable[str] + An iterable (e.g., list or tuple) of files to process. Returns ------- - ItemsView[int, list[str]] - A tuple of integer keys and list of strs representing absolute paths - of extracted files. + ItemsView[int | None, list[str]] + A tuple of keys (either integer or None) and list of strs representing + the absolute paths of files. Notes ----- - For 'classical' heudiconv, if multiple tarballs are provided, they + For 'classical' heudiconv, if multiple archives are provided, they correspond to different sessions, so here we would group into sessions - and return pairs `sessionid`, `files` with `sessionid` being None if no + and return pairs `sessionid`, `files` with `sessionid` being None if no "sessions" detected for that file or there was just a single tarball in the list. - When contents of fl are directories whose names do not have a suffix - that is recognized as a common archive format (e.g., .tgz), then the - corresponding item will contain a list of the files in that directory. - - When contents of fl appeart to be an unpackable archive, the contents are - extracted into utils.TempDirs(f'heudiconvDCM-{ses}') and the mode of all + When contents of fl appear to be an unpackable archive, the contents are + extracted into utils.TempDirs(f'heudiconvDCM') and the mode of all extracted files is set to 700. + + When contents of fl are a list of files, they are treated as a single + session. """ - # TODO: bring check back? - # if any(not tarfile.is_tarfile(i) for i in fl): - # raise ValueError("some but not all input files are tar files") + input_list_of_unpacked = any(fl) and all( + not t.endswith(_UNPACK_FORMATS) for t in fl + ) + + if not ( + input_list_of_unpacked or + all(t.endswith(_UNPACK_FORMATS) for t in fl) + ): + raise ValueError("Some but not all input files are archives.") + sessions: typing.DefaultDict[ typing.Optional[int], typing.List[str] ] = defaultdict(list) - if not isinstance(fl, (list, tuple)): + + if not isinstance(fl, list): fl = list(fl) - # keep track of session manually to ensure that the variable is bound - # when it is used after the loop (e.g., consider situation with - # fl being empty) - session = 0 - # needs sorting to keep the generated "session" label deterministic - for _, t in enumerate(sorted(fl)): - # Each file extracted must be associated with the proper session, - # but the high-level shutil does not have a way to list the files - # contained within each archive. So, files are temporarily - # extracted into unique tempdirs - # cannot use TempDirs since will trigger cleanup with __del__ - tmpdir = tempdirs(prefix="heudiconvDCM") - - if t.endswith(_UNPACK_FORMATS): + if input_list_of_unpacked: + sessions[None] = fl + else: + # keep track of session manually to ensure that the variable is bound + # when it is used after the loop (e.g., consider situation with + # fl being empty) + session = 0 + # needs sorting to keep the generated "session" label deterministic + for _, t in enumerate(sorted(fl)): + # Each file extracted must be associated with the proper session, + # but the high-level shutil does not have a way to list the files + # contained within each archive. So, files are temporarily + # extracted into unique tempdirs + # cannot use TempDirs since will trigger cleanup with __del__ + tmpdir = tempdirs(prefix="heudiconvDCM") + shutil.unpack_archive(t, extract_dir=tmpdir) - tf_content = _get_files_in_dir(tmpdir) + tf_content = list(find_files(regex=".*", topdir=tmpdir)) # check content and sanitize permission bits for f in tf_content: os.chmod(f, mode=0o700) - else: - tf_content = _get_files_in_dir(t) - - # store full paths to each file, so we don't need to drag along - # tmpdir as some basedir - sessions[session] = tf_content - session += 1 - - if session == 1: - # we had only 1 session, so not really multiple sessions according - # to classical 'heudiconv' assumptions, thus just move them all into - # None - sessions[None] += sessions.pop(0) + # store full paths to each file, so we don't need to drag along + # tmpdir as some basedir + sessions[session] = tf_content + session += 1 + + if session == 1: + # we had only 1 session (and at least 1), so not really multiple + # sessions according to classical 'heudiconv' assumptions, thus + # just move them all into None + sessions[None] += sessions.pop(0) return sessions.items() diff --git a/heudiconv/tests/test_archives.py b/heudiconv/tests/test_archives.py index 3d921d61..6e9c6b77 100644 --- a/heudiconv/tests/test_archives.py +++ b/heudiconv/tests/test_archives.py @@ -1,3 +1,4 @@ +from glob import glob import os import os.path as op from pathlib import Path @@ -34,8 +35,8 @@ def get_dicoms_zip(tmpdir: Path) -> typing.List[str]: @pytest.fixture -def get_dicoms_folder() -> typing.List[str]: - return [op.join(TESTS_DATA_PATH, "01-anat-scout")] +def get_dicoms_list() -> typing.List[str]: + return glob(op.join(TESTS_DATA_PATH, "01-anat-scout", "*")) def test_get_extracted_dicoms_single_session_is_none(get_dicoms_gztar: typing.List[str]): @@ -63,12 +64,23 @@ def test_get_extracted_dicoms_from_zip(get_dicoms_zip: typing.List[str]): assert all(file.endswith("0001.dcm") for file in files) -def test_get_extracted_dicoms_from_folder(get_dicoms_folder: typing.List[str]): - for _, files in get_extracted_dicoms(get_dicoms_folder): - # check that the only file is the one called "0001.dcm" - assert all(file.endswith("0001.dcm") for file in files) +def test_get_extracted_dicoms_from_file_list(get_dicoms_list: typing.List[str]): + for _, files in get_extracted_dicoms(get_dicoms_list): + assert all(op.isfile(file) for file in files) def test_get_extracted_have_correct_permissions(get_dicoms_gztar: typing.List[str]): for _, files in get_extracted_dicoms(get_dicoms_gztar): assert all(stat.S_IMODE(os.stat(file).st_mode) == 448 for file in files) + + +def test_get_extracted_are_absolute(get_dicoms_gztar: typing.List[str]): + for _, files in get_extracted_dicoms(get_dicoms_gztar): + assert all(op.isabs(file) for file in files) + + +def test_get_extracted_fails_when_mixing_archive_and_unarchived( + get_dicoms_gztar: typing.List[str], + get_dicoms_list: typing.List[str]): + with pytest.raises(ValueError): + get_extracted_dicoms(get_dicoms_gztar + get_dicoms_list) From a2965548bd7e26600d0889b10bdf0e4676eed44d Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Thu, 23 Mar 2023 14:09:10 -0400 Subject: [PATCH 116/176] add test for result of empty input --- heudiconv/tests/test_archives.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/heudiconv/tests/test_archives.py b/heudiconv/tests/test_archives.py index 6e9c6b77..6cd4c27a 100644 --- a/heudiconv/tests/test_archives.py +++ b/heudiconv/tests/test_archives.py @@ -84,3 +84,7 @@ def test_get_extracted_fails_when_mixing_archive_and_unarchived( get_dicoms_list: typing.List[str]): with pytest.raises(ValueError): get_extracted_dicoms(get_dicoms_gztar + get_dicoms_list) + + +def test_get_extracted_from_empty_list(): + assert not len(get_extracted_dicoms([])) From 495abc3666159686bdba335ea1facc11ae6df35b Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Wed, 29 Mar 2023 10:50:06 -0400 Subject: [PATCH 117/176] Replace py.path with pathlib --- heudiconv/cli/monitor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/heudiconv/cli/monitor.py b/heudiconv/cli/monitor.py index ba873460..e9e4be8a 100644 --- a/heudiconv/cli/monitor.py +++ b/heudiconv/cli/monitor.py @@ -2,6 +2,7 @@ import logging import os import os.path as op +from pathlib import Path import re import subprocess import time @@ -11,7 +12,6 @@ from datetime import date import inotify.adapters from inotify.constants import IN_MODIFY, IN_CREATE, IN_ISDIR -from py.path import local as localpath from tinydb import TinyDB _DEFAULT_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' @@ -67,9 +67,9 @@ def process(paths2process, db, wait=WAIT_TIME, logdir='log'): process_dict.update(run_dict) db.insert(process_dict) # save log - logdir = localpath(logdir) - log = logdir.join(process_dict['accession_number'] + '.log') - log.write(stdout) + logdir = Path(logdir) + log = logdir / (process_dict['accession_number'] + '.log') + log.write_text(stdout) # if we processed it, or it failed, # we need to remove it to avoid running it again processed.append(path) From b76d432cd01bf4c34f4a08d02b772d07e1838b7f Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Wed, 29 Mar 2023 10:54:50 -0400 Subject: [PATCH 118/176] Remove use of `six` --- heudiconv/info.py | 1 - heudiconv/tests/test_heuristics.py | 2 +- heudiconv/tests/test_main.py | 2 +- heudiconv/tests/test_tarballs.py | 1 - 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/heudiconv/info.py b/heudiconv/info.py index 8e354823..e0df20ae 100644 --- a/heudiconv/info.py +++ b/heudiconv/info.py @@ -33,7 +33,6 @@ ] TESTS_REQUIRES = [ - 'six', 'pytest', 'mock', 'tinydb', diff --git a/heudiconv/tests/test_heuristics.py b/heudiconv/tests/test_heuristics.py index 906e15f7..0e98ac01 100644 --- a/heudiconv/tests/test_heuristics.py +++ b/heudiconv/tests/test_heuristics.py @@ -1,9 +1,9 @@ from heudiconv.cli.run import main as runner +from io import StringIO import os import os.path as op from mock import patch -from six.moves import StringIO from glob import glob from os.path import join as pjoin, dirname diff --git a/heudiconv/tests/test_main.py b/heudiconv/tests/test_main.py index c67ff899..b7e5f242 100644 --- a/heudiconv/tests/test_main.py +++ b/heudiconv/tests/test_main.py @@ -24,9 +24,9 @@ import pytest import sys +from io import StringIO from mock import patch from os.path import join as opj -from six.moves import StringIO import stat import os.path as op diff --git a/heudiconv/tests/test_tarballs.py b/heudiconv/tests/test_tarballs.py index fbde536d..29aaad30 100644 --- a/heudiconv/tests/test_tarballs.py +++ b/heudiconv/tests/test_tarballs.py @@ -6,7 +6,6 @@ from mock import patch from os.path import join as opj from os.path import dirname -from six.moves import StringIO from glob import glob from heudiconv.dicoms import compress_dicoms From fed0cfb77aa99d945fae117b8199b30c0925a256 Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Wed, 29 Mar 2023 18:16:25 -0400 Subject: [PATCH 119/176] simplify aggregation of unpack formats Co-authored-by: Yaroslav Halchenko --- heudiconv/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/parser.py b/heudiconv/parser.py index d53d3dad..aea9b2c0 100644 --- a/heudiconv/parser.py +++ b/heudiconv/parser.py @@ -25,7 +25,7 @@ _VCS_REGEX = r'%s\.(?:git|gitattributes|svn|bzr|hg)(?:%s|$)' % (op.sep, op.sep) -_UNPACK_FORMATS = tuple(chain.from_iterable([x[1] for x in shutil.get_unpack_formats()])) +_UNPACK_FORMATS = sum((x[1] for x in shutil.get_unpack_formats()), []) @docstring_parameter(_VCS_REGEX) def find_files(regex, topdir=op.curdir, exclude=None, From 1c21dbcfb9350ac3b644f2b9c84063702b47027c Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Wed, 29 Mar 2023 18:24:42 -0400 Subject: [PATCH 120/176] consolidate fixtures Co-authored-by: Yaroslav Halchenko --- heudiconv/tests/test_archives.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/heudiconv/tests/test_archives.py b/heudiconv/tests/test_archives.py index 6cd4c27a..46610097 100644 --- a/heudiconv/tests/test_archives.py +++ b/heudiconv/tests/test_archives.py @@ -12,26 +12,22 @@ from ..parser import get_extracted_dicoms -@pytest.fixture -def get_dicoms_gztar(tmpdir: Path) -> typing.List[str]: +def get_dicoms_archive(format: str, tmpdir: Path) -> typing.List[str]: tmp_file = tmpdir / "dicom" archive = shutil.make_archive( str(tmp_file), - format="gztar", + format=format, root_dir=TESTS_DATA_PATH, base_dir="01-anat-scout") return [archive] +@pytest.fixture +def get_dicoms_gztar(tmpdir: Path) -> typing.List[str]: + return get_dicoms_archive("gztar", tmpdir) @pytest.fixture def get_dicoms_zip(tmpdir: Path) -> typing.List[str]: - tmp_file = tmpdir / "dicom" - archive = shutil.make_archive( - str(tmp_file), - format="zip", - root_dir=TESTS_DATA_PATH, - base_dir="01-anat-scout") - return [archive] + return get_dicoms_archive("zip", tmpdir) @pytest.fixture From fd15602a59acc598720c02d84747f01c493b7ff9 Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Wed, 29 Mar 2023 18:26:02 -0400 Subject: [PATCH 121/176] tidying Co-authored-by: Yaroslav Halchenko --- heudiconv/tests/test_archives.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/heudiconv/tests/test_archives.py b/heudiconv/tests/test_archives.py index 46610097..db9a5bd7 100644 --- a/heudiconv/tests/test_archives.py +++ b/heudiconv/tests/test_archives.py @@ -41,9 +41,10 @@ def test_get_extracted_dicoms_single_session_is_none(get_dicoms_gztar: typing.Li def test_get_extracted_dicoms_multple_session_integers(get_dicoms_gztar: typing.List[str]): - sessions = [] - for session, _ in get_extracted_dicoms(get_dicoms_gztar + get_dicoms_gztar): - sessions.append(session) + sessions = [ + session + for session, _ in get_extracted_dicoms(get_dicoms_gztar + get_dicoms_gztar) + ] assert sessions == [0, 1] From bca7986c40d1afd2770d3b78cfef5bf20f6d0488 Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Wed, 29 Mar 2023 18:27:35 -0400 Subject: [PATCH 122/176] being more consistent Co-authored-by: Yaroslav Halchenko --- heudiconv/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/parser.py b/heudiconv/parser.py index aea9b2c0..7eee6369 100644 --- a/heudiconv/parser.py +++ b/heudiconv/parser.py @@ -101,7 +101,7 @@ def get_extracted_dicoms( if not ( input_list_of_unpacked or all(t.endswith(_UNPACK_FORMATS) for t in fl) - ): + ): raise ValueError("Some but not all input files are archives.") sessions: typing.DefaultDict[ From 4743650ba73728ab1d2ebd549ab40eac11f01aae Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Wed, 29 Mar 2023 18:42:00 -0400 Subject: [PATCH 123/176] remove duplicated typing information Co-authored-by: Yaroslav Halchenko --- heudiconv/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/parser.py b/heudiconv/parser.py index 7eee6369..ee62b649 100644 --- a/heudiconv/parser.py +++ b/heudiconv/parser.py @@ -70,7 +70,7 @@ def get_extracted_dicoms( Parameters ---------- - fl : Iterable[str] + fl An iterable (e.g., list or tuple) of files to process. Returns From b8c37bf241d0c84000eccc750e62536f060e5d1c Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Wed, 29 Mar 2023 18:37:31 -0400 Subject: [PATCH 124/176] restore ability to mix unarchived & archived files --- heudiconv/parser.py | 85 +++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/heudiconv/parser.py b/heudiconv/parser.py index ee62b649..49e1fdff 100644 --- a/heudiconv/parser.py +++ b/heudiconv/parser.py @@ -9,7 +9,6 @@ import shutil import typing -from itertools import chain from .dicoms import group_dicoms_into_seqinfos from .utils import ( @@ -25,7 +24,7 @@ _VCS_REGEX = r'%s\.(?:git|gitattributes|svn|bzr|hg)(?:%s|$)' % (op.sep, op.sep) -_UNPACK_FORMATS = sum((x[1] for x in shutil.get_unpack_formats()), []) +_UNPACK_FORMATS = tuple(sum((x[1] for x in shutil.get_unpack_formats()), [])) @docstring_parameter(_VCS_REGEX) def find_files(regex, topdir=op.curdir, exclude=None, @@ -91,19 +90,13 @@ def get_extracted_dicoms( extracted into utils.TempDirs(f'heudiconvDCM') and the mode of all extracted files is set to 700. - When contents of fl are a list of files, they are treated as a single - session. + When contents of fl are a list of unarchived files, they are treated as + a single session. + + When contents of fl are a list of unarchived and archived files, the + unarchived files are grouped into a single session (key: None), and the + archived files are each grouped into separate sessions. """ - input_list_of_unpacked = any(fl) and all( - not t.endswith(_UNPACK_FORMATS) for t in fl - ) - - if not ( - input_list_of_unpacked or - all(t.endswith(_UNPACK_FORMATS) for t in fl) - ): - raise ValueError("Some but not all input files are archives.") - sessions: typing.DefaultDict[ typing.Optional[int], typing.List[str] @@ -112,37 +105,39 @@ def get_extracted_dicoms( if not isinstance(fl, list): fl = list(fl) - if input_list_of_unpacked: - sessions[None] = fl - else: - # keep track of session manually to ensure that the variable is bound - # when it is used after the loop (e.g., consider situation with - # fl being empty) - session = 0 - # needs sorting to keep the generated "session" label deterministic - for _, t in enumerate(sorted(fl)): - # Each file extracted must be associated with the proper session, - # but the high-level shutil does not have a way to list the files - # contained within each archive. So, files are temporarily - # extracted into unique tempdirs - # cannot use TempDirs since will trigger cleanup with __del__ - tmpdir = tempdirs(prefix="heudiconvDCM") - - shutil.unpack_archive(t, extract_dir=tmpdir) - tf_content = list(find_files(regex=".*", topdir=tmpdir)) - # check content and sanitize permission bits - for f in tf_content: - os.chmod(f, mode=0o700) - # store full paths to each file, so we don't need to drag along - # tmpdir as some basedir - sessions[session] = tf_content - session += 1 - - if session == 1: - # we had only 1 session (and at least 1), so not really multiple - # sessions according to classical 'heudiconv' assumptions, thus - # just move them all into None - sessions[None] += sessions.pop(0) + # keep track of session manually to ensure that the variable is bound + # when it is used after the loop (e.g., consider situation with + # fl being empty) + session = 0 + # needs sorting to keep the generated "session" label deterministic + for _, t in enumerate(sorted(fl)): + + if not t.endswith(_UNPACK_FORMATS): + sessions[None].append(t) + continue + + # Each file extracted must be associated with the proper session, + # but the high-level shutil does not have a way to list the files + # contained within each archive. So, files are temporarily + # extracted into unique tempdirs + # cannot use TempDirs since will trigger cleanup with __del__ + tmpdir = tempdirs(prefix="heudiconvDCM") + + shutil.unpack_archive(t, extract_dir=tmpdir) + tf_content = list(find_files(regex=".*", topdir=tmpdir)) + # check content and sanitize permission bits + for f in tf_content: + os.chmod(f, mode=0o700) + # store full paths to each file, so we don't need to drag along + # tmpdir as some basedir + sessions[session] = tf_content + session += 1 + + if session == 1: + # we had only 1 session (and at least 1), so not really multiple + # sessions according to classical 'heudiconv' assumptions, thus + # just move them all into None + sessions[None] += sessions.pop(0) return sessions.items() From 254dce7f996760dc50d890530a9b028e6851a36c Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Wed, 29 Mar 2023 20:20:14 -0400 Subject: [PATCH 125/176] parameterize test for archive format --- heudiconv/parser.py | 41 +++++++++++----------- heudiconv/tests/test_archives.py | 60 ++++++++++++++------------------ 2 files changed, 47 insertions(+), 54 deletions(-) diff --git a/heudiconv/parser.py b/heudiconv/parser.py index 49e1fdff..2bd6171a 100644 --- a/heudiconv/parser.py +++ b/heudiconv/parser.py @@ -8,7 +8,7 @@ from collections import defaultdict import shutil -import typing +from typing import DefaultDict, ItemsView, Iterable, List, Optional from .dicoms import group_dicoms_into_seqinfos from .utils import ( @@ -62,21 +62,20 @@ def find_files(regex, topdir=op.curdir, exclude=None, def get_extracted_dicoms( - fl: typing.Iterable[str] - ) -> typing.ItemsView[typing.Optional[int], typing.List[str]]: - """Given a collection of files and/or directories, possibly extract - some from tarballs. + fl: Iterable[str] + ) -> ItemsView[Optional[int], List[str]]: + """Given a collection of files and/or directories, list out and possibly + extract the contents from archives. Parameters ---------- fl - An iterable (e.g., list or tuple) of files to process. + Files (possibly archived) to process. Returns ------- ItemsView[int | None, list[str]] - A tuple of keys (either integer or None) and list of strs representing - the absolute paths of files. + The absolute paths of (possibly newly extracted) files. Notes ----- @@ -94,17 +93,13 @@ def get_extracted_dicoms( a single session. When contents of fl are a list of unarchived and archived files, the - unarchived files are grouped into a single session (key: None), and the - archived files are each grouped into separate sessions. + unarchived files are grouped into a single session (key: None). If there is + only one archived file, the contents of that file are grouped with + the unarchived file. If there are multiple archived files, they are grouped + into separate sessions. """ - sessions: typing.DefaultDict[ - typing.Optional[int], - typing.List[str] - ] = defaultdict(list) + sessions: DefaultDict[Optional[int], List[str]] = defaultdict(list) - if not isinstance(fl, list): - fl = list(fl) - # keep track of session manually to ensure that the variable is bound # when it is used after the loop (e.g., consider situation with # fl being empty) @@ -123,14 +118,18 @@ def get_extracted_dicoms( # cannot use TempDirs since will trigger cleanup with __del__ tmpdir = tempdirs(prefix="heudiconvDCM") + # check content and sanitize permission bits before extraction + os.chmod(tmpdir, mode=0o700) shutil.unpack_archive(t, extract_dir=tmpdir) - tf_content = list(find_files(regex=".*", topdir=tmpdir)) - # check content and sanitize permission bits - for f in tf_content: + + archive_content = list(find_files(regex=".*", topdir=tmpdir)) + + # may be too cautious (tmpdir is already 700). + for f in archive_content: os.chmod(f, mode=0o700) # store full paths to each file, so we don't need to drag along # tmpdir as some basedir - sessions[session] = tf_content + sessions[session] = archive_content session += 1 if session == 1: diff --git a/heudiconv/tests/test_archives.py b/heudiconv/tests/test_archives.py index db9a5bd7..b93841d9 100644 --- a/heudiconv/tests/test_archives.py +++ b/heudiconv/tests/test_archives.py @@ -3,7 +3,7 @@ import os.path as op from pathlib import Path import stat -import typing +from typing import List import pytest @@ -12,35 +12,37 @@ from ..parser import get_extracted_dicoms -def get_dicoms_archive(format: str, tmpdir: Path) -> typing.List[str]: +def _get_dicoms_archive(tmpdir: Path, format: str) -> List[str]: tmp_file = tmpdir / "dicom" archive = shutil.make_archive( str(tmp_file), - format=format, + format=format, root_dir=TESTS_DATA_PATH, base_dir="01-anat-scout") return [archive] + @pytest.fixture -def get_dicoms_gztar(tmpdir: Path) -> typing.List[str]: - return get_dicoms_archive("gztar", tmpdir) +def get_dicoms_archive(tmpdir: Path, request: pytest.FixtureRequest) -> List[str]: + return _get_dicoms_archive(tmpdir, format=request.param) + @pytest.fixture -def get_dicoms_zip(tmpdir: Path) -> typing.List[str]: - return get_dicoms_archive("zip", tmpdir) +def get_dicoms_gztar(tmpdir: Path) -> List[str]: + return _get_dicoms_archive(tmpdir, "gztar") @pytest.fixture -def get_dicoms_list() -> typing.List[str]: +def get_dicoms_list() -> List[str]: return glob(op.join(TESTS_DATA_PATH, "01-anat-scout", "*")) -def test_get_extracted_dicoms_single_session_is_none(get_dicoms_gztar: typing.List[str]): +def test_get_extracted_dicoms_single_session_is_none(get_dicoms_gztar: List[str]): for session_, _ in get_extracted_dicoms(get_dicoms_gztar): assert session_ is None -def test_get_extracted_dicoms_multple_session_integers(get_dicoms_gztar: typing.List[str]): +def test_get_extracted_dicoms_multple_session_integers(get_dicoms_gztar: List[str]): sessions = [ session for session, _ in get_extracted_dicoms(get_dicoms_gztar + get_dicoms_gztar) @@ -49,38 +51,30 @@ def test_get_extracted_dicoms_multple_session_integers(get_dicoms_gztar: typing. assert sessions == [0, 1] -def test_get_extracted_dicoms_from_tgz(get_dicoms_gztar: typing.List[str]): - for _, files in get_extracted_dicoms(get_dicoms_gztar): - # check that the only file is the one called "0001.dcm" - assert all(file.endswith("0001.dcm") for file in files) +@pytest.mark.parametrize("get_dicoms_archive", ("tar", "gztar", "zip", "bztar", "xztar"), indirect=True) +def test_get_extracted_dicoms_from_archives(get_dicoms_archive: List[str]): + for _, files in get_extracted_dicoms(get_dicoms_archive): - -def test_get_extracted_dicoms_from_zip(get_dicoms_zip: typing.List[str]): - for _, files in get_extracted_dicoms(get_dicoms_zip): # check that the only file is the one called "0001.dcm" - assert all(file.endswith("0001.dcm") for file in files) - + endswith = all(file.endswith("0001.dcm") for file in files) -def test_get_extracted_dicoms_from_file_list(get_dicoms_list: typing.List[str]): - for _, files in get_extracted_dicoms(get_dicoms_list): - assert all(op.isfile(file) for file in files) + # check that permissions were set + mode = all(stat.S_IMODE(os.stat(file).st_mode) == 448 for file in files) + # check for absolute paths + absolute = all(op.isabs(file) for file in files) -def test_get_extracted_have_correct_permissions(get_dicoms_gztar: typing.List[str]): - for _, files in get_extracted_dicoms(get_dicoms_gztar): - assert all(stat.S_IMODE(os.stat(file).st_mode) == 448 for file in files) + assert endswith and mode and absolute -def test_get_extracted_are_absolute(get_dicoms_gztar: typing.List[str]): - for _, files in get_extracted_dicoms(get_dicoms_gztar): - assert all(op.isabs(file) for file in files) +def test_get_extracted_dicoms_from_file_list(get_dicoms_list: List[str]): + for _, files in get_extracted_dicoms(get_dicoms_list): + assert all(op.isfile(file) for file in files) -def test_get_extracted_fails_when_mixing_archive_and_unarchived( - get_dicoms_gztar: typing.List[str], - get_dicoms_list: typing.List[str]): - with pytest.raises(ValueError): - get_extracted_dicoms(get_dicoms_gztar + get_dicoms_list) +def test_get_extracted_dicoms_from_mixed_list(get_dicoms_list: List[str], get_dicoms_gztar: List[str]): + for _, files in get_extracted_dicoms(get_dicoms_list + get_dicoms_gztar): + assert all(op.isfile(file) for file in files) def test_get_extracted_from_empty_list(): From 99b0bf6109732c365721250fd7843c70a74002c1 Mon Sep 17 00:00:00 2001 From: Patrick Sadil Date: Wed, 29 Mar 2023 20:59:45 -0400 Subject: [PATCH 126/176] ensure test does not short-circuit --- heudiconv/tests/test_archives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/tests/test_archives.py b/heudiconv/tests/test_archives.py index b93841d9..e4133098 100644 --- a/heudiconv/tests/test_archives.py +++ b/heudiconv/tests/test_archives.py @@ -64,7 +64,7 @@ def test_get_extracted_dicoms_from_archives(get_dicoms_archive: List[str]): # check for absolute paths absolute = all(op.isabs(file) for file in files) - assert endswith and mode and absolute + assert all([endswith, mode, absolute]) def test_get_extracted_dicoms_from_file_list(get_dicoms_list: List[str]): From 8e2b8b381730317b958b678d41e6b30063c41255 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Mon, 10 Apr 2023 11:12:43 -0400 Subject: [PATCH 127/176] Improve pytest config --- .coveragerc | 3 --- .gitignore | 9 +++++---- heudiconv/bids.py | 10 +++++----- heudiconv/parser.py | 2 +- pytest.ini | 3 --- tox.ini | 29 ++++++++++++++++++++++++++--- 6 files changed, 37 insertions(+), 19 deletions(-) delete mode 100644 .coveragerc mode change 100755 => 100644 .gitignore delete mode 100644 pytest.ini diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index f14665ff..00000000 --- a/.coveragerc +++ /dev/null @@ -1,3 +0,0 @@ -[run] -include = heudiconv/* - setup.py diff --git a/.gitignore b/.gitignore old mode 100755 new mode 100644 index d354a5e0..3ba4380e --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,12 @@ +*.egg-info/ *.pyc .cache/ .coverage -*.egg-info/ .idea/ -venvs/ +.tox/ +.vscode/ _build/ +_version.py build/ dist/ -.vscode/ -_version.py +venvs/ diff --git a/heudiconv/bids.py b/heudiconv/bids.py index 82c77c1d..710ae488 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -165,10 +165,10 @@ def populate_bids_templates(path, defaults={}): def populate_aggregated_jsons(path): - """Aggregate across the entire BIDS dataset ``.json``\s into top level ``.json``\s + """Aggregate across the entire BIDS dataset ``.json``\\s into top level ``.json``\\s Top level .json files would contain only the fields which are - common to all ``subject[/session]/type/*_modality.json``\s. + common to all ``subject[/session]/type/*_modality.json``\\s. ATM aggregating only for ``*_task*_bold.json`` files. Only the task- and OPTIONAL _acq- field is retained within the aggregated filename. The other @@ -184,16 +184,16 @@ def populate_aggregated_jsons(path): # way too many -- let's just collect all which are the same! # FIELDS_TO_TRACK = {'RepetitionTime', 'FlipAngle', 'EchoTime', # 'Manufacturer', 'SliceTiming', ''} - for fpath in find_files('.*_task-.*\_bold\.json', + for fpath in find_files(r'.*_task-.*\_bold\.json', topdir=glob(op.join(path, 'sub-*')), exclude_vcs=True, - exclude="/\.(datalad|heudiconv)/"): + exclude=r"/\.(datalad|heudiconv)/"): # # According to BIDS spec I think both _task AND _acq (may be more? # _rec, _dir, ...?) should be retained? # TODO: if we are to fix it, then old ones (without _acq) should be # removed first - task = re.sub('.*_(task-[^_\.]*(_acq-[^_\.]*)?)_.*', r'\1', fpath) + task = re.sub(r'.*_(task-[^_\.]*(_acq-[^_\.]*)?)_.*', r'\1', fpath) json_ = load_json(fpath, retry=100) if task not in tasks: tasks[task] = json_ diff --git a/heudiconv/parser.py b/heudiconv/parser.py index 2bd6171a..997c7817 100644 --- a/heudiconv/parser.py +++ b/heudiconv/parser.py @@ -226,7 +226,7 @@ def get_study_sessions(dicom_dir_template, files_opt, heuristic, outdir, raise NotImplementedError("Cannot guarantee subject id - add " "`infotoids` to heuristic file or " "provide `--subjects` option") - lgr.warn("Heuristic is missing an `infotoids` method, assigning " + lgr.warning("Heuristic is missing an `infotoids` method, assigning " "empty method and using provided subject id %s. " "Provide `session` and `locator` fields for best results." , sid) diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 58f538f6..00000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -# monitor.py requires optional linotify, but would blow tests discovery, does not contain tests within -addopts = --doctest-modules --tb=short --ignore heudiconv/cli/monitor.py diff --git a/tox.ini b/tox.ini index 5f8f5fcd..8b19ee22 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,29 @@ [tox] -envlist = py27,py33,py34,py35 +envlist = py3 [testenv] -commands = python -m pytest -s -v {posargs} tests -deps = -r{toxinidir}/dev-requirements.txt +extras = all +commands = pytest -s -v {posargs} heudiconv + +[pytest] +# monitor.py requires optional linotify, but would blow tests discovery, does not contain tests within +addopts = --doctest-modules --tb=short --ignore heudiconv/cli/monitor.py +filterwarnings = + error + # + ignore:module 'sre_.*' is deprecated:DeprecationWarning:traits + # pytest generates a number of inscrutable warnings about open files never + # being closed. I (jwodder) expect these are due to DataLad not shutting + # down batch processes prior to garbage collection. + ignore::pytest.PytestUnraisableExceptionWarning + # I don't understand why this warning occurs, as we're using six 1.16, + # which has the named method. + ignore:_SixMetaPathImporter.find_spec\(\) not found:ImportWarning + # + ignore:.*pkg_resources:DeprecationWarning + # + ignore:Use setlocale\(\), getencoding\(\) and getlocale\(\) instead:DeprecationWarning:nipype + +[coverage:run] +include = heudiconv/* + setup.py From c2fc71bb3fece7936ac95877f71c2c7a392c5130 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Mon, 10 Apr 2023 08:34:01 -0400 Subject: [PATCH 128/176] Add pre-commit config --- .codespellrc | 2 +- .pre-commit-config.yaml | 34 ++++++++++++++++++++++++++++++++++ tox.ini | 18 ++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 .pre-commit-config.yaml diff --git a/.codespellrc b/.codespellrc index 4757ec97..4d5c4658 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,4 +1,4 @@ [codespell] skip = .git,.venv,venvs,*.svg,_build # te -- TE as codespell is case insensitive -ignore-words-list = te +ignore-words-list = bu,nd,te diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..029e041b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-json + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/codespell-project/codespell + rev: v2.2.4 + hooks: + - id: codespell + + - repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black + + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + + - repo: https://github.com/PyCQA/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + additional_dependencies: + - flake8-bugbear + - flake8-builtins + - flake8-unused-arguments diff --git a/tox.ini b/tox.ini index 8b19ee22..75cba5f7 100644 --- a/tox.ini +++ b/tox.ini @@ -27,3 +27,21 @@ filterwarnings = [coverage:run] include = heudiconv/* setup.py + +[flake8] +doctests = True +exclude = .*/,build/,dist/,test/data,venv/ +hang-closing = False +unused-arguments-ignore-stub-functions = True +select = A,B,B902,C,E,E242,F,U100,W +ignore = A003,B005,E203,E262,E266,E501,W503 + +[isort] +atomic = True +force_sort_within_sections = True +honor_noqa = True +lines_between_sections = 1 +profile = black +reverse_relative = True +sort_relative_in_force_sorted_sections = True +known_first_party = heudiconv From 3c7dfa25f6e657a0a8da2a37cce9f176255ecbd8 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Mon, 10 Apr 2023 08:57:13 -0400 Subject: [PATCH 129/176] Add lint job --- .github/workflows/lint.yml | 27 +++++++++++++++++++++++++++ tox.ini | 12 +++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..54e6b5b5 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,27 @@ +name: Linters + +on: + - push + - pull_request + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Set up environment + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.7' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade tox + + - name: Run linters + run: tox -e lint diff --git a/tox.ini b/tox.ini index 75cba5f7..e27b954c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py3 +envlist = lint,py3 [testenv] extras = all @@ -28,6 +28,16 @@ filterwarnings = include = heudiconv/* setup.py +[testenv:lint] +skip_install = True +deps = + flake8 + flake8-bugbear + flake8-builtins + flake8-unused-arguments +commands = + flake8 heudiconv + [flake8] doctests = True exclude = .*/,build/,dist/,test/data,venv/ From 591d1a92d252a169976d23bdee9b737e43a4b662 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Mon, 10 Apr 2023 08:54:56 -0400 Subject: [PATCH 130/176] Apply pre-commit --- .github/ISSUE_TEMPLATE.md | 4 +- README.rst | 2 +- custom/dbic/singularity-env.def | 2 +- docs/Makefile | 2 +- docs/conf.py | 62 +- docs/installation.rst | 1 - docs/tutorials.rst | 2 +- heudiconv/__init__.py | 9 +- heudiconv/bids.py | 661 ++++++----- heudiconv/cli/monitor.py | 138 ++- heudiconv/cli/run.py | 280 +++-- heudiconv/convert.py | 566 ++++++---- heudiconv/dicoms.py | 281 +++-- heudiconv/due.py | 21 +- heudiconv/external/dcmstack.py | 10 +- heudiconv/external/dlad.py | 139 +-- heudiconv/external/pydicom.py | 2 + heudiconv/external/tests/test_dlad.py | 28 +- heudiconv/heuristics/banda-bids.py | 140 ++- heudiconv/heuristics/bids_ME.py | 19 +- heudiconv/heuristics/bids_PhoenixReport.py | 34 +- heudiconv/heuristics/bids_with_ses.py | 83 +- heudiconv/heuristics/cmrr_heuristic.py | 120 +- heudiconv/heuristics/convertall.py | 10 +- heudiconv/heuristics/example.py | 115 +- heudiconv/heuristics/multires_7Tbold.py | 80 +- heudiconv/heuristics/reproin.py | 375 ++++--- heudiconv/heuristics/reproin_validator.cfg | 11 +- heudiconv/heuristics/studyforrest_phase2.py | 45 +- heudiconv/heuristics/test_b0dwi_for_fmap.py | 11 +- heudiconv/heuristics/test_reproin.py | 258 ++--- heudiconv/heuristics/uc_bids.py | 108 +- heudiconv/info.py | 52 +- heudiconv/main.py | 253 +++-- heudiconv/parser.py | 127 ++- heudiconv/queue.py | 26 +- heudiconv/tests/anonymize_script.py | 6 +- heudiconv/tests/conftest.py | 11 +- heudiconv/tests/test_archives.py | 23 +- heudiconv/tests/test_bids.py | 1114 +++++++++++-------- heudiconv/tests/test_convert.py | 187 ++-- heudiconv/tests/test_dicoms.py | 117 +- heudiconv/tests/test_heuristics.py | 171 +-- heudiconv/tests/test_main.py | 258 ++--- heudiconv/tests/test_monitor.py | 84 +- heudiconv/tests/test_queue.py | 100 +- heudiconv/tests/test_regression.py | 83 +- heudiconv/tests/test_tarballs.py | 17 +- heudiconv/tests/test_utils.py | 114 +- heudiconv/tests/utils.py | 48 +- heudiconv/utils.py | 229 ++-- setup.py | 58 +- 52 files changed, 3815 insertions(+), 2882 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 184824c5..7da05c45 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,4 +1,4 @@ - @@ -20,5 +20,5 @@ Choose one: - [ ] Container -- Heudiconv version: +- Heudiconv version: diff --git a/README.rst b/README.rst index 1d4c1cad..67cc861a 100644 --- a/README.rst +++ b/README.rst @@ -54,7 +54,7 @@ into structured directory layouts. Installation ------------ -See our `installation page `_ +See our `installation page `_ on heudiconv.readthedocs.io . HOWTO 101 diff --git a/custom/dbic/singularity-env.def b/custom/dbic/singularity-env.def index a7722e4a..b8fe72b3 100644 --- a/custom/dbic/singularity-env.def +++ b/custom/dbic/singularity-env.def @@ -37,7 +37,7 @@ MirrorURL: http://ftp.us.debian.org/debian/ %post echo "Configuring the environment" apt-get update - apt-get -y install eatmydata + apt-get -y install eatmydata eatmydata apt-get -y install vim wget strace time ncdu gnupg curl procps wget -q -O/tmp/nd-configurerepo https://raw.githubusercontent.com/neurodebian/neurodebian/4d26c8f30433145009aa3f74516da12f560a5a13/tools/nd-configurerepo bash /tmp/nd-configurerepo diff --git a/docs/Makefile b/docs/Makefile index 298ea9e2..51285967 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -16,4 +16,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py index b5d762f7..d2f47807 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,12 +21,12 @@ from heudiconv import __version__ -project = 'heudiconv' -copyright = '2014-2022, Heudiconv team' -author = 'Heudiconv team' +project = "heudiconv" +copyright = "2014-2022, Heudiconv team" # noqa: A001 +author = "Heudiconv team" # The short X.Y version -version = '.'.join(__version__.split('.')[:2]) +version = ".".join(__version__.split(".")[:2]) # The full version, including alpha/beta/rc tags release = __version__ @@ -41,34 +41,34 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinxarg.ext', - 'sphinx.ext.napoleon', + "sphinx.ext.autodoc", + "sphinxarg.ext", + "sphinx.ext.napoleon", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -source_suffix = ['.rst', '.md'] +source_suffix = [".rst", ".md"] # source_suffix = '.rst' # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = 'en' +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None @@ -79,7 +79,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -106,7 +106,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'heudiconvdoc' +htmlhelp_basename = "heudiconvdoc" # -- Options for LaTeX output ------------------------------------------------ @@ -115,15 +115,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -133,8 +130,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'heudiconv.tex', 'heudiconv Documentation', - 'Heudiconv team', 'manual'), + ( + master_doc, + "heudiconv.tex", + "heudiconv Documentation", + "Heudiconv team", + "manual", + ), ] @@ -142,10 +144,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'heudiconv', 'heudiconv Documentation', - [author], 1) -] +man_pages = [(master_doc, "heudiconv", "heudiconv Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -154,9 +153,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'heudiconv', 'heudiconv Documentation', - author, 'heudiconv', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "heudiconv", + "heudiconv Documentation", + author, + "heudiconv", + "One line description of project.", + "Miscellaneous", + ), ] @@ -175,9 +180,8 @@ # epub_uid = '' # A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] +epub_exclude_files = ["search.html"] # -- Extension configuration ------------------------------------------------- -autodoc_default_options={ - 'members': None} +autodoc_default_options = {"members": None} diff --git a/docs/installation.rst b/docs/installation.rst index 921233ea..1b9599ee 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -64,4 +64,3 @@ the documentation of this collection. **Note:** With the ``datalad containers-run`` command, the images in this collection work on macOS (OSX) as well for ``repronim/containers`` helpers automagically take care of running the Singularity containers via Docker. - diff --git a/docs/tutorials.rst b/docs/tutorials.rst index 6074c5c4..ec12e36a 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -22,5 +22,5 @@ other users' tutorials covering their experience with ``heudiconv``. of a ReproNim Webinar on ``heudiconv``. .. caution:: - Some of these tutorials may not be up to date with + Some of these tutorials may not be up to date with the latest releases of ``heudiconv``. diff --git a/heudiconv/__init__.py b/heudiconv/__init__.py index a1f7a78a..4015e511 100644 --- a/heudiconv/__init__.py +++ b/heudiconv/__init__.py @@ -1,13 +1,16 @@ # set logger handler import logging import os -from .info import __packagename__ + from ._version import __version__ +from .info import __packagename__ + +__all__ = ["__packagename__", "__version__"] # Rudimentary logging support. lgr = logging.getLogger(__name__) logging.basicConfig( - format='%(levelname)s: %(message)s', - level=getattr(logging, os.environ.get('HEUDICONV_LOG_LEVEL', 'INFO')) + format="%(levelname)s: %(message)s", + level=getattr(logging, os.environ.get("HEUDICONV_LOG_LEVEL", "INFO")), ) lgr.debug("Starting the abomination") # just to "run-test" logging diff --git a/heudiconv/bids.py b/heudiconv/bids.py index 710ae488..48eef1df 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -2,57 +2,63 @@ __docformat__ = "numpy" +from collections import OrderedDict +import csv +from datetime import datetime +import errno +from glob import glob import hashlib +import logging import os import os.path as op -import logging -import numpy as np import re -from collections import OrderedDict -from datetime import datetime -import csv -from random import sample -from glob import glob -import errno import typing import warnings -from .external.pydicom import dcm +import numpy as np +from . import __version__, dicoms +from .external.pydicom import dcm from .parser import find_files from .utils import ( - load_json, - save_json, create_file_if_missing, - json_dumps, - update_json, - set_readonly, is_readonly, - get_datetime, - remove_suffix, + json_dumps, + load_json, remove_prefix, + remove_suffix, + save_json, + set_readonly, + update_json, ) -from . import __version__ -from . import dicoms lgr = logging.getLogger(__name__) # Fields to be populated in _scans files. Order matters -SCANS_FILE_FIELDS = OrderedDict([ - ("filename", OrderedDict([ - ("Description", "Name of the nifti file")])), - ("acq_time", OrderedDict([ - ("LongName", "Acquisition time"), - ("Description", "Acquisition time of the particular scan")])), - ("operator", OrderedDict([ - ("Description", "Name of the operator")])), - ("randstr", OrderedDict([ - ("LongName", "Random string"), - ("Description", "md5 hash of UIDs")])), -]) +SCANS_FILE_FIELDS = OrderedDict( + [ + ("filename", OrderedDict([("Description", "Name of the nifti file")])), + ( + "acq_time", + OrderedDict( + [ + ("LongName", "Acquisition time"), + ("Description", "Acquisition time of the particular scan"), + ] + ), + ), + ("operator", OrderedDict([("Description", "Name of the operator")])), + ( + "randstr", + OrderedDict( + [("LongName", "Random string"), ("Description", "md5 hash of UIDs")] + ), + ), + ] +) #: JSON Key where we will embed our version in the newly produced .json files -HEUDICONV_VERSION_JSON_KEY = 'HeudiconvVersion' +HEUDICONV_VERSION_JSON_KEY = "HeudiconvVersion" class BIDSError(Exception): @@ -62,13 +68,13 @@ class BIDSError(Exception): BIDS_VERSION = "1.4.1" # List defining allowed parameter matching for fmap assignment: -SHIM_KEY = 'ShimSetting' +SHIM_KEY = "ShimSetting" AllowedFmapParameterMatching = [ - 'Shims', - 'ImagingVolume', - 'ModalityAcquisitionLabel', - 'CustomAcquisitionLabel', - 'Force', + "Shims", + "ImagingVolume", + "ModalityAcquisitionLabel", + "CustomAcquisitionLabel", + "Force", ] # Key info returned by get_key_info_for_fmap_assignment when # matching_parameter = "Force" @@ -76,8 +82,8 @@ class BIDSError(Exception): # List defining allowed criteria to assign a given fmap to a non-fmap run # among the different fmaps with matching parameters: AllowedCriteriaForFmapAssignment = [ - 'First', - 'Closest', + "First", + "Closest", ] @@ -90,76 +96,104 @@ def maybe_na(val): if val is not None: val = str(val) val = val.strip() - return 'n/a' if (not val or val in ('N/A', 'NA')) else val + return "n/a" if (not val or val in ("N/A", "NA")) else val def treat_age(age): """Age might encounter 'Y' suffix or be a float""" age = str(age) - if age.endswith('M'): - age = age.rstrip('M') + if age.endswith("M"): + age = age.rstrip("M") age = float(age) / 12 - age = ('%.2f' if age != int(age) else '%d') % age + age = ("%.2f" if age != int(age) else "%d") % age else: - age = age.rstrip('Y') + age = age.rstrip("Y") if age: # strip all leading 0s but allow to scan a newborn (age 0Y) - age = '0' if not age.lstrip('0') else age.lstrip('0') - if age.startswith('.'): + age = "0" if not age.lstrip("0") else age.lstrip("0") + if age.startswith("."): # we had float point value, let's prepend 0 - age = '0' + age + age = "0" + age return age -def populate_bids_templates(path, defaults={}): +def populate_bids_templates(path, defaults=None): """Premake BIDS text files with templates""" lgr.info("Populating template files under %s", path) - descriptor = op.join(path, 'dataset_description.json') + descriptor = op.join(path, "dataset_description.json") + if defaults is None: + defaults = {} if not op.lexists(descriptor): - save_json(descriptor, - OrderedDict([ - ('Name', "TODO: name of the dataset"), - ('BIDSVersion', BIDS_VERSION), - ('License', defaults.get('License', - "TODO: choose a license, e.g. PDDL " - "(http://opendatacommons.org/licenses/pddl/)")), - ('Authors', defaults.get('Authors', - ["TODO:", "First1 Last1", "First2 Last2", "..."])), - ('Acknowledgements', defaults.get('Acknowledgements', - 'TODO: whom you want to acknowledge')), - ('HowToAcknowledge', + save_json( + descriptor, + OrderedDict( + [ + ("Name", "TODO: name of the dataset"), + ("BIDSVersion", BIDS_VERSION), + ( + "License", + defaults.get( + "License", + "TODO: choose a license, e.g. PDDL " + "(http://opendatacommons.org/licenses/pddl/)", + ), + ), + ( + "Authors", + defaults.get( + "Authors", ["TODO:", "First1 Last1", "First2 Last2", "..."] + ), + ), + ( + "Acknowledgements", + defaults.get( + "Acknowledgements", "TODO: whom you want to acknowledge" + ), + ), + ( + "HowToAcknowledge", "TODO: describe how to acknowledge -- either cite a " "corresponding paper, or just in acknowledgement " - "section"), - ('Funding', ["TODO", "GRANT #1", "GRANT #2"]), - ('ReferencesAndLinks', - ["TODO", "List of papers or websites"]), - ('DatasetDOI', 'TODO: eventually a DOI for the dataset') - ])) - sourcedata_README = op.join(path, 'sourcedata', 'README') + "section", + ), + ("Funding", ["TODO", "GRANT #1", "GRANT #2"]), + ("ReferencesAndLinks", ["TODO", "List of papers or websites"]), + ("DatasetDOI", "TODO: eventually a DOI for the dataset"), + ] + ), + ) + sourcedata_README = op.join(path, "sourcedata", "README") if op.exists(op.dirname(sourcedata_README)): - create_file_if_missing(sourcedata_README, - ("TODO: Provide description about source data, e.g. \n" - "Directory below contains DICOMS compressed into tarballs per " - "each sequence, replicating directory hierarchy of the BIDS dataset" - " itself.")) - create_file_if_missing(op.join(path, 'CHANGES'), + create_file_if_missing( + sourcedata_README, + ( + "TODO: Provide description about source data, e.g. \n" + "Directory below contains DICOMS compressed into tarballs per " + "each sequence, replicating directory hierarchy of the BIDS dataset" + " itself." + ), + ) + create_file_if_missing( + op.join(path, "CHANGES"), "0.0.1 Initial data acquired\n" "TODOs:\n\t- verify and possibly extend information in participants.tsv" " (see for example http://datasets.datalad.org/?dir=/openfmri/ds000208)" "\n\t- fill out dataset_description.json, README, sourcedata/README" " (if present)\n\t- provide _events.tsv file for each _bold.nii.gz with" - " onsets of events (see '8.5 Task events' of BIDS specification)") - create_file_if_missing(op.join(path, 'README'), + " onsets of events (see '8.5 Task events' of BIDS specification)", + ) + create_file_if_missing( + op.join(path, "README"), "TODO: Provide description for the dataset -- basic details about the " - "study, possibly pointing to pre-registration (if public or embargoed)") - create_file_if_missing(op.join(path, 'scans.json'), - json_dumps(SCANS_FILE_FIELDS, sort_keys=False) + "study, possibly pointing to pre-registration (if public or embargoed)", ) - create_file_if_missing(op.join(path, '.bidsignore'), ".duecredit.p") - if op.lexists(op.join(path, '.git')): - create_file_if_missing(op.join(path, '.gitignore'), ".duecredit.p") + create_file_if_missing( + op.join(path, "scans.json"), json_dumps(SCANS_FILE_FIELDS, sort_keys=False) + ) + create_file_if_missing(op.join(path, ".bidsignore"), ".duecredit.p") + if op.lexists(op.join(path, ".git")): + create_file_if_missing(op.join(path, ".gitignore"), ".duecredit.p") populate_aggregated_jsons(path) @@ -184,16 +218,18 @@ def populate_aggregated_jsons(path): # way too many -- let's just collect all which are the same! # FIELDS_TO_TRACK = {'RepetitionTime', 'FlipAngle', 'EchoTime', # 'Manufacturer', 'SliceTiming', ''} - for fpath in find_files(r'.*_task-.*\_bold\.json', - topdir=glob(op.join(path, 'sub-*')), - exclude_vcs=True, - exclude=r"/\.(datalad|heudiconv)/"): + for fpath in find_files( + r".*_task-.*\_bold\.json", + topdir=glob(op.join(path, "sub-*")), + exclude_vcs=True, + exclude=r"/\.(datalad|heudiconv)/", + ): # # According to BIDS spec I think both _task AND _acq (may be more? # _rec, _dir, ...?) should be retained? # TODO: if we are to fix it, then old ones (without _acq) should be # removed first - task = re.sub(r'.*_(task-[^_\.]*(_acq-[^_\.]*)?)_.*', r'\1', fpath) + task = re.sub(r".*_(task-[^_\.]*(_acq-[^_\.]*)?)_.*", r"\1", fpath) json_ = load_json(fpath, retry=100) if task not in tasks: tasks[task] = json_ @@ -204,39 +240,42 @@ def populate_aggregated_jsons(path): if field not in json_ or json_[field] != rec[field]: del rec[field] # create a stub onsets file for each one of those - suf = '_bold.json' + suf = "_bold.json" assert fpath.endswith(suf) # specify the name of the '_events.tsv' file: - if '_echo-' in fpath: + if "_echo-" in fpath: # multi-echo sequence: bids (1.1.0) specifies just one '_events.tsv' # file, common for all echoes. The name will not include _echo-. # TODO: RF to use re.match for better readability/robustness # So, find out the echo number: - fpath_split = fpath.split('_echo-', 1) # split fpath using '_echo-' - fpath_split_2 = fpath_split[1].split('_', 1) # split the second part of fpath_split using '_' - echoNo = fpath_split_2[0] # get echo number - if echoNo == '1': + fpath_split = fpath.split("_echo-", 1) # split fpath using '_echo-' + fpath_split_2 = fpath_split[1].split( + "_", 1 + ) # split the second part of fpath_split using '_' + echoNo = fpath_split_2[0] # get echo number + if echoNo == "1": if len(fpath_split_2) != 2: raise ValueError("Found no trailer after _echo-") # we modify fpath to exclude '_echo-' + echoNo: - fpath = fpath_split[0] + '_' + fpath_split_2[1] + fpath = fpath_split[0] + "_" + fpath_split_2[1] else: # for echoNo greater than 1, don't create the events file, so go to # the next for loop iteration: continue - events_file = remove_suffix(fpath, suf) + '_events.tsv' + events_file = remove_suffix(fpath, suf) + "_events.tsv" # do not touch any existing thing, it may be precious if not op.lexists(events_file): lgr.debug("Generating %s", events_file) - with open(events_file, 'w') as f: + with open(events_file, "w") as f: f.write( "onset\tduration\ttrial_type\tresponse_time\tstim_file" "\tTODO -- fill in rows and add more tab-separated " - "columns if desired") + "columns if desired" + ) # extract tasks files stubs for task_acq, fields in tasks.items(): - task_file = op.join(path, task_acq + '_bold.json') + task_file = op.join(path, task_acq + "_bold.json") # Since we are pulling all unique fields we have to possibly # rewrite this file to guarantee consistency. # See https://github.com/nipy/heudiconv/issues/277 for a usecase/bug @@ -244,8 +283,9 @@ def populate_aggregated_jsons(path): # But the fields we enter (TaskName and CogAtlasID) might need need # to be populated from the file if it already exists placeholders = { - "TaskName": ("TODO: full task name for %s" % - task_acq.split('_')[0].split('-')[1]), + "TaskName": ( + "TODO: full task name for %s" % task_acq.split("_")[0].split("-")[1] + ), "CogAtlasID": "http://www.cognitiveatlas.org/task/id/TODO", } if op.lexists(task_file): @@ -263,21 +303,21 @@ def populate_aggregated_jsons(path): def tuneup_bids_json_files(json_files): - """Given a list of BIDS .json files, e.g. """ + """Given a list of BIDS .json files, e.g.""" if not json_files: return # Harmonize generic .json formatting for jsonfile in json_files: json_ = load_json(jsonfile) # sanitize! - for f1 in ['Acquisition', 'Study', 'Series']: - for f2 in ['DateTime', 'Date']: + for f1 in ["Acquisition", "Study", "Series"]: + for f2 in ["DateTime", "Date"]: json_.pop(f1 + f2, None) # TODO: should actually be placed into series file which must # go under annex (not under git) and marked as sensitive # MG - Might want to replace with flag for data sensitivity # related - https://github.com/nipy/heudiconv/issues/92 - if 'Date' in str(json_): + if "Date" in str(json_): # Let's hope no word 'Date' comes within a study name or smth like # that raise ValueError("There must be no dates in .json sidecar") @@ -292,12 +332,12 @@ def tuneup_bids_json_files(json_files): # MG - want to expand this for other _epi # possibly add IntendedFor automatically as well? - if seqtype == 'fmap': - json_basename = '_'.join(jsonfile.split('_')[:-1]) + if seqtype == "fmap": + json_basename = "_".join(jsonfile.split("_")[:-1]) # if we got by now all needed .json files -- we can fix them up # unfortunately order of "items" is not guaranteed atm - json_phasediffname = json_basename + '_phasediff.json' - json_mag = json_basename + '_magnitude*.json' + json_phasediffname = json_basename + "_phasediff.json" + json_mag = json_basename + "_magnitude*.json" if op.exists(json_phasediffname) and len(glob(json_mag)) >= 1: json_ = load_json(json_phasediffname) # TODO: we might want to reorder them since ATM @@ -306,8 +346,9 @@ def tuneup_bids_json_files(json_files): lgr.debug("Placing EchoTime fields into phasediff file") for i in 1, 2: try: - json_['EchoTime%d' % i] = (load_json(json_basename + - '_magnitude%d.json' % i)['EchoTime']) + json_["EchoTime%d" % i] = load_json( + json_basename + "_magnitude%d.json" % i + )["EchoTime"] except IOError as exc: lgr.error("Failed to open magnitude file: %s", exc) # might have been made R/O already, but if not -- it will be set @@ -321,55 +362,98 @@ def tuneup_bids_json_files(json_files): def add_participant_record(studydir, subject, age, sex): - participants_tsv = op.join(studydir, 'participants.tsv') - participant_id = 'sub-%s' % subject + participants_tsv = op.join(studydir, "participants.tsv") + participant_id = "sub-%s" % subject - if not create_file_if_missing(participants_tsv, - '\t'.join(['participant_id', 'age', 'sex', 'group']) + '\n'): + if not create_file_if_missing( + participants_tsv, "\t".join(["participant_id", "age", "sex", "group"]) + "\n" + ): # check if may be subject record already exists with open(participants_tsv) as f: f.readline() - known_subjects = {l.split('\t')[0] for l in f.readlines()} + known_subjects = {ln.split("\t")[0] for ln in f.readlines()} if participant_id in known_subjects: return else: # Populate particpants.json (an optional file to describe column names in # participant.tsv). This auto generation will make BIDS-validator happy. - participants_json = op.join(studydir, 'participants.json') + participants_json = op.join(studydir, "participants.json") if not op.lexists(participants_json): - save_json(participants_json, - OrderedDict([ - ("participant_id", OrderedDict([ - ("Description", "Participant identifier")])), - ("age", OrderedDict([ - ("Description", "Age in years (TODO - verify) as in the initial" - " session, might not be correct for other sessions")])), - ("sex", OrderedDict([ - ("Description", "self-rated by participant, M for male/F for " - "female (TODO: verify)")])), - ("group", OrderedDict([ - ("Description", "(TODO: adjust - by default everyone is in " - "control group)")])), - ]), - sort_keys=False) + save_json( + participants_json, + OrderedDict( + [ + ( + "participant_id", + OrderedDict([("Description", "Participant identifier")]), + ), + ( + "age", + OrderedDict( + [ + ( + "Description", + "Age in years (TODO - verify) as in the initial" + " session, might not be correct for other sessions", + ) + ] + ), + ), + ( + "sex", + OrderedDict( + [ + ( + "Description", + "self-rated by participant, M for male/F for " + "female (TODO: verify)", + ) + ] + ), + ), + ( + "group", + OrderedDict( + [ + ( + "Description", + "(TODO: adjust - by default everyone is in " + "control group)", + ) + ] + ), + ), + ] + ), + sort_keys=False, + ) # Add a new participant - with open(participants_tsv, 'a') as f: + with open(participants_tsv, "a") as f: f.write( - '\t'.join(map(str, [participant_id, - maybe_na(treat_age(age)), - maybe_na(sex), - 'control'])) + '\n') + "\t".join( + map( + str, + [ + participant_id, + maybe_na(treat_age(age)), + maybe_na(sex), + "control", + ], + ) + ) + + "\n" + ) def find_subj_ses(f_name): """Given a path to the bids formatted filename parse out subject/session""" # we will allow the match at either directories or within filename # assuming that bids layout is "correct" - regex = re.compile('sub-(?P[a-zA-Z0-9]*)([/_]ses-(?P[a-zA-Z0-9]*))?') + regex = re.compile("sub-(?P[a-zA-Z0-9]*)([/_]ses-(?P[a-zA-Z0-9]*))?") regex_res = regex.search(f_name) res = regex_res.groupdict() if regex_res else {} - return res.get('subj', None), res.get('ses', None) + return res.get("subj", None), res.get("ses", None) def save_scans_key(item, bids_files): @@ -392,21 +476,22 @@ def save_scans_key(item, bids_files): subj, ses = None, None for bids_file in bids_files: # get filenames - f_name = '/'.join(bids_file.split('/')[-2:]) - f_name = f_name.replace('json', 'nii.gz') + f_name = "/".join(bids_file.split("/")[-2:]) + f_name = f_name.replace("json", "nii.gz") rows[f_name] = get_formatted_scans_key_row(item[-1][0]) subj_, ses_ = find_subj_ses(f_name) if not subj_: lgr.warning( "Failed to detect fulfilled BIDS layout. " "No scans.tsv file(s) will be produced for %s", - ", ".join(bids_files) + ", ".join(bids_files), ) return if subj and subj_ != subj: raise ValueError( "We found before subject %s but now deduced %s from %s" - % (subj, subj_, f_name)) + % (subj, subj_, f_name) + ) subj = subj_ if ses and ses_ != ses: raise ValueError( @@ -417,9 +502,10 @@ def save_scans_key(item, bids_files): # where should we store it? output_dir = op.dirname(op.dirname(bids_file)) # save - ses = '_ses-%s' % ses if ses else '' + ses = "_ses-%s" % ses if ses else "" add_rows_to_scans_keys_file( - op.join(output_dir, 'sub-{0}{1}_scans.tsv'.format(subj, ses)), rows) + op.join(output_dir, "sub-{0}{1}_scans.tsv".format(subj, ses)), rows + ) def add_rows_to_scans_keys_file(fn, newrows): @@ -433,8 +519,8 @@ def add_rows_to_scans_keys_file(fn, newrows): extra rows to add (acquisition time, referring physician, random string) """ if op.lexists(fn): - with open(fn, 'r') as csvfile: - reader = csv.reader(csvfile, delimiter='\t') + with open(fn, "r") as csvfile: + reader = csv.reader(csvfile, delimiter="\t") existing_rows = [row for row in reader] # skip header fnames2info = {row[0]: row[1:] for row in existing_rows[1:]} @@ -458,8 +544,8 @@ def add_rows_to_scans_keys_file(fn, newrows): lgr.warning("Sorting scans by date failed: %s", str(exc)) data_rows_sorted = sorted(data_rows) # save - with open(fn, 'a') as csvfile: - writer = csv.writer(csvfile, delimiter='\t') + with open(fn, "a") as csvfile: + writer = csv.writer(csvfile, delimiter="\t") writer.writerows([header] + data_rows_sorted) @@ -481,19 +567,18 @@ def get_formatted_scans_key_row(dcm_fn) -> typing.List[str]: # add random string # But let's make it reproducible by using all UIDs # (might change across versions?) - randcontent = u''.join( - [getattr(dcm_data, f) or '' for f in sorted(dir(dcm_data)) - if f.endswith('UID')] + randcontent = "".join( + [getattr(dcm_data, f) or "" for f in sorted(dir(dcm_data)) if f.endswith("UID")] ) randstr = hashlib.md5(randcontent.encode()).hexdigest()[:8] try: perfphys = dcm_data.PerformingPhysicianName except AttributeError: - perfphys = '' + perfphys = "" row = [acq_datetime.isoformat() if acq_datetime else "", perfphys, randstr] # empty entries should be 'n/a' # https://github.com/dartmouth-pbs/heudiconv/issues/32 - row = ['n/a' if not str(e) else e for e in row] + row = ["n/a" if not str(e) else e for e in row] return row @@ -511,9 +596,11 @@ def convert_sid_bids(subject_id): subject_id : string Original subject ID """ - warnings.warn('convert_sid_bids() is deprecated, ' - 'please use sanitize_label() instead.', - DeprecationWarning) + warnings.warn( + "convert_sid_bids() is deprecated, please use sanitize_label() instead.", + DeprecationWarning, + stacklevel=2, + ) return sanitize_label(subject_id) @@ -533,10 +620,13 @@ def get_shim_setting(json_file): data = load_json(json_file) try: shims = data[SHIM_KEY] - except KeyError as e: - lgr.error('File %s does not have "%s". ' - 'Please use a different "matching_parameters" in your heuristic file', - json_file, SHIM_KEY) + except KeyError: + lgr.error( + 'File %s does not have "%s". ' + 'Please use a different "matching_parameters" in your heuristic file', + json_file, + SHIM_KEY, + ) raise KeyError return shims @@ -559,30 +649,35 @@ def find_fmap_groups(fmap_dir): key: prefix common to the group (e.g. no "dir" entity, "_phase"/"_magnitude", ...) value: list of all fmap paths in the group """ - if op.basename(fmap_dir) != 'fmap': - lgr.error('%s is not a fieldmap folder', fmap_dir) + if op.basename(fmap_dir) != "fmap": + lgr.error("%s is not a fieldmap folder", fmap_dir) # Get a list of all fmap json files in the session: - fmap_jsons = sorted(glob(op.join(fmap_dir, '*.json'))) + fmap_jsons = sorted(glob(op.join(fmap_dir, "*.json"))) # RegEx to remove fmap-specific substrings from fmap file names # "_phase[1,2]", "_magnitude[1,2]", "_phasediff", "_dir-