From 0f62a7215010c78c45846491178488b346abc3b0 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Tue, 12 May 2020 13:38:51 -0400 Subject: [PATCH 1/9] Retain sub-second resolution in scans files. --- heudiconv/bids.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/heudiconv/bids.py b/heudiconv/bids.py index 6a9c136c..1e6e5fc5 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -406,9 +406,9 @@ def get_formatted_scans_key_row(dcm_fn): # parse date and time and get it into isoformat try: date = dcm_data.ContentDate - time = dcm_data.ContentTime.split('.')[0] - td = time + date - acq_time = datetime.strptime(td, '%H%M%S%Y%m%d').isoformat() + time = dcm_data.ContentTime + td = time + ':' + date + acq_time = datetime.strptime(td, '%H%M%S.%f:%Y%m%d').isoformat() except (AttributeError, ValueError) as exc: lgr.warning("Failed to get date/time for the content: %s", str(exc)) acq_time = '' From 5d1d4c6dfd4ee0634e6b3199f3d9114b06e923b9 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Tue, 12 May 2020 15:00:13 -0400 Subject: [PATCH 2/9] Fix tests. --- heudiconv/tests/test_heuristics.py | 2 +- heudiconv/tests/test_main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/heudiconv/tests/test_heuristics.py b/heudiconv/tests/test_heuristics.py index f36bbb4c..32870d9a 100644 --- a/heudiconv/tests/test_heuristics.py +++ b/heudiconv/tests/test_heuristics.py @@ -116,7 +116,7 @@ def test_scans_keys_reproin(tmpdir, invocation): if i != 0: assert(os.path.exists(pjoin(dirname(scans_keys[0]), row[0]))) assert(re.match( - '^[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}$', + '^[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}.\d]{6}$', row[1])) diff --git a/heudiconv/tests/test_main.py b/heudiconv/tests/test_main.py index 91291378..ab1dd73d 100644 --- a/heudiconv/tests/test_main.py +++ b/heudiconv/tests/test_main.py @@ -173,7 +173,7 @@ def test_get_formatted_scans_key_row(): row1 = get_formatted_scans_key_row(dcm_fn) assert len(row1) == 3 - assert row1[0] == '2016-10-14T09:26:36' + assert row1[0] == '2016-10-14T09:26:36.693000' assert row1[1] == 'n/a' prandstr1 = row1[2] From 609aa27eb3a0a8d8a0add4e4184cee271991cb27 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Tue, 12 May 2020 15:10:00 -0400 Subject: [PATCH 3/9] Fix test again. --- heudiconv/tests/test_heuristics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/tests/test_heuristics.py b/heudiconv/tests/test_heuristics.py index 32870d9a..eedf61f2 100644 --- a/heudiconv/tests/test_heuristics.py +++ b/heudiconv/tests/test_heuristics.py @@ -116,7 +116,7 @@ def test_scans_keys_reproin(tmpdir, invocation): if i != 0: assert(os.path.exists(pjoin(dirname(scans_keys[0]), row[0]))) assert(re.match( - '^[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}.\d]{6}$', + '^[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}.[\d]{6}$', row[1])) From 4c25a8b343575d942f38f881fc3febc1ff215ff2 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Tue, 12 May 2020 16:26:29 -0400 Subject: [PATCH 4/9] Move datetime to new util function and add test. --- heudiconv/bids.py | 4 ++-- heudiconv/tests/test_utils.py | 15 +++++++++++++++ heudiconv/utils.py | 25 +++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/heudiconv/bids.py b/heudiconv/bids.py index 1e6e5fc5..b90953a6 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -21,6 +21,7 @@ json_dumps_pretty, set_readonly, is_readonly, + get_datetime, ) lgr = logging.getLogger(__name__) @@ -407,8 +408,7 @@ def get_formatted_scans_key_row(dcm_fn): try: date = dcm_data.ContentDate time = dcm_data.ContentTime - td = time + ':' + date - acq_time = datetime.strptime(td, '%H%M%S.%f:%Y%m%d').isoformat() + acq_time = get_datetime(date, time) except (AttributeError, ValueError) as exc: lgr.warning("Failed to get date/time for the content: %s", str(exc)) acq_time = '' diff --git a/heudiconv/tests/test_utils.py b/heudiconv/tests/test_utils.py index 00f8a0a2..6bd3f864 100644 --- a/heudiconv/tests/test_utils.py +++ b/heudiconv/tests/test_utils.py @@ -10,6 +10,7 @@ load_json, create_tree, save_json, + get_datetime, JSONDecodeError) import pytest @@ -85,3 +86,17 @@ def test_load_json(tmpdir, caplog): save_json(valid_json_file, vcontent) assert load_json(valid_json_file) == vcontent + + +def test_get_datetime(): + """ + Test utils.get_datetime() + """ + date = '20200512' + time = '162130' + datetime_str = get_datetime(date, time) + assert datetime_str == '2020-05-12T16:21:30.000000' + date = '20200512' + time = '162130.5' + datetime_str = get_datetime(date, time) + assert datetime_str == '2020-05-12T16:21:30.500000' diff --git a/heudiconv/utils.py b/heudiconv/utils.py index da3ebc56..44aaae31 100644 --- a/heudiconv/utils.py +++ b/heudiconv/utils.py @@ -13,6 +13,7 @@ from collections import namedtuple from glob import glob from subprocess import check_output +from datetime import datetime from nipype.utils.filemanip import which @@ -505,3 +506,27 @@ def get_typed_attr(obj, attr, _type, default=None): except (TypeError, ValueError): return default return val + + +def get_datetime(date, time): + """ + Convert date and time from dicom to isoformat. + + Parameters + ---------- + date : str + Date in YYYYMMDD format. + time : str + Time in either HHMMSS.ffffff format or HHMMSS format. + + Returns + ------- + datetime_str : str + Combined date and time in ISO format, with milliseconds. + """ + if '.' not in time: + # add milliseconds if not available + time += '.000' + td = time + ':' + date + datetime_str = datetime.strptime(td, '%H%M%S.%f:%Y%m%d').isoformat() + return datetime_str From 09a9ea224514fdf304d6282b3e429699dc7190bc Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Tue, 12 May 2020 16:33:38 -0400 Subject: [PATCH 5/9] Fix test. --- heudiconv/tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/tests/test_utils.py b/heudiconv/tests/test_utils.py index 6bd3f864..eb70feaa 100644 --- a/heudiconv/tests/test_utils.py +++ b/heudiconv/tests/test_utils.py @@ -95,7 +95,7 @@ def test_get_datetime(): date = '20200512' time = '162130' datetime_str = get_datetime(date, time) - assert datetime_str == '2020-05-12T16:21:30.000000' + assert datetime_str == '2020-05-12T16:21:30' date = '20200512' time = '162130.5' datetime_str = get_datetime(date, time) From b95f67b15d2a4ea61110dbe0635f0e0458b652dd Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 12 May 2020 20:29:36 -0400 Subject: [PATCH 6/9] ENH: Adjust to reflect it is microseconds that are parsed/reported Made it also .000000 just for the sake of making robust since %f expects microseconds --- heudiconv/utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/heudiconv/utils.py b/heudiconv/utils.py index 44aaae31..2bdf3292 100644 --- a/heudiconv/utils.py +++ b/heudiconv/utils.py @@ -510,7 +510,7 @@ def get_typed_attr(obj, attr, _type, default=None): def get_datetime(date, time): """ - Convert date and time from dicom to isoformat. + Combine date and time from dicom to isoformat. Parameters ---------- @@ -522,11 +522,12 @@ def get_datetime(date, time): Returns ------- datetime_str : str - Combined date and time in ISO format, with milliseconds. + Combined date and time in ISO format (with microseconds if + they were available in provided time). """ if '.' not in time: - # add milliseconds if not available - time += '.000' + # add dummy microseconds if not available for strptime to parse + time += '.000000' td = time + ':' + date datetime_str = datetime.strptime(td, '%H%M%S.%f:%Y%m%d').isoformat() return datetime_str From a1e5b330e65b4f817d37fe46c781388fda0975e5 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 12 May 2020 20:59:26 -0400 Subject: [PATCH 7/9] RF(TST): remove one-time use variables - oneliner is much easier to grasp in such tests IMHO --- heudiconv/tests/test_utils.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/heudiconv/tests/test_utils.py b/heudiconv/tests/test_utils.py index eb70feaa..8a5b7dc6 100644 --- a/heudiconv/tests/test_utils.py +++ b/heudiconv/tests/test_utils.py @@ -92,11 +92,5 @@ def test_get_datetime(): """ Test utils.get_datetime() """ - date = '20200512' - time = '162130' - datetime_str = get_datetime(date, time) - assert datetime_str == '2020-05-12T16:21:30' - date = '20200512' - time = '162130.5' - datetime_str = get_datetime(date, time) - assert datetime_str == '2020-05-12T16:21:30.500000' + assert get_datetime('20200512', '162130') == '2020-05-12T16:21:30' + assert get_datetime('20200512', '162130.5') == '2020-05-12T16:21:30.500000' From d3e247cedf27812215c5bafdcca217582b000875 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 12 May 2020 21:03:55 -0400 Subject: [PATCH 8/9] ENH: make get_datetime accept microseconds kwarg to not provide microseconds --- heudiconv/tests/test_utils.py | 1 + heudiconv/utils.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/heudiconv/tests/test_utils.py b/heudiconv/tests/test_utils.py index 8a5b7dc6..b8c8c160 100644 --- a/heudiconv/tests/test_utils.py +++ b/heudiconv/tests/test_utils.py @@ -94,3 +94,4 @@ def test_get_datetime(): """ assert get_datetime('20200512', '162130') == '2020-05-12T16:21:30' assert get_datetime('20200512', '162130.5') == '2020-05-12T16:21:30.500000' + assert get_datetime('20200512', '162130.5', microseconds=False) == '2020-05-12T16:21:30' diff --git a/heudiconv/utils.py b/heudiconv/utils.py index 2bdf3292..f2631757 100644 --- a/heudiconv/utils.py +++ b/heudiconv/utils.py @@ -508,7 +508,7 @@ def get_typed_attr(obj, attr, _type, default=None): return val -def get_datetime(date, time): +def get_datetime(date, time, *, microseconds=True): """ Combine date and time from dicom to isoformat. @@ -518,16 +518,21 @@ def get_datetime(date, time): Date in YYYYMMDD format. time : str Time in either HHMMSS.ffffff format or HHMMSS format. + microseconds: bool, optional + Either to include microseconds in the output Returns ------- datetime_str : str - Combined date and time in ISO format (with microseconds if - they were available in provided time). + Combined date and time in ISO format, with microseconds as + if fraction was provided in 'time', and 'microseconds' was + True. """ if '.' not in time: # add dummy microseconds if not available for strptime to parse time += '.000000' td = time + ':' + date datetime_str = datetime.strptime(td, '%H%M%S.%f:%Y%m%d').isoformat() + if not microseconds: + datetime_str = datetime_str.split('.', 1)[0] return datetime_str From c5fe64fdbf59081cfa36509bec53c5fa69452c60 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 13 May 2020 16:33:41 -0400 Subject: [PATCH 9/9] RF: centralize definition of columns in the _scans files That allows to remove code duplication of header column names --- heudiconv/bids.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/heudiconv/bids.py b/heudiconv/bids.py index b90953a6..ce8ab4c3 100644 --- a/heudiconv/bids.py +++ b/heudiconv/bids.py @@ -26,6 +26,19 @@ lgr = logging.getLogger(__name__) +# Fields to be populated in _scans files. Order matters +SCANS_FILE_FIELDS = OrderedDict([ + ("filename", OrderedDict([ + ("Description", "Name of the nifti file")])), + ("acq_time", OrderedDict([ + ("LongName", "Acquisition time"), + ("Description", "Acquisition time of the particular scan")])), + ("operator", OrderedDict([ + ("Description", "Name of the operator")])), + ("randstr", OrderedDict([ + ("LongName", "Random string"), + ("Description", "md5 hash of UIDs")])), +]) class BIDSError(Exception): pass @@ -360,22 +373,9 @@ def add_rows_to_scans_keys_file(fn, newrows): # _scans.tsv). This auto generation will make BIDS-validator happy. scans_json = '.'.join(fn.split('.')[:-1] + ['json']) if not op.lexists(scans_json): - save_json(scans_json, - OrderedDict([ - ("filename", OrderedDict([ - ("Description", "Name of the nifti file")])), - ("acq_time", OrderedDict([ - ("LongName", "Acquisition time"), - ("Description", "Acquisition time of the particular scan")])), - ("operator", OrderedDict([ - ("Description", "Name of the operator")])), - ("randstr", OrderedDict([ - ("LongName", "Random string"), - ("Description", "md5 hash of UIDs")])), - ]), - sort_keys=False) + save_json(scans_json, SCANS_FILE_FIELDS, sort_keys=False) - header = ['filename', 'acq_time', 'operator', 'randstr'] + header = SCANS_FILE_FIELDS # prepare all the data rows data_rows = [[k] + v for k, v in fnames2info.items()] # sort by the date/filename