(WIP) Harmonize json

requirements.txt

@@ -6,6 +6,7 @@ cvxpy==1.3.*
 cycler==0.11.*
 Cython==0.29.*, !=0.29.29
 dipy==1.7.*
+deepdiff==6.3.0
 dmri-amico==1.5.*
 dmri-commit==1.6.*
 docopt==0.6.*

scilpy/utils/util.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 
+import collections.abc
+
 from dipy.io.utils import get_reference_info
 import numpy as np
 from numpy.lib.index_tricks import r_ as row
@@ -92,3 +94,29 @@ def is_float(value):
         return True
     except ValueError:
         return False
+
+
+def recursive_update(d, u, from_existing=False):
+    """Harmonize a dictionary to garantee all keys exists at all sub-levels."""
+    for k, v in u.items():
+        if isinstance(v, collections.abc.Mapping):
+            if k not in d and from_existing:
+                d[k] = u[k]
+            else:
+                d[k] = recursive_update(d.get(k, {}), v,
+                                        from_existing=from_existing)
+        else:
+            if not from_existing:
+                d[k] = float('nan')
+            elif k not in d:
+                d[k] = float('nan')
+    return d
+
+
+def recursive_print(data):
+    """Print the keys of all layers. Dictionary must be harmonized first."""
+    if isinstance(data, collections.abc.Mapping):
+        print(list(data.keys()))
+        recursive_print(data[list(data.keys())[0]])
+    else:
+        return

scripts/scil_harmonize_json.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+This script will harmonize a json file by adding missing keys and values that
+differs between the different layers of the dictionary.
+
+This is use only (for now) in Aggregate_All_* portion of tractometry-flow,
+to counter the problem of missing bundles/metrics/lesions between subjects.
+"""
+
+import argparse
+from copy import deepcopy
+import json
+
+from deepdiff import DeepDiff
+
+from scilpy.io.utils import (assert_inputs_exist,
+                             add_json_args,
+                             assert_outputs_exist,
+                             add_overwrite_arg,
+                             add_verbose_arg)
+from scilpy.utils.util import recursive_update, recursive_print
+
+
+def _build_arg_parser():
+    p = argparse.ArgumentParser(description=__doc__,
+                                formatter_class=argparse.RawTextHelpFormatter)
+
+    p.add_argument('in_file',
+                   help='Input file (json).')
+    p.add_argument('out_file',
+                   help='Output file (json).')
+    add_json_args(p)
+    add_overwrite_arg(p)
+    add_verbose_arg(p)
+
+    return p
+
+
+def main():
+    parser = _build_arg_parser()
+    args = parser.parse_args()
+
+    assert_inputs_exist(parser, args.in_file)
+    assert_outputs_exist(parser, args, args.out_file)
+
+    with open(args.in_file) as f:
+        data = json.load(f)
+        data_old = deepcopy(data)
+
+    # Generate the reference (full) dictionary. Skip level 0, complete all
+    # levels, but write NaN at last level (leaf).
+    new = {}
+    for key in data.keys():
+        new = recursive_update(new, data[key], from_existing=False)
+
+    # Harmonize the original dictionary, missing keys are added, when a <leaf>
+    # is missing NaN will be stored
+    for key in data.keys():
+        data[key] = recursive_update(data[key], new, from_existing=True)
+
+    if args.verbose:
+        print('Layered keys of the dictionary:')
+        recursive_print(data)
+        print()
+
+        dd = DeepDiff(data, data_old, ignore_order=True)
+        if 'dictionary_item_removed' in dd:
+            print('Missing keys that were harmonized:')
+            print(dd['dictionary_item_removed'])
+
+    with open(args.out_file, "w") as f:
+        json.dump(data, f, indent=args.indent, sort_keys=args.sort_keys)
+
+
+if __name__ == "__main__":
+    main()