From 54ebdb5a719d68010e20c4f95666ae1eabc97b39 Mon Sep 17 00:00:00 2001 From: Seperman Date: Mon, 8 Apr 2024 12:17:27 -0700 Subject: [PATCH] Include type info and change the "unknown" value for flat rows to something that is friendly for Postgres enums --- deepdiff/delta.py | 38 ++++++++++++++---- deepdiff/helper.py | 2 +- docs/serialization.rst | 28 +++++++------- tests/test_delta.py | 87 +++++++++++++++++++++++------------------- 4 files changed, 93 insertions(+), 62 deletions(-) diff --git a/deepdiff/delta.py b/deepdiff/delta.py index 62068dd..39f7d36 100644 --- a/deepdiff/delta.py +++ b/deepdiff/delta.py @@ -878,7 +878,7 @@ def to_dict(self): return dict(self.diff) @staticmethod - def _get_flat_row(action, info, _parse_path, keys_and_funcs): + def _get_flat_row(action, info, _parse_path, keys_and_funcs, report_type_changes=True): for path, details in info.items(): row = {'path': _parse_path(path), 'action': action} for key, new_key, func in keys_and_funcs: @@ -887,6 +887,11 @@ def _get_flat_row(action, info, _parse_path, keys_and_funcs): row[new_key] = func(details[key]) else: row[new_key] = details[key] + if report_type_changes: + if 'value' in row and 'type' not in row: + row['type'] = type(row['value']) + if 'old_value' in row and 'old_type' not in row: + row['old_type'] = type(row['old_value']) yield FlatDeltaRow(**row) @staticmethod @@ -1060,6 +1065,9 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) - 'iterable_items_removed_at_indexes': 'unordered_iterable_item_removed', } for action, info in self.diff.items(): + if action == '_iterable_opcodes': + result.extend(self._flatten_iterable_opcodes()) + continue if action.startswith('_'): continue if action in FLATTENING_NEW_ACTION_MAP: @@ -1072,12 +1080,20 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) - path2.append((index, 'GET')) else: path2.append(index) - result.append(FlatDeltaRow(path=path2, value=value, action=new_action)) + if report_type_changes: + row = FlatDeltaRow(path=path2, value=value, action=new_action, type=type(value)) + else: + row = FlatDeltaRow(path=path2, value=value, action=new_action) + result.append(row) elif action in {'set_item_added', 'set_item_removed'}: for path, values in info.items(): path = _parse_path(path) for value in values: - result.append(FlatDeltaRow(path=path, value=value, action=action)) + if report_type_changes: + row = FlatDeltaRow(path=path, value=value, action=action, type=type(value)) + else: + row = FlatDeltaRow(path=path, value=value, action=action) + result.append(row) elif action == 'dictionary_item_added': for path, value in info.items(): path = _parse_path(path) @@ -1092,14 +1108,22 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) - elif isinstance(value, set) and len(value) == 1: value = value.pop() action = 'set_item_added' - result.append(FlatDeltaRow(path=path, value=value, action=action)) + if report_type_changes: + row = FlatDeltaRow(path=path, value=value, action=action, type=type(value)) + else: + row = FlatDeltaRow(path=path, value=value, action=action) + result.append(row) elif action in { 'dictionary_item_removed', 'iterable_item_added', 'iterable_item_removed', 'attribute_removed', 'attribute_added' }: for path, value in info.items(): path = _parse_path(path) - result.append(FlatDeltaRow(path=path, value=value, action=action)) + if report_type_changes: + row = FlatDeltaRow(path=path, value=value, action=action, type=type(value)) + else: + row = FlatDeltaRow(path=path, value=value, action=action) + result.append(row) elif action == 'type_changes': if not report_type_changes: action = 'values_changed' @@ -1109,16 +1133,16 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) - info=info, _parse_path=_parse_path, keys_and_funcs=keys_and_funcs, + report_type_changes=report_type_changes, ): result.append(row) - elif action == '_iterable_opcodes': - result.extend(self._flatten_iterable_opcodes()) else: for row in self._get_flat_row( action=action, info=info, _parse_path=_parse_path, keys_and_funcs=keys_and_funcs, + report_type_changes=report_type_changes, ): result.append(row) return result diff --git a/deepdiff/helper.py b/deepdiff/helper.py index cdf34ca..22846f1 100644 --- a/deepdiff/helper.py +++ b/deepdiff/helper.py @@ -771,7 +771,7 @@ class FlatDataAction(str, enum.Enum): unordered_iterable_item_removed = 'unordered_iterable_item_removed' -UnkownValueCode = '*-UNKNOWN-*' +UnkownValueCode = 'unknown___' class FlatDeltaRow(NamedTuple): diff --git a/docs/serialization.rst b/docs/serialization.rst index 0f63428..92ef757 100644 --- a/docs/serialization.rst +++ b/docs/serialization.rst @@ -181,7 +181,7 @@ Flat Row Specs: unordered_iterable_item_removed = 'unordered_iterable_item_removed' - UnkownValueCode = '*-UNKNOWN-*' + UnkownValueCode = 'unknown___' class FlatDeltaRow(NamedTuple): @@ -205,7 +205,7 @@ Delta Serialize To Flat Dictionaries Sometimes, it is desired to serialize a :ref:`delta_label` object to a list of flat dictionaries. For example, to store them in relation databases. In that case, you can use the Delta.to_flat_dicts to achieve the desired outcome. -Since None is a valid value, we use a special hard-coded string to signify "unkown": '*-UNKNOWN-*' +Since None is a valid value, we use a special hard-coded string to signify "unkown": 'unknown___' .. note:: Many new keys are added to the flat dicts in DeepDiff 7.0.0 @@ -226,25 +226,25 @@ For example: >>> pprint(flat_dicts, indent=2) [ { 'action': 'dictionary_item_added', 'new_path': None, - 'old_type': '*-UNKNOWN-*', - 'old_value': '*-UNKNOWN-*', + 'old_type': 'unknown___', + 'old_value': 'unknown___', 'path': ['field2', 'key2'], 't1_from_index': None, 't1_to_index': None, 't2_from_index': None, 't2_to_index': None, - 'type': '*-UNKNOWN-*', + 'type': 'unknown___', 'value': 'value2'}, { 'action': 'dictionary_item_removed', 'new_path': None, - 'old_type': '*-UNKNOWN-*', - 'old_value': '*-UNKNOWN-*', + 'old_type': 'unknown___', + 'old_value': 'unknown___', 'path': ['key1'], 't1_from_index': None, 't1_to_index': None, 't2_from_index': None, 't2_to_index': None, - 'type': '*-UNKNOWN-*', + 'type': 'unknown___', 'value': 'value1'}] @@ -261,25 +261,25 @@ Example 2: >>> pprint(flat_dicts, indent=2) [ { 'action': 'iterable_item_added', 'new_path': None, - 'old_type': '*-UNKNOWN-*', - 'old_value': '*-UNKNOWN-*', + 'old_type': 'unknown___', + 'old_value': 'unknown___', 'path': [2], 't1_from_index': None, 't1_to_index': None, 't2_from_index': None, 't2_to_index': None, - 'type': '*-UNKNOWN-*', + 'type': 'unknown___', 'value': 'C'}, { 'action': 'iterable_item_added', 'new_path': None, - 'old_type': '*-UNKNOWN-*', - 'old_value': '*-UNKNOWN-*', + 'old_type': 'unknown___', + 'old_value': 'unknown___', 'path': [3], 't1_from_index': None, 't1_to_index': None, 't2_from_index': None, 't2_to_index': None, - 'type': '*-UNKNOWN-*', + 'type': 'unknown___', 'value': 'D'}] diff --git a/tests/test_delta.py b/tests/test_delta.py index b03b9e6..72386e7 100644 --- a/tests/test_delta.py +++ b/tests/test_delta.py @@ -73,8 +73,8 @@ def test_list_difference_add_delta(self): flat_result1 = delta.to_flat_rows() flat_expected1 = [ - FlatDeltaRow(path=[3], value=5, action='iterable_item_added'), - FlatDeltaRow(path=[2], value=3, action='iterable_item_added'), + FlatDeltaRow(path=[3], value=5, action='iterable_item_added', type=int), + FlatDeltaRow(path=[2], value=3, action='iterable_item_added', type=int), ] assert flat_expected1 == flat_result1 @@ -291,9 +291,9 @@ def test_list_difference3_delta(self): flat_result1 = delta.to_flat_rows() flat_expected1 = [ - FlatDeltaRow(path=[4, 'b', 2], action='values_changed', value=2, old_value=5), - FlatDeltaRow(path=[4, 'b', 1], action='values_changed', value=3, old_value=2), - FlatDeltaRow(path=[4, 'b', 3], value=5, action='iterable_item_added'), + FlatDeltaRow(path=[4, 'b', 2], action='values_changed', value=2, old_value=5, type=int, old_type=int), + FlatDeltaRow(path=[4, 'b', 1], action='values_changed', value=3, old_value=2, type=int, old_type=int), + FlatDeltaRow(path=[4, 'b', 3], value=5, action='iterable_item_added', type=int), ] assert flat_expected1 == flat_result1 @@ -332,9 +332,9 @@ def test_list_difference_delta_raises_error_if_prev_value_does_not_match(self): flat_result2 = delta2.to_flat_rows() flat_expected2 = [ - FlatDeltaRow(path=[2], action='values_changed', value=2, old_value=5), - FlatDeltaRow(path=[1], action='values_changed', value=3, old_value=2), - FlatDeltaRow(path=[3], value=5, action='iterable_item_added'), + FlatDeltaRow(path=[2], action='values_changed', value=2, old_value=5, type=int, old_type=int), + FlatDeltaRow(path=[1], action='values_changed', value=3, old_value=2, type=int, old_type=int), + FlatDeltaRow(path=[3], value=5, action='iterable_item_added', type=int), ] assert flat_expected2 == flat_result2 @@ -363,8 +363,8 @@ def test_list_difference_delta1(self): flat_result = delta.to_flat_rows() flat_expected = [ - FlatDeltaRow(path=[4, 'b', 2], value='to_be_removed', action='iterable_item_removed'), - FlatDeltaRow(path=[4, 'b', 3], value='to_be_removed2', action='iterable_item_removed'), + FlatDeltaRow(path=[4, 'b', 2], value='to_be_removed', action='iterable_item_removed', type=str), + FlatDeltaRow(path=[4, 'b', 3], value='to_be_removed2', action='iterable_item_removed', type=str), ] assert flat_expected == flat_result @@ -567,7 +567,8 @@ def compare_func(item1, item2, level=None): 'professionalDesignation': '', 'suffix': 'SR', 'nameIdentifier': '00003'}, - action='unordered_iterable_item_added'), + action='unordered_iterable_item_added', + type=dict), FlatDeltaRow(path=['individualNames', 1], value={'firstName': 'John', 'lastName': 'Doe', @@ -577,7 +578,9 @@ def compare_func(item1, item2, level=None): 'professionalDesignation': '', 'suffix': 'SR', 'nameIdentifier': '00002'}, - action='unordered_iterable_item_removed')] + action='unordered_iterable_item_removed', + type=dict), + ] preserved_flat_dict_list = copy.deepcopy(flat_rows_list) # Use this later for assert comparison @@ -1405,13 +1408,13 @@ def test_list_ignore_order_various_deltas2(self): flat_result1 = delta1.to_flat_rows() flat_expected1 = [ - {'path': [0], 'value': 7, 'action': 'unordered_iterable_item_added'}, - {'path': [6], 'value': 8, 'action': 'unordered_iterable_item_added'}, - {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added'}, - {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added'}, - {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added'}, - {'path': [6], 'value': 6, 'action': 'unordered_iterable_item_removed'}, - {'path': [0], 'value': 5, 'action': 'unordered_iterable_item_removed'}, + {'path': [0], 'value': 7, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [6], 'value': 8, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [6], 'value': 6, 'action': 'unordered_iterable_item_removed', 'type': int}, + {'path': [0], 'value': 5, 'action': 'unordered_iterable_item_removed', 'type': int}, ] flat_expected1 = [FlatDeltaRow(**i) for i in flat_expected1] assert flat_expected1 == flat_result1 @@ -1422,11 +1425,11 @@ def test_list_ignore_order_various_deltas2(self): flat_result2 = delta2.to_flat_rows() flat_expected2 = [ - {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added'}, - {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added'}, - {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added'}, - {'path': [6], 'action': 'values_changed', 'value': 7}, - {'path': [0], 'action': 'values_changed', 'value': 8}, + {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [6], 'action': 'values_changed', 'value': 7, 'type': int}, + {'path': [0], 'action': 'values_changed', 'value': 8, 'type': int}, ] flat_expected2 = [FlatDeltaRow(**i) for i in flat_expected2] assert flat_expected2 == flat_result2 @@ -1565,7 +1568,7 @@ def test_apply_delta_to_incompatible_object6_value_change(self): assert [] == t4 flat_result2 = delta2.to_flat_rows() - flat_expected2 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5}] + flat_expected2 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5, 'type': int}] flat_expected2 = [FlatDeltaRow(**i) for i in flat_expected2] assert flat_expected2 == flat_result2 @@ -1575,7 +1578,7 @@ def test_apply_delta_to_incompatible_object6_value_change(self): delta3 = Delta(diff, raise_errors=False, bidirectional=True) flat_result3 = delta3.to_flat_rows() - flat_expected3 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5, 'old_value': 4}] + flat_expected3 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5, 'old_value': 4, 'type': int, 'old_type': int}] flat_expected3 = [FlatDeltaRow(**i) for i in flat_expected3] assert flat_expected3 == flat_result3 @@ -1685,7 +1688,7 @@ def test_delta_to_dict(self): assert expected == result flat_result = delta.to_flat_rows() - flat_expected = [{'action': 'unordered_iterable_item_removed', 'path': [2], 'value': 'B'}] + flat_expected = [{'action': 'unordered_iterable_item_removed', 'path': [2], 'value': 'B', 'type': str}] flat_expected = [FlatDeltaRow(**i) for i in flat_expected] assert flat_expected == flat_result @@ -1766,10 +1769,10 @@ def test_delta_set_in_objects(self): delta = Delta(DeepDiff(t1, t2)) flat_result = delta.to_flat_rows() flat_expected = [ - {'path': [0, 1], 'value': 10, 'action': 'set_item_added'}, - {'path': [0, 0], 'action': 'values_changed', 'value': 2}, - {'path': [0, 1], 'value': 'A', 'action': 'set_item_removed'}, - {'path': [0, 1], 'value': 'C', 'action': 'set_item_added'}, + {'path': [0, 1], 'value': 10, 'action': 'set_item_added', 'type': int}, + {'path': [0, 0], 'action': 'values_changed', 'value': 2, 'type': int}, + {'path': [0, 1], 'value': 'A', 'action': 'set_item_removed', 'type': str}, + {'path': [0, 1], 'value': 'C', 'action': 'set_item_added', 'type': str}, ] flat_expected = [FlatDeltaRow(**i) for i in flat_expected] @@ -1885,11 +1888,11 @@ def test_compare_func_with_duplicates_removed(self): flat_result = delta.to_flat_rows() flat_expected = [ - {'path': [2], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed'}, - {'path': [0], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed'}, - {'path': [3], 'value': {'id': 3, 'val': 3}, 'action': 'iterable_item_removed'}, - {'path': [0], 'action': 'iterable_item_moved', 'value': {'id': 1, 'val': 3}, 'new_path': [2]}, - {'path': [3], 'action': 'iterable_item_moved', 'value': {'id': 3, 'val': 3}, 'new_path': [0]}, + {'path': [2], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed', 'type': dict}, + {'path': [0], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed', 'type': dict}, + {'path': [3], 'value': {'id': 3, 'val': 3}, 'action': 'iterable_item_removed', 'type': dict}, + {'path': [0], 'action': 'iterable_item_moved', 'value': {'id': 1, 'val': 3}, 'new_path': [2], 'type': dict}, + {'path': [3], 'action': 'iterable_item_moved', 'value': {'id': 3, 'val': 3}, 'new_path': [0], 'type': dict}, ] flat_expected = [FlatDeltaRow(**i) for i in flat_expected] @@ -2289,11 +2292,13 @@ def test_subtract_delta_made_from_flat_dicts1(self): expected_flat_dicts = [{ 'path': ['field_name1', 0], 'value': 'xxx', - 'action': 'iterable_item_removed' + 'action': 'iterable_item_removed', + 'type': str, }, { 'path': ['field_name1', 1], 'value': 'yyy', - 'action': 'iterable_item_removed' + 'action': 'iterable_item_removed', + 'type': str, }] expected_flat_dicts = [FlatDeltaRow(**i) for i in expected_flat_dicts] @@ -2318,11 +2323,13 @@ def test_subtract_delta_made_from_flat_dicts2(self): expected_flat_dicts = [{ 'path': ['field_name1', 0], 'value': 'xxx', - 'action': 'iterable_item_added' + 'action': 'iterable_item_added', + 'type': str, }, { 'path': ['field_name1', 1], 'value': 'yyy', - 'action': 'iterable_item_added' + 'action': 'iterable_item_added', + 'type': str, }] expected_flat_dicts = [FlatDeltaRow(**i) for i in expected_flat_dicts]