diff --git a/deepdiff/diff.py b/deepdiff/diff.py index 346f6271..680afb71 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -181,7 +181,7 @@ def __init__(self, self.custom_operators = custom_operators or [] self.ignore_order = ignore_order - self.ignore_order_func = ignore_order_func or (lambda *_args, **_kwargs: ignore_order) + self.ignore_order_func = ignore_order_func ignore_type_in_groups = ignore_type_in_groups or [] if numbers == ignore_type_in_groups or numbers in ignore_type_in_groups: @@ -649,7 +649,7 @@ def _iterables_subscriptable(t1, t2): def _diff_iterable(self, level, parents_ids=frozenset(), _original_type=None, local_tree=None): """Difference of iterables""" - if self.ignore_order_func(level): + if (self.ignore_order_func and self.ignore_order_func(level)) or self.ignore_order: self._diff_iterable_with_deephash(level, parents_ids, _original_type=_original_type, local_tree=local_tree) else: self._diff_iterable_in_order(level, parents_ids, _original_type=_original_type, local_tree=local_tree) @@ -1103,7 +1103,9 @@ def _get_most_in_common_pairs_in_iterables( # And the objects with the same distances are grouped together in an ordered set. # It also includes a "max" key that is just the value of the biggest current distance in the # most_in_common_pairs dictionary. - most_in_common_pairs = defaultdict(lambda: defaultdict(OrderedSetPlus)) + def defaultdict_orderedset(): + return defaultdict(OrderedSetPlus) + most_in_common_pairs = defaultdict(defaultdict_orderedset) pairs = dict_() pre_calced_distances = None @@ -1390,7 +1392,7 @@ def _diff_numpy_array(self, level, parents_ids=frozenset(), local_tree=None): # which means numpy module needs to be available. So np can't be None. raise ImportError(CANT_FIND_NUMPY_MSG) # pragma: no cover - if not self.ignore_order_func(level): + if (self.ignore_order_func and not self.ignore_order_func(level)) or not self.ignore_order: # fast checks if self.significant_digits is None: if np.array_equal(level.t1, level.t2, equal_nan=self.ignore_nan_inequality): @@ -1416,7 +1418,7 @@ def _diff_numpy_array(self, level, parents_ids=frozenset(), local_tree=None): dimensions = len(shape) if dimensions == 1: self._diff_iterable(level, parents_ids, _original_type=_original_type, local_tree=local_tree) - elif self.ignore_order_func(level): + elif (self.ignore_order_func and self.ignore_order_func(level)) or self.ignore_order: # arrays are converted to python lists so that certain features of DeepDiff can apply on them easier. # They will be converted back to Numpy at their final dimension. level.t1 = level.t1.tolist() diff --git a/tests/test_diff_other.py b/tests/test_diff_other.py index e7bc27df..067ee669 100644 --- a/tests/test_diff_other.py +++ b/tests/test_diff_other.py @@ -2,10 +2,27 @@ import datetime from time import sleep from unittest import mock +from functools import partial +from collections import namedtuple +from deepdiff import DeepHash +from deepdiff.helper import pypy3 from deepdiff.model import DiffLevel from deepdiff.diff import ( DeepDiff, PROGRESS_MSG, INVALID_VIEW_MSG, VERBOSE_LEVEL_RANGE_MSG, PURGE_LEVEL_RANGE_MSG) +from concurrent.futures.process import ProcessPoolExecutor +from concurrent.futures import as_completed + +# Only the prep part of DeepHash. We don't need to test the actual hash function. +DeepHashPrep = partial(DeepHash, apply_hash=False) + + +def prep_str(obj, ignore_string_type_changes=True): + return obj if ignore_string_type_changes else 'str:{}'.format(obj) + + +Point = namedtuple('Point', ["x"]) +point_obj = Point(x=11) class SlowDiffLevel(DiffLevel): @@ -120,3 +137,66 @@ def test_bool_str2(self): def test_get_distance_cache_key(self): result = DeepDiff._get_distance_cache_key(added_hash=5, removed_hash=20) assert b'0x14--0x5dc' == result + + def test_multi_processing1(self): + + t1 = [[1, 2, 3, 9], [1, 2, 4, 10]] + t2 = [[1, 2, 4, 10], [1, 2, 3, 10]] + + futures = [] + expected_result = { + 'values_changed': { + 'root[0][2]': { + 'new_value': 4, + 'old_value': 3 + }, + 'root[0][3]': { + 'new_value': 10, + 'old_value': 9 + }, + 'root[1][2]': { + 'new_value': 3, + 'old_value': 4 + } + } + } + + with ProcessPoolExecutor(max_workers=1) as executor: + futures.append(executor.submit(DeepDiff, t1, t2)) + + for future in as_completed(futures, timeout=10): + assert not future._exception + assert expected_result == future._result + + def test_multi_processing2_with_ignore_order(self): + + t1 = [[1, 2, 3, 9], [1, 2, 4, 10]] + t2 = [[1, 2, 4, 10], [1, 2, 3, 10]] + + futures = [] + expected_result = {'values_changed': {'root[0][3]': {'new_value': 10, 'old_value': 9}}} + + with ProcessPoolExecutor(max_workers=1) as executor: + futures.append(executor.submit(DeepDiff, t1, t2, ignore_order=True)) + + for future in as_completed(futures, timeout=10): + assert not future._exception + assert expected_result == future._result + + @pytest.mark.skipif(pypy3, reason="pypy3 expected results are different") + def test_multi_processing3_deephash(self): + x = "x" + x_prep = prep_str(x) + expected_result = { + x: x_prep, + point_obj: "ntPoint:{%s:int:11}" % x, + 11: 'int:11', + } + + futures = [] + with ProcessPoolExecutor(max_workers=1) as executor: + futures.append(executor.submit(DeepHashPrep, point_obj, ignore_string_type_changes=True)) + + for future in as_completed(futures, timeout=10): + assert not future._exception + assert expected_result == future._result