From aff4c40a209b9b0d6566835b6cf52f6cd7343e91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis-Philippe=20V=C3=A9ronneau?= Date: Mon, 11 Apr 2022 15:03:37 -0400 Subject: [PATCH 1/7] Fix testsuite on 32-bits architectures. Fixes #302 --- tests/test_delta.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_delta.py b/tests/test_delta.py index 4c13ec36..c66a48a1 100644 --- a/tests/test_delta.py +++ b/tests/test_delta.py @@ -2,6 +2,7 @@ import os import io import json +import sys from decimal import Decimal from unittest import mock from deepdiff import Delta, DeepDiff @@ -1054,7 +1055,7 @@ def test_ignore_order_delta_cases( 'root[5]': 10 }, '_numpy_paths': { - 'root': 'int64' + 'root': np.where((sys.maxsize > 2**32), 'int64', 'int32') } }, 'expected_result': 't2' From 1c643cba43e88f8ca8ec63b3019e788870e37a0e Mon Sep 17 00:00:00 2001 From: Seperman Date: Wed, 4 May 2022 19:06:35 -0700 Subject: [PATCH 2/7] fixing the issue with Using ignore_order and group_by simultaneously --- deepdiff/diff.py | 1 + tests/test_ignore_order.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/deepdiff/diff.py b/deepdiff/diff.py index e1177f10..81d2bc59 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -231,6 +231,7 @@ def __init__(self, self.progress_logger = progress_logger self.cache_size = cache_size _parameters = self.__dict__.copy() + _parameters['group_by'] = None # overwriting since these parameters will be passed on to other passes. # Non-Root if _shared_parameters: diff --git a/tests/test_ignore_order.py b/tests/test_ignore_order.py index 10f6bf19..0b380e93 100644 --- a/tests/test_ignore_order.py +++ b/tests/test_ignore_order.py @@ -749,7 +749,7 @@ def test_cutoff_distance_for_pairs(self): assert expected == diff_with_dist - def test_ignore_order_and_group_by(self): + def test_ignore_order_and_group_by1(self): t1 = [ {'id': 'AA', 'name': 'Joe', 'ate': ['Nothing']}, {'id': 'BB', 'name': 'James', 'ate': ['Chips', 'Cheese']}, @@ -781,6 +781,13 @@ def test_ignore_order_and_group_by(self): expected2 = {'iterable_item_added': {"root['BB']['ate'][1]": 'Brownies'}} assert expected2 == diff2 + def test_ignore_order_and_group_by2(self): + t1_data = [{'id': '1', 'codes': ['1', '2', '3']}] + t2_data = [{'id': '1', 'codes': ['1', '2', '4']}] + diff = DeepDiff(t1_data, t2_data, group_by='id', ignore_order=True) + expected = {'values_changed': {"root['1']['codes'][2]": {'new_value': '4', 'old_value': '3'}}} + assert expected == diff + class TestCompareFuncIgnoreOrder: From 25546ce8d7ec56f1fe5e0307b71ebc6c4083e76c Mon Sep 17 00:00:00 2001 From: Seperman Date: Wed, 4 May 2022 19:12:46 -0700 Subject: [PATCH 3/7] adding one more test --- tests/test_ignore_order.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_ignore_order.py b/tests/test_ignore_order.py index 0b380e93..a2ae098b 100644 --- a/tests/test_ignore_order.py +++ b/tests/test_ignore_order.py @@ -788,6 +788,41 @@ def test_ignore_order_and_group_by2(self): expected = {'values_changed': {"root['1']['codes'][2]": {'new_value': '4', 'old_value': '3'}}} assert expected == diff + def test_ignore_order_and_group_by3(self): + t1 = [{ + 'id': + '5ec52e', + 'products': [{ + 'lineNumber': 1, + 'productPrice': '2.39', + 'productQuantity': 2 + }, { + 'lineNumber': 2, + 'productPrice': '4.44', + 'productQuantity': 1 + }], + }] + + t2 = [{ + 'id': + '5ec52e', + 'products': [ + { + 'lineNumber': 2, + 'productPrice': '4.44', + 'productQuantity': 1 + }, + { + 'lineNumber': 1, + 'productPrice': '2.39', + 'productQuantity': 2 + }, + ], + }] + + diff = DeepDiff(t1, t2, group_by='id', ignore_order=True) + assert {} == diff + class TestCompareFuncIgnoreOrder: From de3f10533f0ee670b98bc9fc326c4c62070096f1 Mon Sep 17 00:00:00 2001 From: Seperman Date: Thu, 12 May 2022 21:56:03 -0700 Subject: [PATCH 4/7] adding detailed__dict__ to address #312 --- deepdiff/diff.py | 6 +++--- deepdiff/helper.py | 19 +++++++++++++++++++ docs/deephash_doc.rst | 6 ++++++ tests/test_diff_text.py | 30 +++++++++++++++++++++++++++++- 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/deepdiff/diff.py b/deepdiff/diff.py index 81d2bc59..7a2f7af3 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -20,7 +20,7 @@ type_is_subclass_of_type_group, type_in_type_group, get_doc, number_to_string, datetime_normalize, KEY_TO_VAL_STR, booleans, np_ndarray, get_numpy_ndarray_rows, OrderedSetPlus, RepeatedTimer, - TEXT_VIEW, TREE_VIEW, DELTA_VIEW, + TEXT_VIEW, TREE_VIEW, DELTA_VIEW, detailed__dict__, np, get_truncate_datetime, dict_, CannotCompare) from deepdiff.serialization import SerializationMixin from deepdiff.distance import DistanceMixin @@ -394,8 +394,8 @@ def _diff_obj(self, level, parents_ids=frozenset(), t1 = level.t1._asdict() t2 = level.t2._asdict() else: - t1 = level.t1.__dict__ - t2 = level.t2.__dict__ + t1 = detailed__dict__(level.t1) + t2 = detailed__dict__(level.t2) except AttributeError: try: t1 = self._dict_from_slots(level.t1) diff --git a/deepdiff/helper.py b/deepdiff/helper.py index 9c782d23..059d5490 100644 --- a/deepdiff/helper.py +++ b/deepdiff/helper.py @@ -582,3 +582,22 @@ def get_homogeneous_numpy_compatible_type_of_seq(seq): return PYTHON_TYPE_TO_NUMPY_TYPE.get(type_, False) else: return False + + +def detailed__dict__(obj, ignore_private_variables=True): + """ + Get the detailed dictionary of an object. + + This is used so we retrieve object properties too. + """ + result = obj.__dict__.copy() # A shallow copy + for key in dir(obj): + if key not in result and ( + not ignore_private_variables or ( + ignore_private_variables and not key.startswith('__') + ) + ): + value = getattr(obj, key) + if not callable(value): + result[key] = value + return result diff --git a/docs/deephash_doc.rst b/docs/deephash_doc.rst index fabf7165..20f30992 100644 --- a/docs/deephash_doc.rst +++ b/docs/deephash_doc.rst @@ -149,6 +149,12 @@ truncate_datetime: string, default = None If your object is nested, it will build hashes of all the objects it contains too. +.. note:: + DeepHash output is not like conventional hash functions. It is a dictionary of object IDs to their hashes. This happens because DeepHash calculates the hash of the object and any other objects found within the object in a recursive manner. If you only need the hash of the object you are passing, all you need to do is to do: + + >>> DeepHash(obj)[obj] + + **Examples** Let's say you have a dictionary object. diff --git a/tests/test_diff_text.py b/tests/test_diff_text.py index 22e4a6e1..d9dbed15 100755 --- a/tests/test_diff_text.py +++ b/tests/test_diff_text.py @@ -3,6 +3,7 @@ import pytest import logging import uuid +import numpy as np from decimal import Decimal from deepdiff import DeepDiff from deepdiff.helper import pypy3 @@ -569,7 +570,15 @@ class MyEnum(Enum): 'root._value_': { 'old_value': 1, 'new_value': 2 - } + }, + 'root.name': { + 'old_value': 'A', + 'new_value': 'B' + }, + 'root.value': { + 'old_value': 1, + 'new_value': 2 + }, } } assert ddiff == result @@ -1594,3 +1603,22 @@ def test_datetime_in_key(self): expected = {'values_changed': {f'root[{repr(now)}]': {'new_value': 2, 'old_value': 1}}} assert expected == diff + + def test_property_values(self): + + class A: + _thing = 0 + + def __init__(self, a): + self.a = a + + @property + def thing(self): + A._thing += 1 + return A._thing + + diff = DeepDiff(A(1), A(1)) + expected = {'values_changed': {'root._thing': {'new_value': 1, 'old_value': 0}, + 'root.thing': {'new_value': 2, 'old_value': 1}}} + + assert expected == diff From 945a69d97b5d75e150704a58b40b8508d53838e3 Mon Sep 17 00:00:00 2001 From: Seperman Date: Thu, 12 May 2022 22:25:45 -0700 Subject: [PATCH 5/7] better support of private variables --- deepdiff/diff.py | 4 +-- deepdiff/helper.py | 3 +- tests/test_diff_text.py | 39 ++++++++++++++++++++++-- tests/test_ignore_order.py | 61 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 5 deletions(-) diff --git a/deepdiff/diff.py b/deepdiff/diff.py index 7a2f7af3..5572c103 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -394,8 +394,8 @@ def _diff_obj(self, level, parents_ids=frozenset(), t1 = level.t1._asdict() t2 = level.t2._asdict() else: - t1 = detailed__dict__(level.t1) - t2 = detailed__dict__(level.t2) + t1 = detailed__dict__(level.t1, ignore_private_variables=self.ignore_private_variables) + t2 = detailed__dict__(level.t2, ignore_private_variables=self.ignore_private_variables) except AttributeError: try: t1 = self._dict_from_slots(level.t1) diff --git a/deepdiff/helper.py b/deepdiff/helper.py index 059d5490..30bdbe91 100644 --- a/deepdiff/helper.py +++ b/deepdiff/helper.py @@ -591,10 +591,11 @@ def detailed__dict__(obj, ignore_private_variables=True): This is used so we retrieve object properties too. """ result = obj.__dict__.copy() # A shallow copy + private_var_prefix = f"_{obj.__class__.__name__}__" # The semi private variables in Python get this prefix for key in dir(obj): if key not in result and ( not ignore_private_variables or ( - ignore_private_variables and not key.startswith('__') + ignore_private_variables and not key.startswith('__') and not key.startswith(private_var_prefix) ) ): value = getattr(obj, key) diff --git a/tests/test_diff_text.py b/tests/test_diff_text.py index d9dbed15..2d28103b 100755 --- a/tests/test_diff_text.py +++ b/tests/test_diff_text.py @@ -1617,8 +1617,43 @@ def thing(self): A._thing += 1 return A._thing + @property + def __thing2(self): + A._thing += 1 + return A._thing + diff = DeepDiff(A(1), A(1)) - expected = {'values_changed': {'root._thing': {'new_value': 1, 'old_value': 0}, - 'root.thing': {'new_value': 2, 'old_value': 1}}} + expected = { + 'values_changed': { + 'root._thing': { + 'new_value': 1, + 'old_value': 0 + }, + 'root.thing': { + 'new_value': 2, + 'old_value': 1 + } + } + } assert expected == diff + + diff2 = DeepDiff(A(1), A(1), ignore_private_variables=False) + expected2 = { + 'values_changed': { + 'root._A__thing2': { + 'new_value': 5, + 'old_value': 3 + }, + 'root._thing': { + 'new_value': 5, + 'old_value': 3 + }, + 'root.thing': { + 'new_value': 6, + 'old_value': 4 + } + } + } + + assert expected2 == diff2 diff --git a/tests/test_ignore_order.py b/tests/test_ignore_order.py index a2ae098b..52016b3f 100644 --- a/tests/test_ignore_order.py +++ b/tests/test_ignore_order.py @@ -823,6 +823,67 @@ def test_ignore_order_and_group_by3(self): diff = DeepDiff(t1, t2, group_by='id', ignore_order=True) assert {} == diff + def test_ignore_order_and_group_by4(self): + t1 = [ + { + "id": "1", + "field_01": { + "subfield_01": { + "subfield_02": {"subfield_03": "1"}, + } + }, + }, + {"id": "2", "field_01": ["1", "2", "3"]}, + {"id": "3", "field_01": ["1", "2", "3"]}, + ] + t2 = [ + { + "id": "1", + "field_01": { + "subfield_01": { + "subfield_02": {"subfield_03": "2"}, + } + }, + }, + {"id": "2", "field_01": ["4", "5", "6"]}, + {"id": "3", "field_01": ["7", "8", "9"]}, + ] + diff = DeepDiff(t1, t2, group_by='id', ignore_order=True) + expected = { + 'values_changed': { + "root['1']['field_01']['subfield_01']['subfield_02']['subfield_03']": { + 'new_value': '2', + 'old_value': '1' + }, + "root['2']['field_01'][1]": { + 'new_value': '5', + 'old_value': '2' + }, + "root['3']['field_01'][2]": { + 'new_value': '9', + 'old_value': '3' + }, + "root['2']['field_01'][0]": { + 'new_value': '4', + 'old_value': '1' + }, + "root['3']['field_01'][1]": { + 'new_value': '8', + 'old_value': '2' + }, + "root['3']['field_01'][0]": { + 'new_value': '7', + 'old_value': '1' + }, + "root['2']['field_01'][2]": { + 'new_value': '6', + 'old_value': '3' + } + } + } + + assert expected == diff + class TestCompareFuncIgnoreOrder: From 4ab5fb5680ff3e0646a226cb90475a7d82c9fe6b Mon Sep 17 00:00:00 2001 From: Seperman Date: Thu, 12 May 2022 22:52:41 -0700 Subject: [PATCH 6/7] =?UTF-8?q?Bump=20version:=205.8.0=20=E2=86=92=205.8.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 30 +++++++++++++++--------------- deepdiff/__init__.py | 2 +- docs/conf.py | 4 ++-- docs/index.rst | 2 +- setup.cfg | 2 +- setup.py | 2 +- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index f857a87b..546af03d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DeepDiff v 5.8.0 +# DeepDiff v 5.8.1 ![Downloads](https://img.shields.io/pypi/dm/deepdiff.svg?style=flat) ![Python Versions](https://img.shields.io/pypi/pyversions/deepdiff.svg?style=flat) @@ -14,7 +14,7 @@ Tested on Python 3.6+ and PyPy3. -- **[Documentation](https://zepworks.com/deepdiff/5.8.0/)** +- **[Documentation](https://zepworks.com/deepdiff/5.8.1/)** ## What is new? @@ -61,13 +61,13 @@ Note: if you want to use DeepDiff via commandline, make sure to run `pip install DeepDiff gets the difference of 2 objects. -> - Please take a look at the [DeepDiff docs](https://zepworks.com/deepdiff/5.8.0/diff.html) -> - The full documentation of all modules can be found on +> - Please take a look at the [DeepDiff docs](https://zepworks.com/deepdiff/5.8.1/diff.html) +> - The full documentation of all modules can be found on > - Tutorials and posts about DeepDiff can be found on ## A few Examples -> Note: This is just a brief overview of what DeepDiff can do. Please visit for full documentation. +> Note: This is just a brief overview of what DeepDiff can do. Please visit for full documentation. ### List difference ignoring order or duplicates @@ -271,8 +271,8 @@ Example: ``` -> - Please take a look at the [DeepDiff docs](https://zepworks.com/deepdiff/5.8.0/diff.html) -> - The full documentation can be found on +> - Please take a look at the [DeepDiff docs](https://zepworks.com/deepdiff/5.8.1/diff.html) +> - The full documentation can be found on # Deep Search @@ -304,8 +304,8 @@ And you can pass all the same kwargs as DeepSearch to grep too: {'matched_paths': {"root['somewhere']": 'around'}, 'matched_values': {"root['long']": 'somewhere'}} ``` -> - Please take a look at the [DeepSearch docs](https://zepworks.com/deepdiff/5.8.0/dsearch.html) -> - The full documentation can be found on +> - Please take a look at the [DeepSearch docs](https://zepworks.com/deepdiff/5.8.1/dsearch.html) +> - The full documentation can be found on # Deep Hash (New in v4-0-0) @@ -313,8 +313,8 @@ And you can pass all the same kwargs as DeepSearch to grep too: DeepHash is designed to give you hash of ANY python object based on its contents even if the object is not considered hashable! DeepHash is supposed to be deterministic in order to make sure 2 objects that contain the same data, produce the same hash. -> - Please take a look at the [DeepHash docs](https://zepworks.com/deepdiff/5.8.0/deephash.html) -> - The full documentation can be found on +> - Please take a look at the [DeepHash docs](https://zepworks.com/deepdiff/5.8.1/deephash.html) +> - The full documentation can be found on Let's say you have a dictionary object. @@ -362,8 +362,8 @@ Which you can write as: At first it might seem weird why DeepHash(obj)[obj] but remember that DeepHash(obj) is a dictionary of hashes of all other objects that obj contains too. -> - Please take a look at the [DeepHash docs](https://zepworks.com/deepdiff/5.8.0/deephash.html) -> - The full documentation can be found on +> - Please take a look at the [DeepHash docs](https://zepworks.com/deepdiff/5.8.1/deephash.html) +> - The full documentation can be found on # Using DeepDiff in unit tests @@ -443,11 +443,11 @@ Thank you! How to cite this library (APA style): - Dehpour, S. (2022). DeepDiff (Version 5.8.0) [Software]. Available from https://github.com/seperman/deepdiff. + Dehpour, S. (2022). DeepDiff (Version 5.8.1) [Software]. Available from https://github.com/seperman/deepdiff. How to cite this library (Chicago style): - Dehpour, Sep. 2022. DeepDiff (version 5.8.0). + Dehpour, Sep. 2022. DeepDiff (version 5.8.1). # Authors diff --git a/deepdiff/__init__.py b/deepdiff/__init__.py index d4e66b22..cf77ac23 100644 --- a/deepdiff/__init__.py +++ b/deepdiff/__init__.py @@ -1,6 +1,6 @@ """This module offers the DeepDiff, DeepSearch, grep, Delta and DeepHash classes.""" # flake8: noqa -__version__ = '5.8.0' +__version__ = '5.8.1' import logging if __name__ == '__main__': diff --git a/docs/conf.py b/docs/conf.py index 215c0b5b..25221941 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '5.8.0' +version = '5.8.1' # The full version, including alpha/beta/rc tags. -release = '5.8.0' +release = '5.8.1' load_dotenv(override=True) DOC_VERSION = os.environ.get('DOC_VERSION', version) diff --git a/docs/index.rst b/docs/index.rst index d751749a..8c90fa79 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ contain the root `toctree` directive. -DeepDiff 5.8.0 documentation! +DeepDiff 5.8.1 documentation! ============================= ***************** diff --git a/setup.cfg b/setup.cfg index a9dfdd63..045f7cf5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.8.0 +current_version = 5.8.1 commit = True tag = True tag_name = {new_version} diff --git a/setup.py b/setup.py index c9b5a53b..555c6a66 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ if os.environ.get('USER', '') == 'vagrant': del os.link -version = '5.8.0' +version = '5.8.1' def get_reqs(filename): From e3c2ba1afae1a495f1ec79f9831f7971dc0b638a Mon Sep 17 00:00:00 2001 From: Seperman Date: Thu, 12 May 2022 23:02:40 -0700 Subject: [PATCH 7/7] updating docs --- README.md | 6 ++++++ docs/index.rst | 20 +++++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 546af03d..abc1dfa8 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,12 @@ Tested on Python 3.6+ and PyPy3. ## What is new? +DeepDiff 5-8-1 includes bug fixes: +- Fixed test suite for 32bit systems (https://github.com/seperman/deepdiff/issues/302) by [Louis-Philippe Véronneau](https://github.com/baldurmen) +- Fixed the issue when using `ignore_order=True` and `group_by` simultaneously +- Added the support for diffing object properties (`@property`) (https://github.com/seperman/deepdiff/issues/312) +- Better support of diffing private variables + DeepDiff 5-8-0 includes bug fixes and improvements: - Fixed the bug with delta randomly not producing the same results when `ignore_order=True` (https://github.com/seperman/deepdiff/issues/277) diff --git a/docs/index.rst b/docs/index.rst index 8c90fa79..60e799ba 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,19 @@ The DeepDiff library includes the following modules: What is New *********** +New In DeepDiff 5-8-1 +--------------------- + +DeepDiff 5-8-1 includes bug fixes: - Fixed test suite for 32bit systems +(https://github.com/seperman/deepdiff/issues/302) by `Louis-Philippe +Véronneau`_ - Fixed the issue when using ``ignore_order=True`` and +``group_by`` simultaneously - Added the support for diffing object +properties (``@property``) +(https://github.com/seperman/deepdiff/issues/312) - Better support of +diffing private variables + +.. _Louis-Philippe Véronneau: https://github.com/baldurmen + New In DeepDiff 5-8-0 --------------------- @@ -45,13 +58,6 @@ DeepDiff 5-8-0 includes bug fixes and improvements: - Fixed TypeError is thrown when comparing bool and str -New In DeepDiff 5-7-0 ---------------------- - -- https://github.com/seperman/deepdiff/pull/284 Bug-Fix: TypeError in _get_numbers_distance() when ignore_order = True by @Dhanvantari -- https://github.com/seperman/deepdiff/pull/280 Add support for UUIDs by @havardthom -- Major bug in delta when it comes to iterable items added or removed is investigated by @uwefladrich and resolved by @seperman - ********* Tutorials *********