Merge pull request #24 from xyngular/josho/featurs-to-support-xdynamo

feat: number of features to support xdynamo.

tests/test_base_model.py

@@ -6,6 +6,7 @@
 from xmodel import JsonModel, Field, XModelError
 from enum import Enum
 
+from xmodel.base.api import Remove
 from xmodel.remote import RemoteModel
 
 
@@ -91,6 +92,22 @@ def test_include_with_fields_option():
     assert m.api.json(only_include_changes=True) == {'a_field': 'a-value-changed', 'b_field': 42}
 
 
+def test_include_removals():
+    original_value = {'a_field': 'a-value', 'b_field': 2}
+    m_obj = MyJModel(original_value)
+
+    # Will return nothing as object was just created.
+    assert m_obj.api.json(only_include_changes=True) is None
+
+    m_obj.a_field = None
+
+    # Will return nothing as object as attribute was only set to be removed/ignored, not changed.
+    assert m_obj.api.json(only_include_changes=True) is None
+
+    # When requesting removals, it should not be included as `Remove`.
+    assert m_obj.api.json(only_include_changes=True, include_removals=True) == {'a_field': Remove}
+
+
 def test_changes_only_embedded_object():
     class JModel(JsonModel):
         field_str: str

tests/test_fields.py

@@ -1,6 +1,10 @@
+from xsentinels import Nullable, Null
+
 from xmodel import Field, JsonModel
 import datetime as dt
 
+from xmodel.base.fields import LowerFilter
+
 
 def test_json_path():
     class TestModel(JsonModel):
@@ -20,3 +24,15 @@ class TestModel(JsonModel):
 
     obj = TestModel(a_datetime_field='2023-04-08T10:11:12Z')
     assert obj.a_datetime_field.isoformat() == '2023-04-08T10:11:12+00:00'
+
+
+def test_lower_filter():
+    class MyModel(JsonModel):
+        filtered_attr: Nullable[str] = Field(post_filter=LowerFilter())
+
+    obj = MyModel()
+    obj.filtered_attr = "HELLO"
+    assert obj.filtered_attr == "hello"
+
+    obj.filtered_attr = Null
+    assert obj.filtered_attr is Null

tests/test_null.py

@@ -1,4 +1,5 @@
-from xsentinels.null import Nullable
+import pytest
+from xsentinels.null import Nullable, Null
 from xmodel import BaseModel
 
 
@@ -17,3 +18,11 @@ def test_nullable():
     assert not non_null_field.nullable
     assert nullable_field.type_hint is str
     assert non_null_field.type_hint is int
+
+
+def test_set_nullable_on_not_nullable_field():
+    model = MyModel()
+    model.nullable_field = Null
+
+    with pytest.raises(AttributeError, match='Null value for field.+does not support NullType'):
+        model.non_nullable = Null

xmodel/base/api.py

@@ -131,6 +131,7 @@ class or model instance.  That way the `BaseApi` instance you use will know the
 from logging import getLogger
 from xmodel.errors import XModelError
 from xsentinels.null import Null, NullType
+from xsentinels import Singleton
 from typing import get_type_hints
 from xinject.context import XContext
 from collections.abc import Mapping
@@ -144,6 +145,25 @@ class or model instance.  That way the `BaseApi` instance you use will know the
 M = TypeVar("M", bound=BaseModel)
 
 
+class RemoveType(Singleton):
+    """
+    Use `Remove`, this is simply the type for the `Remove` sentinel instance.
+    """
+    pass
+
+
+Remove = RemoveType()
+"""
+When requested, Used to indicate something has been removed in the JSON returned from the
+`BaseApi.json` vs what was originally there; for the purposes of determining if there was
+a change.
+
+Only included in the JSON when explicitly requested; not included by default.
+
+"""
+
+
+
 class BaseApi(Generic[M]):
     """
         This class is a sort of "Central Hub" that ties all intrested parties together.
@@ -305,6 +325,18 @@ def model_type(self) -> Type[M]:
         # noinspection PyTypeChecker
         return self.structure.model_cls
 
+    # used as an internal class property
+    _CACHED_TYPE_HINTS = {}
+
+    @classmethod
+    def resolved_type_hints(cls) -> Dict[str, Type]:
+        if hints := BaseApi._CACHED_TYPE_HINTS.get(cls):
+            return hints
+
+        hints = get_type_hints(cls)
+        BaseApi._CACHED_TYPE_HINTS[cls] = hints
+        return hints
+
     # ---------------------------
     # --------- Methods ---------
 
@@ -380,8 +412,8 @@ def __init__(self, *, api: "BaseApi[M]" = None, model: BaseModel = None):
         if model:
             # If we have a model, the structure should be exactly the same as it's BaseModel type.
             self._structure = api.structure
-            self.default_converters = api.default_converters
             self._api_state = PrivateApiState(model=model)
+            self.default_converters = api.default_converters
             return
 
         # If We don't have a BaseModel, then we need to copy the structure, it could change
@@ -390,7 +422,7 @@ def __init__(self, *, api: "BaseApi[M]" = None, model: BaseModel = None):
         # type.
 
         # We lookup the structure type that our associated model-type/class wants to use.
-        structure_type = get_type_hints(type(self)).get(
+        structure_type = type(self).resolved_type_hints().get(
             'structure',
             BaseStructure[Field]
         )
@@ -401,6 +433,7 @@ def __init__(self, *, api: "BaseApi[M]" = None, model: BaseModel = None):
         # We have a root BaseModel with the abstract BaseModel as its super class,
         # in this case we need to allocate a blank structure object.
         # todo: allocate structure with new args
+        # We look up the structure type that our associated model-type/class wants to use.
         existing_struct = api.structure if api else None
         self._structure = structure_type(
             parent=existing_struct,
@@ -515,7 +548,10 @@ def have_changes(self) -> bool:
         return self.json(only_include_changes=True) is not None
 
     def json(
-        self, only_include_changes: bool = False, log_output: bool = False
+        self,
+        only_include_changes: bool = False,
+        log_output: bool = False,
+        include_removals: bool = False
     ) -> Optional[JsonDict]:
         """ REQUIRES associated model object (see `BaseApi.model` for details on this).
 
@@ -550,6 +586,11 @@ def json(
             log_output (bool): If False (default): won't log anything.
                 If True: Logs what method returns at debug level.
 
+           include_removals (bool): If False (default): won't include in response any fields
+                that have been removed
+                (vs when compared to the original JSON that updated this object).
+                The value will be the special sentinel object `Remove`
+                (see top of this module/file for `Remove` object, and it's `RemoveType` class).
 
         Returns:
             JsonDict: Will the needed attributes that should be sent to API.
@@ -683,6 +724,17 @@ def set_value_into_json_dict(value, field_name, *, json=json):
             # Sets field value into a sub-dictionary of the original `json` dict.
             set_value_into_json_dict(v, name, json=d)
 
+        if include_removals:
+            removals = self.fields_to_remove_for_json(json, field_objs)
+            for f in removals:
+                if f in json:
+                    raise XynModelError(
+                        f"Sanity check, we were about to overwrite real value with `Remove` "
+                        f"in json field ({f}) for model ({self.model})."
+                    )
+
+                json[f] = Remove
+
         # If the `last_original_update_json` is None, then we never got update via JSON
         # so there is nothing to compare, include everything!
         if only_include_changes:
@@ -693,7 +745,7 @@ def set_value_into_json_dict(value, field_name, *, json=json):
                 del json[f]
 
             if not json:
-                # If nothing in JSON, then return None.
+                # There were no changes, return None.
                 return None
         else:
             due_to_msg = "unknown"
@@ -719,6 +771,22 @@ def set_value_into_json_dict(value, field_name, *, json=json):
 
         return json
 
+    def fields_to_remove_for_json(self, json: dict, field_objs: List[Field]) -> Set[str]:
+        """
+        Returns set of fields that should be considered 'changed' because they were removed
+        when compared to the original JSON values used to originally update this object.
+
+        The names will be the fields json_path.
+        """
+        fields_to_remove = set()
+        for field in field_objs:
+            # A `None` in the `json` means a null, so we use `Default` as our sentinel type.
+            new_value = json.get(field.json_path, Default)
+            old_value = self._get_old_json_value(field=field.json_path, as_type=type(new_value))
+            if new_value is Default and old_value is not Default:
+                fields_to_remove.add(field.json_path)
+        return fields_to_remove
+
     def fields_to_pop_for_json(
             self, json: dict, field_objs: List[Field], log_output: bool
     ) -> Set[Any]:
@@ -735,7 +803,6 @@ def fields_to_pop_for_json(
         """
         fields_to_pop = set()
         for field, new_value in json.items():
-
             # json has simple strings, numbers, lists, dict;
             # so makes general comparison simpler.
             old_value = self._get_old_json_value(field=field, as_type=type(new_value))
@@ -779,14 +846,26 @@ def fields_to_pop_for_json(
 
     def should_include_field_in_json(self, new_value: Any, old_value: Any, field: str) -> bool:
         """
-        Returns True if the the value for field should be included in the JSON.
+        Returns True if the value for field should be included in the JSON.
         This only gets called if only_include_changes is True when passed to self.json::
 
             # Passed in like so:
             self.json(only_include_changes=True)
 
         This method is an easy way to change the comparison logic.
 
+        `new_value` could be `xyn_types.default.Default`, to indicate that a value's
+        absence is significant (ie: to remove an attribute from destination).
+
+        Most of the time, a value's absence does not affect the destination when object
+        is sent to API/Service because whatever value is currently there for the attribute
+        is left intact/alone.
+
+        But sometimes a service will remove the attribute if it does not exist.
+        When this is the case, the absence of the value is significant for comparison purposes;
+        ie: when deciding if a value has changed.
+
+
         :param new_value: New value that will be put into JSON.
         :param old_value:
             Old value originals in original JSON [normalized if possible to the same type as

xmodel/base/fields.py

@@ -124,6 +124,9 @@ def __call__(self, api: "BaseApi", name: str, value: T) -> T:
 class LowerFilter(Filter):
     """ Lower-cases a value on a `xmodel.base.model.BaseModel` field.
 
+        If the value is false-like, in the case of Null, None or a blank string,
+        it will return that value unaltered.
+
         You can see a real-world example of using this filter on:
         `hubspot.api.Contact.email`
 
@@ -139,6 +142,9 @@ class LowerFilter(Filter):
         >>> assert obj.filtered_attr == "hello"
     """
     def __call__(self, api: "BaseApi", name: str, value: str) -> str:
+        if not value:
+            return value
+
         return value.lower()
 
 

xmodel/base/model.py

@@ -404,21 +404,30 @@ def __setattr__(self, name, value):
         field = structure.get_field(name)
         type_hint = None
 
+        # By default, we go through the special set/converter logic.
+        do_default_attr_set = False
+
         if inspect.isclass(self):
             # If we are a class, just pass it along
-            pass
+            do_default_attr_set = True
         elif name == "api":
             # Don't do anything special with the 'api' var.
-            pass
+            do_default_attr_set = True
         elif name.startswith("_"):
             # don't do anything with private vars
-            pass
+            do_default_attr_set = True
         elif name.endswith("_id") and structure.is_field_a_child(name[:-3], and_has_id=True):
             # We have a virtual field for a related field id, redirect to special setter.
             state = _private.api.get_api_state(api)
             state.set_related_field_id(name[:-3], value)
             return
 
+        if do_default_attr_set:
+            # We don't do anything more if it's a special attribute/field/etc.
+            super().__setattr__(name, value)
+            return
+
+        field = structure.get_field(name)
         if not field:
             # We don't do anything more without a field object
             # (ie: just a normal python attribute of some sort, not tied with API).
@@ -431,7 +440,6 @@ def __setattr__(self, name, value):
             # otherwise an error will be thrown if we can't verify type or auto-convert it.
             type_hint = field.type_hint
             value_type = type(value)
-            field_obj: Field = structure.get_field(name)
 
             # todo: idea: We could cache some of these details [perhaps even using closures]
             #       or use dict/set's someday for a speed improvement, if we ever need to.
@@ -448,7 +456,9 @@ def __setattr__(self, name, value):
 
             state = _private.api.get_api_state(api)
             if (
-                # Check for nullability first, as an optimization.
+                # If we have a blank string, but field is not of type str,
+                # and field is also nullable; we then we convert the value into a Null.
+                # (ie: user is setting a blank-string on a non-string field)
                 field.nullable and
                 type_hint not in [str, None] and
                 value_type is str and
@@ -472,16 +482,17 @@ def __setattr__(self, name, value):
             elif value is Null:
                 # If type_hint supported the Null type, then it would have been dealt with in
                 # the previous if statement.
-                XModelError(
-                    f"Setting a Null value for field ({name}) when typehint ({type_hint}) "
-                    f"does not support NullType, for object ({self})."
-                )
-            elif field_obj.converter:
+                if not field.nullable:
+                    raise AttributeError(
+                        f"Setting a Null value for field ({name}) when typehint ({type_hint}) "
+                        f"does not support NullType, for object ({self})."
+                    )
+            elif field.converter:
                 # todo: Someday map str/int/bool (basic conversions) to standard converter methods;
                 #   kind of like I we do it for date/time... have some default converter methods.
                 #
                 # This handles datetime, date, etc...
-                value = field_obj.converter(api, Converter.Direction.to_model, field_obj, value)
+                value = field.converter(api, Converter.Direction.to_model, field, value)
             elif type_hint in (dict, JsonDict) and value_type in (dict, JsonDict):
                 # this is fine for now, keep it as-is!
                 #
@@ -506,7 +517,7 @@ def __setattr__(self, name, value):
                         basic_type_converter(
                             api,
                             Converter.Direction.to_model,
-                            field_obj,
+                            field,
                             x
                         ) for x in loop(value)
                     ]
@@ -548,8 +559,8 @@ def __setattr__(self, name, value):
             if value.startswith('#########'):
                 value = ''
 
-        if field_obj.post_filter:
-            value = field_obj.post_filter(api=api, name=name, value=value)
+        if field.post_filter:
+            value = field.post_filter(api=api, name=name, value=value)
 
         if field.fset:
             field.fset(self, value)