From 2abd1d7a103e205b50ada9b1db54b52bdda9952b Mon Sep 17 00:00:00 2001 From: Josh Orr Date: Wed, 27 Sep 2023 08:15:28 -0600 Subject: [PATCH] feat: number of features to support xdynamo. --- tests/test_base_model.py | 17 ++++++++ tests/test_fields.py | 16 +++++++ tests/test_null.py | 11 ++++- xmodel/base/api.py | 91 +++++++++++++++++++++++++++++++++++++--- xmodel/base/fields.py | 6 +++ xmodel/base/model.py | 39 ++++++++++------- xmodel/remote/api.py | 21 +++++++--- 7 files changed, 175 insertions(+), 26 deletions(-) diff --git a/tests/test_base_model.py b/tests/test_base_model.py index 6e8029b..8cd1c92 100644 --- a/tests/test_base_model.py +++ b/tests/test_base_model.py @@ -6,6 +6,7 @@ from xmodel import JsonModel, Field, XModelError from enum import Enum +from xmodel.base.api import Remove from xmodel.remote import RemoteModel @@ -91,6 +92,22 @@ def test_include_with_fields_option(): assert m.api.json(only_include_changes=True) == {'a_field': 'a-value-changed', 'b_field': 42} +def test_include_removals(): + original_value = {'a_field': 'a-value', 'b_field': 2} + m_obj = MyJModel(original_value) + + # Will return nothing as object was just created. + assert m_obj.api.json(only_include_changes=True) is None + + m_obj.a_field = None + + # Will return nothing as object as attribute was only set to be removed/ignored, not changed. + assert m_obj.api.json(only_include_changes=True) is None + + # When requesting removals, it should not be included as `Remove`. + assert m_obj.api.json(only_include_changes=True, include_removals=True) == {'a_field': Remove} + + def test_changes_only_embedded_object(): class JModel(JsonModel): field_str: str diff --git a/tests/test_fields.py b/tests/test_fields.py index 4787bc2..e83aaf5 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,6 +1,10 @@ +from xsentinels import Nullable, Null + from xmodel import Field, JsonModel import datetime as dt +from xmodel.base.fields import LowerFilter + def test_json_path(): class TestModel(JsonModel): @@ -20,3 +24,15 @@ class TestModel(JsonModel): obj = TestModel(a_datetime_field='2023-04-08T10:11:12Z') assert obj.a_datetime_field.isoformat() == '2023-04-08T10:11:12+00:00' + + +def test_lower_filter(): + class MyModel(JsonModel): + filtered_attr: Nullable[str] = Field(post_filter=LowerFilter()) + + obj = MyModel() + obj.filtered_attr = "HELLO" + assert obj.filtered_attr == "hello" + + obj.filtered_attr = Null + assert obj.filtered_attr is Null diff --git a/tests/test_null.py b/tests/test_null.py index bf3a8c5..264d89b 100644 --- a/tests/test_null.py +++ b/tests/test_null.py @@ -1,4 +1,5 @@ -from xsentinels.null import Nullable +import pytest +from xsentinels.null import Nullable, Null from xmodel import BaseModel @@ -17,3 +18,11 @@ def test_nullable(): assert not non_null_field.nullable assert nullable_field.type_hint is str assert non_null_field.type_hint is int + + +def test_set_nullable_on_not_nullable_field(): + model = MyModel() + model.nullable_field = Null + + with pytest.raises(AttributeError, match='Null value for field.+does not support NullType'): + model.non_nullable = Null diff --git a/xmodel/base/api.py b/xmodel/base/api.py index 2cc2387..92b7d6a 100644 --- a/xmodel/base/api.py +++ b/xmodel/base/api.py @@ -131,6 +131,7 @@ class or model instance. That way the `BaseApi` instance you use will know the from logging import getLogger from xmodel.errors import XModelError from xsentinels.null import Null, NullType +from xsentinels import Singleton from typing import get_type_hints from xinject.context import XContext from collections.abc import Mapping @@ -144,6 +145,25 @@ class or model instance. That way the `BaseApi` instance you use will know the M = TypeVar("M", bound=BaseModel) +class RemoveType(Singleton): + """ + Use `Remove`, this is simply the type for the `Remove` sentinel instance. + """ + pass + + +Remove = RemoveType() +""" +When requested, Used to indicate something has been removed in the JSON returned from the +`BaseApi.json` vs what was originally there; for the purposes of determining if there was +a change. + +Only included in the JSON when explicitly requested; not included by default. + +""" + + + class BaseApi(Generic[M]): """ This class is a sort of "Central Hub" that ties all intrested parties together. @@ -305,6 +325,18 @@ def model_type(self) -> Type[M]: # noinspection PyTypeChecker return self.structure.model_cls + # used as an internal class property + _CACHED_TYPE_HINTS = {} + + @classmethod + def resolved_type_hints(cls) -> Dict[str, Type]: + if hints := BaseApi._CACHED_TYPE_HINTS.get(cls): + return hints + + hints = get_type_hints(cls) + BaseApi._CACHED_TYPE_HINTS[cls] = hints + return hints + # --------------------------- # --------- Methods --------- @@ -380,8 +412,8 @@ def __init__(self, *, api: "BaseApi[M]" = None, model: BaseModel = None): if model: # If we have a model, the structure should be exactly the same as it's BaseModel type. self._structure = api.structure - self.default_converters = api.default_converters self._api_state = PrivateApiState(model=model) + self.default_converters = api.default_converters return # If We don't have a BaseModel, then we need to copy the structure, it could change @@ -390,7 +422,7 @@ def __init__(self, *, api: "BaseApi[M]" = None, model: BaseModel = None): # type. # We lookup the structure type that our associated model-type/class wants to use. - structure_type = get_type_hints(type(self)).get( + structure_type = type(self).resolved_type_hints().get( 'structure', BaseStructure[Field] ) @@ -401,6 +433,7 @@ def __init__(self, *, api: "BaseApi[M]" = None, model: BaseModel = None): # We have a root BaseModel with the abstract BaseModel as its super class, # in this case we need to allocate a blank structure object. # todo: allocate structure with new args + # We look up the structure type that our associated model-type/class wants to use. existing_struct = api.structure if api else None self._structure = structure_type( parent=existing_struct, @@ -515,7 +548,10 @@ def have_changes(self) -> bool: return self.json(only_include_changes=True) is not None def json( - self, only_include_changes: bool = False, log_output: bool = False + self, + only_include_changes: bool = False, + log_output: bool = False, + include_removals: bool = False ) -> Optional[JsonDict]: """ REQUIRES associated model object (see `BaseApi.model` for details on this). @@ -550,6 +586,11 @@ def json( log_output (bool): If False (default): won't log anything. If True: Logs what method returns at debug level. + include_removals (bool): If False (default): won't include in response any fields + that have been removed + (vs when compared to the original JSON that updated this object). + The value will be the special sentinel object `Remove` + (see top of this module/file for `Remove` object, and it's `RemoveType` class). Returns: JsonDict: Will the needed attributes that should be sent to API. @@ -683,6 +724,17 @@ def set_value_into_json_dict(value, field_name, *, json=json): # Sets field value into a sub-dictionary of the original `json` dict. set_value_into_json_dict(v, name, json=d) + if include_removals: + removals = self.fields_to_remove_for_json(json, field_objs) + for f in removals: + if f in json: + raise XynModelError( + f"Sanity check, we were about to overwrite real value with `Remove` " + f"in json field ({f}) for model ({self.model})." + ) + + json[f] = Remove + # If the `last_original_update_json` is None, then we never got update via JSON # so there is nothing to compare, include everything! if only_include_changes: @@ -693,7 +745,7 @@ def set_value_into_json_dict(value, field_name, *, json=json): del json[f] if not json: - # If nothing in JSON, then return None. + # There were no changes, return None. return None else: due_to_msg = "unknown" @@ -719,6 +771,22 @@ def set_value_into_json_dict(value, field_name, *, json=json): return json + def fields_to_remove_for_json(self, json: dict, field_objs: List[Field]) -> Set[str]: + """ + Returns set of fields that should be considered 'changed' because they were removed + when compared to the original JSON values used to originally update this object. + + The names will be the fields json_path. + """ + fields_to_remove = set() + for field in field_objs: + # A `None` in the `json` means a null, so we use `Default` as our sentinel type. + new_value = json.get(field.json_path, Default) + old_value = self._get_old_json_value(field=field.json_path, as_type=type(new_value)) + if new_value is Default and old_value is not Default: + fields_to_remove.add(field.json_path) + return fields_to_remove + def fields_to_pop_for_json( self, json: dict, field_objs: List[Field], log_output: bool ) -> Set[Any]: @@ -735,7 +803,6 @@ def fields_to_pop_for_json( """ fields_to_pop = set() for field, new_value in json.items(): - # json has simple strings, numbers, lists, dict; # so makes general comparison simpler. old_value = self._get_old_json_value(field=field, as_type=type(new_value)) @@ -779,7 +846,7 @@ def fields_to_pop_for_json( def should_include_field_in_json(self, new_value: Any, old_value: Any, field: str) -> bool: """ - Returns True if the the value for field should be included in the JSON. + Returns True if the value for field should be included in the JSON. This only gets called if only_include_changes is True when passed to self.json:: # Passed in like so: @@ -787,6 +854,18 @@ def should_include_field_in_json(self, new_value: Any, old_value: Any, field: st This method is an easy way to change the comparison logic. + `new_value` could be `xyn_types.default.Default`, to indicate that a value's + absence is significant (ie: to remove an attribute from destination). + + Most of the time, a value's absence does not affect the destination when object + is sent to API/Service because whatever value is currently there for the attribute + is left intact/alone. + + But sometimes a service will remove the attribute if it does not exist. + When this is the case, the absence of the value is significant for comparison purposes; + ie: when deciding if a value has changed. + + :param new_value: New value that will be put into JSON. :param old_value: Old value originals in original JSON [normalized if possible to the same type as diff --git a/xmodel/base/fields.py b/xmodel/base/fields.py index 58b14eb..5d65c53 100644 --- a/xmodel/base/fields.py +++ b/xmodel/base/fields.py @@ -124,6 +124,9 @@ def __call__(self, api: "BaseApi", name: str, value: T) -> T: class LowerFilter(Filter): """ Lower-cases a value on a `xmodel.base.model.BaseModel` field. + If the value is false-like, in the case of Null, None or a blank string, + it will return that value unaltered. + You can see a real-world example of using this filter on: `hubspot.api.Contact.email` @@ -139,6 +142,9 @@ class LowerFilter(Filter): >>> assert obj.filtered_attr == "hello" """ def __call__(self, api: "BaseApi", name: str, value: str) -> str: + if not value: + return value + return value.lower() diff --git a/xmodel/base/model.py b/xmodel/base/model.py index f6c7638..a113e71 100644 --- a/xmodel/base/model.py +++ b/xmodel/base/model.py @@ -404,21 +404,30 @@ def __setattr__(self, name, value): field = structure.get_field(name) type_hint = None + # By default, we go through the special set/converter logic. + do_default_attr_set = False + if inspect.isclass(self): # If we are a class, just pass it along - pass + do_default_attr_set = True elif name == "api": # Don't do anything special with the 'api' var. - pass + do_default_attr_set = True elif name.startswith("_"): # don't do anything with private vars - pass + do_default_attr_set = True elif name.endswith("_id") and structure.is_field_a_child(name[:-3], and_has_id=True): # We have a virtual field for a related field id, redirect to special setter. state = _private.api.get_api_state(api) state.set_related_field_id(name[:-3], value) return + if do_default_attr_set: + # We don't do anything more if it's a special attribute/field/etc. + super().__setattr__(name, value) + return + + field = structure.get_field(name) if not field: # We don't do anything more without a field object # (ie: just a normal python attribute of some sort, not tied with API). @@ -431,7 +440,6 @@ def __setattr__(self, name, value): # otherwise an error will be thrown if we can't verify type or auto-convert it. type_hint = field.type_hint value_type = type(value) - field_obj: Field = structure.get_field(name) # todo: idea: We could cache some of these details [perhaps even using closures] # or use dict/set's someday for a speed improvement, if we ever need to. @@ -448,7 +456,9 @@ def __setattr__(self, name, value): state = _private.api.get_api_state(api) if ( - # Check for nullability first, as an optimization. + # If we have a blank string, but field is not of type str, + # and field is also nullable; we then we convert the value into a Null. + # (ie: user is setting a blank-string on a non-string field) field.nullable and type_hint not in [str, None] and value_type is str and @@ -472,16 +482,17 @@ def __setattr__(self, name, value): elif value is Null: # If type_hint supported the Null type, then it would have been dealt with in # the previous if statement. - XModelError( - f"Setting a Null value for field ({name}) when typehint ({type_hint}) " - f"does not support NullType, for object ({self})." - ) - elif field_obj.converter: + if not field.nullable: + raise AttributeError( + f"Setting a Null value for field ({name}) when typehint ({type_hint}) " + f"does not support NullType, for object ({self})." + ) + elif field.converter: # todo: Someday map str/int/bool (basic conversions) to standard converter methods; # kind of like I we do it for date/time... have some default converter methods. # # This handles datetime, date, etc... - value = field_obj.converter(api, Converter.Direction.to_model, field_obj, value) + value = field.converter(api, Converter.Direction.to_model, field, value) elif type_hint in (dict, JsonDict) and value_type in (dict, JsonDict): # this is fine for now, keep it as-is! # @@ -506,7 +517,7 @@ def __setattr__(self, name, value): basic_type_converter( api, Converter.Direction.to_model, - field_obj, + field, x ) for x in loop(value) ] @@ -548,8 +559,8 @@ def __setattr__(self, name, value): if value.startswith('#########'): value = '' - if field_obj.post_filter: - value = field_obj.post_filter(api=api, name=name, value=value) + if field.post_filter: + value = field.post_filter(api=api, name=name, value=value) if field.fset: field.fset(self, value) diff --git a/xmodel/remote/api.py b/xmodel/remote/api.py index 5d84288..8df4169 100644 --- a/xmodel/remote/api.py +++ b/xmodel/remote/api.py @@ -1,4 +1,5 @@ import dataclasses +from decimal import Decimal from logging import getLogger from typing import ( TypeVar, Type, get_type_hints, Union, List, Dict, Iterable, Set, Optional, Generic, Mapping, @@ -238,8 +239,14 @@ def get_via_id( if id is None: return None - structure = self.structure + value_type = type(id) + + # Treat a Decimal as a string for the purposes of querying for it. + if type(id) is Decimal: + id = str(id) + value_type = str + structure = self.structure max_query_by_id = structure.max_query_by_id # todo: Someday, adjust this to only iterate on id as needed, ie: get the first @@ -275,7 +282,6 @@ def get_via_id( f"in order to currently be used in `get_via_id` method at the moment." ) - value_type = type(id) result_is_list = False if typing_inspect.is_union_type(field_type): @@ -501,7 +507,10 @@ def get( # --------- Things REQUIRING an Associated BaseModel ----- def json( - self, only_include_changes: bool = False, log_output: bool = False + self, + only_include_changes: bool = False, + log_output: bool = False, + include_removals: bool = False ) -> Optional[JsonDict]: """ `xmodel.base.api.BaseApi.json` to see superclass's documentation for this method. @@ -518,7 +527,7 @@ def json( if only_include_changes and not have_id_value: only_include_changes = False - json = super().json(only_include_changes, log_output) + json = super().json(only_include_changes, log_output, include_removals) if have_id_value and json: # todo: Check to see if we have 'id' already? Also, use the 'id' field's converter! @@ -526,8 +535,10 @@ def json( json['id'] = model.id if only_include_changes and json: - fields_to_pop = self.fields_to_pop_for_json(json, self.structure.fields, log_output) + fields_to_pop = self.fields_to_pop_for_json(json, self.structure.fields, False) + # Determine if we need to include the 'id' or not + # (which we always do, unless there are no other changes when only_include_changes) have_usable_id = self.structure.has_id_field() id_is_same = False for f in fields_to_pop: