fix(serializers): message_to_data should have a special model handling

django_socio_grpc/proto_serializers.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, MutableSequence
+from typing import Any, Dict, MutableSequence
 
 from asgiref.sync import sync_to_async
 from django.core.validators import MaxLengthValidator
@@ -57,89 +57,8 @@ def __init__(self, *args, **kwargs):
 
     def message_to_data(self, message):
         """Protobuf message -> Dict of python primitive datatypes."""
-        data_dict = message_to_dict(message)
-        data_dict = self._populate_dict_with_none_if_not_required(data_dict, message=message)
-        return data_dict
-
-    def _populate_dict_with_none_if_not_required(self, data_dict, message=None):
-        """
-        This method allow to populate the data dictionary with None for optional field that allow_null and not send in the request.
-        It's also allow to deal with partial update correctly.
-        This is mandatory for having null value received in request as DRF expect to have None value for field that are required.
-        We can't rely only on required True/False as in DSG if a field is required it will have the default value of it's type (empty string for string type) and not None
-
-        When refactoring serializer to only use message we will be able to determine the default value of the field depending of the same logic followed here
-
-        set default value for field except if optional or partial update
-        """
-        # INFO - AM - 04/01/2024 - If we are in a partial serializer with a message we need to have the PARTIAL_UPDATE_FIELD_NAME in the data_dict. If not we raise an exception
-        if self.partial and PARTIAL_UPDATE_FIELD_NAME not in data_dict:
-            raise ValidationError(
-                {
-                    PARTIAL_UPDATE_FIELD_NAME: [
-                        f"Field {PARTIAL_UPDATE_FIELD_NAME} not set in message when using partial=True"
-                    ]
-                },
-                code="missing_partial_message_attribute",
-            )
-
-        is_update_process = (
-            hasattr(self.Meta, "model") and get_model_pk(self.Meta.model).name in data_dict
-        )
-        clean_dict = {}
-        partial_fields: list[str] = data_dict.get(PARTIAL_UPDATE_FIELD_NAME, [])
-        for field in self.fields.values():
-            # INFO - AM - 04/01/2024 - If we are in a partial serializer we only
-            # need to have field specified in PARTIAL_UPDATE_FIELD_NAME attribute
-            # in the data. Meaning deleting fields that should not be here and not adding
-            # None to allow_null field that are not specified
-            if self.partial and field.field_name not in partial_fields:
-                continue
-
-            if field.field_name in data_dict:
-                clean_dict[field.field_name] = data_dict[field.field_name]
-                continue
-
-            try:
-                clean_dict[field.field_name] = self._get_cleaned_data(
-                    partial_fields, field, is_update_process, message
-                )
-            except _NoDictData:
-                continue
-
-        return clean_dict
-
-    def _get_cleaned_data(
-        self,
-        partial_fields: List[str],
-        field: Field,
-        is_update_process: bool,
-        message,
-    ):
-        # INFO - AM - 04/01/2024 - if field is not in the data_dict but in PARTIAL_UPDATE_FIELD_NAME
-        # we need to set the default value if existing or raise exception to avoid having default
-        # grpc value by mistake
-        if field.field_name in partial_fields:
-            if field.allow_null:
-                return None
-            elif field.default not in [None, empty]:
-                return get_default_value(field.default)
-            # INFO - AM - 11/03/2024 - Here we set the default value especially for the blank authorized data.
-            # We debated about raising a ValidaitonError but prefered this behavior.
-            # Can be changed if it create issue with users
-            return message.DESCRIPTOR.fields_by_name[field.field_name].default_value
-
-        elif field.allow_null or (field.default in [None, empty] and field.required is True):
-            if is_update_process or field.default not in [None, empty]:
-                return None
-            try:
-                deferred_attribute = getattr(self.Meta.model, field.field_name)
-                if deferred_attribute.field.default != NOT_PROVIDED:
-                    return get_default_value(deferred_attribute.field.default)
-            except AttributeError:
-                return None
-
-        raise _NoDictData
+        message_to_data = self._MessageToData(message, self)
+        return message_to_data()
 
     def data_to_message(self, data):
         """Protobuf message <- Dict of python primitive datatypes."""
@@ -202,6 +121,93 @@ def to_proto_message(self):
             "If you want to use BaseProtoSerializer instead of ProtoSerializer you need to implement 'to_proto_message' method as there is no fields to introspect from. Please read the documentation"
         )
 
+    class _MessageToData:
+        def __init__(self, message, serializer):
+            self.message = message
+            self.serializer: Serializer = serializer
+            self.base_data = message_to_dict(message)
+
+        @property
+        def partial_fields(self):
+            return self.base_data.get(PARTIAL_UPDATE_FIELD_NAME, [])
+
+        def __call__(self) -> Dict[str, Any]:
+            return self._populate_dict_with_none_if_not_required()
+
+        def _populate_dict_with_none_if_not_required(self):
+            """
+            This method allow to populate the data dictionary with None for optional field that allow_null and not send in the request.
+            It's also allow to deal with partial update correctly.
+            This is mandatory for having null value received in request as DRF expect to have None value for field that are required.
+            We can't rely only on required True/False as in DSG if a field is required it will have the default value of it's type (empty string for string type) and not None
+
+            When refactoring serializer to only use message we will be able to determine the default value of the field depending of the same logic followed here
+
+            set default value for field except if optional or partial update
+            """
+            # INFO - AM - 04/01/2024 - If we are in a partial serializer with a message we need to have the PARTIAL_UPDATE_FIELD_NAME in the data_dict. If not we raise an exception
+            if self.serializer.partial and PARTIAL_UPDATE_FIELD_NAME not in self.base_data:
+                raise ValidationError(
+                    {
+                        PARTIAL_UPDATE_FIELD_NAME: [
+                            f"Field {PARTIAL_UPDATE_FIELD_NAME} not set in message when using partial=True"
+                        ]
+                    },
+                    code="missing_partial_message_attribute",
+                )
+
+            cleaned_data = {}
+
+            for name, field in self.serializer.fields.items():
+                try:
+                    cleaned_data[name] = self.get_cleaned_field_value(field)
+                except _NoDictData:
+                    continue
+
+            return cleaned_data
+
+        def get_nullable_field_value(self, field: Field):
+            if field.allow_null or (field.default in [None, empty] and field.required):
+                return None
+
+            raise _NoDictData
+
+        def get_partial_field_value(self, field: Field):
+            # INFO - AM - 04/01/2024 - if field is not in the data_dict but in PARTIAL_UPDATE_FIELD_NAME
+            # we need to set the default value if existing or raise exception to avoid having default
+            # grpc value by mistake
+            if field.field_name in self.partial_fields:
+                if field.allow_null:
+                    return None
+                elif field.default not in [None, empty]:
+                    return get_default_value(field.default)
+                # INFO - AM - 11/03/2024 - Here we set the default value especially for the blank authorized data.
+                # We debated about raising a ValidationError but prefered this behavior.
+                # Can be changed if it create issue with users
+                return self.message.DESCRIPTOR.fields_by_name[field.field_name].default_value
+            raise _NoDictData
+
+        def get_cleaned_field_value(
+            self,
+            field: Field,
+        ):
+            # INFO - AM - 04/01/2024 - If we are in a partial serializer we only
+            # need to have field specified in PARTIAL_UPDATE_FIELD_NAME attribute
+            # in the data. Meaning deleting fields that should not be here and not adding
+            # None to allow_null field that are not specified
+            if self.serializer.partial and field.field_name not in self.partial_fields:
+                raise _NoDictData
+
+            if field.field_name in self.base_data:
+                return self.base_data[field.field_name]
+
+            try:
+                return self.get_partial_field_value(field)
+            except _NoDictData:
+                pass
+
+            return self.get_nullable_field_value(field)
+
 
 class ProtoSerializer(BaseProtoSerializer, Serializer):
     pass
@@ -284,6 +290,29 @@ def build_property_field(self, field_name, model_class):
 
         return field_class, field_kwargs
 
+    class _MessageToData(ProtoSerializer._MessageToData):
+        @property
+        def model(self):
+            return self.serializer.Meta.model
+
+        @property
+        def updating(self):
+            return get_model_pk(self.model).name in self.base_data
+
+        def get_nullable_field_value(
+            self,
+            field: Field,
+        ):
+            value = super().get_nullable_field_value(field)
+            if self.updating or field.default not in [None, empty]:
+                return value
+            try:
+                deferred_attribute = getattr(self.model, field.field_name)
+                if deferred_attribute.field.default != NOT_PROVIDED:
+                    return get_default_value(deferred_attribute.field.default)
+            except AttributeError:
+                return value
+
 
 class BinaryField(Field):
     default_error_messages = {

django_socio_grpc/tests/fakeapp/grpc/fakeapp.proto

@@ -18,6 +18,7 @@ service BasicController {
     rpc MyMethod(CustomNameForRequest) returns (CustomNameForResponse) {}
     rpc TestBaseProtoSerializer(BaseProtoExampleRequest) returns (BaseProtoExampleListResponse) {}
     rpc TestEmptyMethod(google.protobuf.Empty) returns (google.protobuf.Empty) {}
+    rpc TestNoMetaSerializer(NoMetaRequest) returns (BasicTestNoMetaSerializerResponse) {}
 }
 
 service DefaultValueController {
@@ -230,6 +231,10 @@ message BasicServiceResponse {
     repeated google.protobuf.Struct list_of_dict = 4;
 }
 
+message BasicTestNoMetaSerializerResponse {
+    string value = 1;
+}
+
 message CustomMixParamForListRequest {
     repeated CustomMixParamForRequest results = 1;
     int32 count = 2;
@@ -385,6 +390,10 @@ message ManyManyModelResponse {
     string name = 2;
 }
 
+message NoMetaRequest {
+    string my_field = 1;
+}
+
 message RecursiveTestModelDestroyRequest {
     string uuid = 1;
 }

django_socio_grpc/tests/fakeapp/grpc/fakeapp_pb2_grpc.py

@@ -75,6 +75,11 @@ def __init__(self, channel):
                 request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString,
                 response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString,
                 )
+        self.TestNoMetaSerializer = channel.unary_unary(
+                '/myproject.fakeapp.BasicController/TestNoMetaSerializer',
+                request_serializer=django__socio__grpc_dot_tests_dot_fakeapp_dot_grpc_dot_fakeapp__pb2.NoMetaRequest.SerializeToString,
+                response_deserializer=django__socio__grpc_dot_tests_dot_fakeapp_dot_grpc_dot_fakeapp__pb2.BasicTestNoMetaSerializerResponse.FromString,
+                )
 
 
 class BasicControllerServicer(object):
@@ -152,6 +157,12 @@ def TestEmptyMethod(self, request, context):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
+    def TestNoMetaSerializer(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
 
 def add_BasicControllerServicer_to_server(servicer, server):
     rpc_method_handlers = {
@@ -215,6 +226,11 @@ def add_BasicControllerServicer_to_server(servicer, server):
                     request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString,
                     response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString,
             ),
+            'TestNoMetaSerializer': grpc.unary_unary_rpc_method_handler(
+                    servicer.TestNoMetaSerializer,
+                    request_deserializer=django__socio__grpc_dot_tests_dot_fakeapp_dot_grpc_dot_fakeapp__pb2.NoMetaRequest.FromString,
+                    response_serializer=django__socio__grpc_dot_tests_dot_fakeapp_dot_grpc_dot_fakeapp__pb2.BasicTestNoMetaSerializerResponse.SerializeToString,
+            ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
             'myproject.fakeapp.BasicController', rpc_method_handlers)
@@ -429,6 +445,23 @@ def TestEmptyMethod(request,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
+    @staticmethod
+    def TestNoMetaSerializer(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/myproject.fakeapp.BasicController/TestNoMetaSerializer',
+            django__socio__grpc_dot_tests_dot_fakeapp_dot_grpc_dot_fakeapp__pb2.NoMetaRequest.SerializeToString,
+            django__socio__grpc_dot_tests_dot_fakeapp_dot_grpc_dot_fakeapp__pb2.BasicTestNoMetaSerializerResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
 
 class DefaultValueControllerStub(object):
     """Missing associated documentation comment in .proto file."""

django_socio_grpc/tests/fakeapp/serializers.py

@@ -223,3 +223,7 @@ class Meta:
         proto_class = fakeapp_pb2.DefaultValueResponse
         proto_class_list = fakeapp_pb2.DefaultValueListResponse
         fields = "__all__"
+
+
+class NoMetaSerializer(proto_serializers.ProtoSerializer):
+    my_field = serializers.CharField()

django_socio_grpc/tests/fakeapp/services/basic_service.py

@@ -4,6 +4,7 @@
     BaseProtoExampleSerializer,
     BasicProtoListChildSerializer,
     BasicServiceSerializer,
+    NoMetaSerializer,
 )
 
 from django_socio_grpc import generics
@@ -138,3 +139,12 @@ async def FetchTranslatedKey(self, request, context):
         # INFO - AM - 14/04/2023 - Test translation here
         message = fakeapp_pb2.BasicFetchTranslatedKeyResponse(text=_("Test translation"))
         return message
+
+    @grpc_action(
+        request=NoMetaSerializer,
+        response=[{"name": "value", "type": "string"}],
+    )
+    async def TestNoMetaSerializer(self, request, context):
+        serializer = NoMetaSerializer(message=request)
+        serializer.is_valid(raise_exception=True)
+        return fakeapp_pb2.BasicTestNoMetaSerializerResponse(value=serializer.data["my_field"])

django_socio_grpc/tests/protos/ALL_APP_GENERATED_NO_SEPARATE/fakeapp.proto

@@ -18,6 +18,7 @@ service BasicController {
     rpc MyMethod(CustomNameForRequest) returns (CustomNameForResponse) {}
     rpc TestBaseProtoSerializer(BaseProtoExample) returns (BaseProtoExampleList) {}
     rpc TestEmptyMethod(google.protobuf.Empty) returns (google.protobuf.Empty) {}
+    rpc TestNoMetaSerializer(NoMeta) returns (BasicTestNoMetaSerializer) {}
 }
 
 service ForeignModelController {
@@ -187,6 +188,10 @@ message BasicServiceList {
     int32 count = 2;
 }
 
+message BasicTestNoMetaSerializer {
+    string value = 1;
+}
+
 message CustomMixParamForRequest {
     string user_name = 1;
 }
@@ -245,6 +250,10 @@ message ManyManyModel {
     string test_write_only_on_nested = 3;
 }
 
+message NoMeta {
+    string my_field = 1;
+}
+
 message RecursiveTestModel {
     optional string uuid = 1;
     optional RecursiveTestModel parent = 2;