fix(ingest): avrogen handling for missing fields with default values

metadata-ingestion/setup.py

@@ -35,7 +35,7 @@ def get_long_description():
     "entrypoints",
     "docker",
     "expandvars>=0.6.5",
-    "avro-gen3==0.7.5",
+    "avro-gen3==0.7.6",
     # "avro-gen3 @ git+https://github.com/acryldata/avro_gen@master#egg=avro-gen3",
     "avro>=1.10.2,<1.11",
     "python-dateutil>=2.8.0",

metadata-ingestion/tests/unit/serde/test_serde.py

@@ -238,3 +238,53 @@ def test_null_hiding() -> None:
         "recursive": False,
         "type": {"type": {"com.linkedin.pegasus2avro.schema.StringType": {}}},
     }
+
+
+def test_missing_optional_simple() -> None:
+    original = models.DataHubResourceFilterClass.from_obj(
+        {
+            "allResources": False,
+            "filter": {
+                "criteria": [
+                    {
+                        "condition": "EQUALS",
+                        "field": "RESOURCE_TYPE",
+                        "values": ["notebook", "dataset", "dashboard"],
+                    }
+                ]
+            },
+        }
+    )
+
+    # This one is missing the optional filters.allResources field.
+    revised_obj = {
+        "filter": {
+            "criteria": [
+                {
+                    "condition": "EQUALS",
+                    "field": "RESOURCE_TYPE",
+                    "values": ["notebook", "dataset", "dashboard"],
+                }
+            ]
+        },
+    }
+    revised = models.DataHubResourceFilterClass.from_obj(revised_obj)
+    assert revised.validate()
+
+    assert original == revised
+
+
+def test_missing_optional_in_union() -> None:
+    # This one doesn't contain any optional fields and should work fine.
+    revised_json = json.loads(
+        '{"lastUpdatedTimestamp":1662356745807,"actors":{"groups":[],"resourceOwners":false,"allUsers":true,"allGroups":false,"users":[]},"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"displayName":"customtest","resources":{"filter":{"criteria":[{"field":"RESOURCE_TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]},"allResources":false},"description":"","state":"ACTIVE","type":"METADATA"}'
+    )
+    revised = models.DataHubPolicyInfoClass.from_obj(revised_json)
+
+    # This one is missing the optional filters.allResources field.
+    original_json = json.loads(
+        '{"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"actors":{"resourceOwners":false,"groups":[],"allGroups":false,"allUsers":true,"users":[]},"lastUpdatedTimestamp":1662356745807,"displayName":"customtest","description":"","resources":{"filter":{"criteria":[{"field":"RESOURCE_TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]}},"state":"ACTIVE","type":"METADATA"}'
+    )
+    original = models.DataHubPolicyInfoClass.from_obj(original_json)
+
+    assert revised == original