diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 6c29db5ba2207..1d4867b0dc4bf 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -35,7 +35,7 @@ def get_long_description(): "entrypoints", "docker", "expandvars>=0.6.5", - "avro-gen3==0.7.5", + "avro-gen3==0.7.6", # "avro-gen3 @ git+https://github.com/acryldata/avro_gen@master#egg=avro-gen3", "avro>=1.10.2,<1.11", "python-dateutil>=2.8.0", diff --git a/metadata-ingestion/tests/unit/serde/test_serde.py b/metadata-ingestion/tests/unit/serde/test_serde.py index 828ccb50a78dd..f1b0ee92c4c55 100644 --- a/metadata-ingestion/tests/unit/serde/test_serde.py +++ b/metadata-ingestion/tests/unit/serde/test_serde.py @@ -238,3 +238,53 @@ def test_null_hiding() -> None: "recursive": False, "type": {"type": {"com.linkedin.pegasus2avro.schema.StringType": {}}}, } + + +def test_missing_optional_simple() -> None: + original = models.DataHubResourceFilterClass.from_obj( + { + "allResources": False, + "filter": { + "criteria": [ + { + "condition": "EQUALS", + "field": "RESOURCE_TYPE", + "values": ["notebook", "dataset", "dashboard"], + } + ] + }, + } + ) + + # This one is missing the optional filters.allResources field. + revised_obj = { + "filter": { + "criteria": [ + { + "condition": "EQUALS", + "field": "RESOURCE_TYPE", + "values": ["notebook", "dataset", "dashboard"], + } + ] + }, + } + revised = models.DataHubResourceFilterClass.from_obj(revised_obj) + assert revised.validate() + + assert original == revised + + +def test_missing_optional_in_union() -> None: + # This one doesn't contain any optional fields and should work fine. + revised_json = json.loads( + '{"lastUpdatedTimestamp":1662356745807,"actors":{"groups":[],"resourceOwners":false,"allUsers":true,"allGroups":false,"users":[]},"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"displayName":"customtest","resources":{"filter":{"criteria":[{"field":"RESOURCE_TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]},"allResources":false},"description":"","state":"ACTIVE","type":"METADATA"}' + ) + revised = models.DataHubPolicyInfoClass.from_obj(revised_json) + + # This one is missing the optional filters.allResources field. + original_json = json.loads( + '{"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"actors":{"resourceOwners":false,"groups":[],"allGroups":false,"allUsers":true,"users":[]},"lastUpdatedTimestamp":1662356745807,"displayName":"customtest","description":"","resources":{"filter":{"criteria":[{"field":"RESOURCE_TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]}},"state":"ACTIVE","type":"METADATA"}' + ) + original = models.DataHubPolicyInfoClass.from_obj(original_json) + + assert revised == original