From 0c15fefd3ef0ea82a25a81c03cae034f3f6aa8f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Wed, 23 Nov 2022 11:54:22 -0600 Subject: [PATCH 1/2] feat: Automatic catalog selection of replication keys --- poetry.lock | 2 +- singer_sdk/_singerlib/catalog.py | 2 ++ tests/_singerlib/test_catalog.py | 6 ++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 99918cdf9..d68f3cd7a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2514,4 +2514,4 @@ xdoctest = [ zipp = [ {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, -] +] \ No newline at end of file diff --git a/singer_sdk/_singerlib/catalog.py b/singer_sdk/_singerlib/catalog.py index 89dd789c5..51954e0e7 100644 --- a/singer_sdk/_singerlib/catalog.py +++ b/singer_sdk/_singerlib/catalog.py @@ -194,6 +194,8 @@ def get_standard_metadata( for field_name in schema.get("properties", {}).keys(): if key_properties and field_name in key_properties: entry = Metadata(inclusion=Metadata.InclusionType.AUTOMATIC) + elif valid_replication_keys and field_name in valid_replication_keys: + entry = Metadata(inclusion=Metadata.InclusionType.AUTOMATIC) else: entry = Metadata(inclusion=Metadata.InclusionType.AVAILABLE) diff --git a/tests/_singerlib/test_catalog.py b/tests/_singerlib/test_catalog.py index 7c1fc04d7..c589d1836 100644 --- a/tests/_singerlib/test_catalog.py +++ b/tests/_singerlib/test_catalog.py @@ -207,6 +207,7 @@ def test_catalog_parsing(): "properties": { "first_name": {"type": "string"}, "last_name": {"type": "string"}, + "updated_at": {"type": "string", "format": "date-time"}, }, "type": "object", }, @@ -265,3 +266,8 @@ def test_standard_metadata( pk_metadata = metadata[("properties", pk)] assert pk_metadata.inclusion == Metadata.InclusionType.AUTOMATIC assert pk_metadata.selected is None + + for rk in valid_replication_keys or []: + rk_metadata = metadata[("properties", rk)] + assert rk_metadata.inclusion == Metadata.InclusionType.AUTOMATIC + assert rk_metadata.selected is None From 288376060e45acef45831e028c67f2897f313e7a Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Mon, 19 Dec 2022 12:36:49 -0600 Subject: [PATCH 2/2] Update poetry.lock --- poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index d68f3cd7a..99918cdf9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2514,4 +2514,4 @@ xdoctest = [ zipp = [ {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, -] \ No newline at end of file +]