From 4e4a6138f49e169ccdfc3df2f4a708135b6fac01 Mon Sep 17 00:00:00 2001 From: Nik Zavada <134077230+nikzavada@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:37:06 -0500 Subject: [PATCH 1/3] Add a branch to to_json_compatible to return a stringified value for uuid.UUID --- singer_sdk/helpers/_typing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index b02ecb512..6d6771944 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -4,6 +4,7 @@ import copy import datetime +import uuid import logging import typing as t from enum import Enum @@ -42,6 +43,8 @@ def to_json_compatible(val: t.Any) -> t.Any: # noqa: ANN401 if isinstance(val, (datetime.datetime,)): # Make naive datetimes UTC return (val.replace(tzinfo=UTC) if val.tzinfo is None else val).isoformat("T") + elif isinstance(val, (uuid.UUID,)): + return str(val) return val From 6a823a7753eee26c8cfd4b47e0f18cba1e8d7025 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 19:42:33 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- singer_sdk/helpers/_typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index 6d6771944..d3b629c11 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -4,9 +4,9 @@ import copy import datetime -import uuid import logging import typing as t +import uuid from enum import Enum from functools import lru_cache @@ -43,7 +43,7 @@ def to_json_compatible(val: t.Any) -> t.Any: # noqa: ANN401 if isinstance(val, (datetime.datetime,)): # Make naive datetimes UTC return (val.replace(tzinfo=UTC) if val.tzinfo is None else val).isoformat("T") - elif isinstance(val, (uuid.UUID,)): + if isinstance(val, (uuid.UUID,)): return str(val) return val From ec5bf31719455987a1e6113a97c6c29a1c3e96ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez-Mondrag=C3=B3n?= Date: Mon, 11 Nov 2024 13:51:24 -0600 Subject: [PATCH 3/3] Add test --- singer_sdk/helpers/_state.py | 4 ++++ tests/core/test_state_handling.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/singer_sdk/helpers/_state.py b/singer_sdk/helpers/_state.py index fd7dee377..565dbd51d 100644 --- a/singer_sdk/helpers/_state.py +++ b/singer_sdk/helpers/_state.py @@ -218,6 +218,10 @@ def increment_state( extra={"replication_key": replication_key}, ) progress_dict = stream_or_partition_state[PROGRESS_MARKERS] + # TODO: Instead of forcing all values to be JSON-compatible strings and hope + # we catch all cases, we should allow the stream to define how to + # the values from the state and the record should be pre-processed. + # https://github.com/meltano/sdk/issues/2753 old_rk_value = to_json_compatible(progress_dict.get("replication_key_value")) new_rk_value = to_json_compatible(latest_record[replication_key]) diff --git a/tests/core/test_state_handling.py b/tests/core/test_state_handling.py index f58a0128b..85fd4812f 100644 --- a/tests/core/test_state_handling.py +++ b/tests/core/test_state_handling.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import uuid import pytest @@ -155,3 +156,21 @@ def test_null_replication_value(caplog): ), "State should not be updated." assert caplog.records[0].levelname == "WARNING" assert "is null" in caplog.records[0].message + + +def test_uuidv7_replication_value(): + stream_state = { + "replication_key": "id", + "replication_key_value": "01931c63-b14e-7ff3-8621-e577ed392dc8", + } + new_string_val = "01931c63-b14e-7ff3-8621-e578edbca9a3" + + _state.increment_state( + stream_state, + latest_record={"id": uuid.UUID(new_string_val)}, + replication_key="id", + is_sorted=True, + check_sorted=True, + ) + + assert stream_state["replication_key_value"] == new_string_val