Skip to content

Commit

Permalink
fix(targets): Check against the unconformed key properties when valid…
Browse files Browse the repository at this point in the history
…ating record keys (#1853)
  • Loading branch information
edgarrmondragon authored Jul 18, 2023
1 parent 4c53313 commit e655386
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 2 deletions.
7 changes: 5 additions & 2 deletions singer_sdk/sinks/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,9 @@ def datetime_error_treatment(self) -> DatetimeErrorTreatmentEnum:
def key_properties(self) -> list[str]:
"""Return key properties.
Override this method to return a list of key properties in a format that is
compatible with the target.
Returns:
A list of stream key properties.
"""
Expand Down Expand Up @@ -331,10 +334,10 @@ def _singer_validate_message(self, record: dict) -> None:
Raises:
MissingKeyPropertiesError: If record is missing one or more key properties.
"""
if not all(key_property in record for key_property in self.key_properties):
if any(key_property not in record for key_property in self._key_properties):
msg = (
f"Record is missing one or more key_properties. \n"
f"Key Properties: {self.key_properties}, "
f"Key Properties: {self._key_properties}, "
f"Record Keys: {list(record.keys())}"
)
raise MissingKeyPropertiesError(
Expand Down
4 changes: 4 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ def process_batch(self, context: dict) -> None:
self.target.records_written.extend(context["records"])
self.target.num_batches_processed += 1

@property
def key_properties(self) -> list[str]:
return [key.upper() for key in super().key_properties]


class TargetMock(Target):
"""A mock Target class."""
Expand Down
25 changes: 25 additions & 0 deletions tests/core/test_target_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

import copy

import pytest

from singer_sdk.exceptions import MissingKeyPropertiesError
from tests.conftest import BatchSinkMock, TargetMock


Expand All @@ -28,3 +31,25 @@ def test_get_sink():
key_properties=key_properties,
)
assert sink_returned == sink


def test_validate_record():
target = TargetMock()
sink = BatchSinkMock(
target=target,
stream_name="test",
schema={
"properties": {
"id": {"type": ["integer"]},
"name": {"type": ["string"]},
},
},
key_properties=["id"],
)

# Test valid record
sink._singer_validate_message({"id": 1, "name": "test"})

# Test invalid record
with pytest.raises(MissingKeyPropertiesError):
sink._singer_validate_message({"name": "test"})

0 comments on commit e655386

Please sign in to comment.