Skip to content

Commit

Permalink
fix: change runner scope to function for target tests (#1752)
Browse files Browse the repository at this point in the history
Co-authored-by: Edgar R. M <[email protected]>
  • Loading branch information
Ken Payne and edgarrmondragon authored Jun 7, 2023
1 parent cf57439 commit 6296599
Show file tree
Hide file tree
Showing 7 changed files with 359 additions and 176 deletions.
1 change: 1 addition & 0 deletions samples/sample_target_csv/csv_target_sink.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ def process_batch(self, context: dict) -> None:
delimiter="\t",
quotechar='"',
quoting=csv.QUOTE_NONNUMERIC,
escapechar="\\",
)
for record in records_to_drain:
if newfile and not records_written:
Expand Down
22 changes: 2 additions & 20 deletions samples/sample_target_parquet/parquet_target_sink.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ def _json_schema_to_arrow_fields(schema: dict[str, t.Any]) -> pa.StructType:
for name, property_schema in schema.get("properties", {}).items():
field = pa.field(name, _json_type_to_arrow_field(property_schema))
fields.append(field)
return fields

return fields if fields else [pa.field("dummy", pa.string())]


def _json_type_to_arrow_field( # noqa: PLR0911
Expand Down Expand Up @@ -99,22 +100,3 @@ def process_batch(self, context: dict) -> None:
table = pa.Table.from_pylist(records_to_drain, schema=schema)
writer.write_table(table)
writer.close()

@staticmethod
def translate_data_type(singer_type: str | dict) -> t.Any:
"""Translate from singer_type to a native type."""
if singer_type in ["decimal", "float", "double"]:
return pa.decimal128
if singer_type in ["date-time"]:
return pa.datetime
if singer_type in ["date"]:
return pa.date64
return pa.string

def _get_parquet_schema(self) -> list[tuple[str, t.Any]]:
col_list: list[tuple[str, t.Any]] = []
for prop in self.schema["properties"]:
col_list.append(
(prop["name"], self.translate_data_type(prop["type"])),
)
return col_list
3 changes: 1 addition & 2 deletions singer_sdk/testing/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from __future__ import annotations

from .config import SuiteConfig
from .factory import get_tap_test_class, get_target_test_class, get_test_class
from .factory import get_tap_test_class, get_target_test_class
from .legacy import (
_get_tap_catalog,
_select_all,
Expand All @@ -19,7 +19,6 @@
__all__ = [
"get_tap_test_class",
"get_target_test_class",
"get_test_class",
"_get_tap_catalog",
"_select_all",
"get_standard_tap_tests",
Expand Down
Loading

0 comments on commit 6296599

Please sign in to comment.