Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: change runner scope to function for target tests #1752

Merged
merged 13 commits into from
Jun 7, 2023
1 change: 1 addition & 0 deletions samples/sample_target_csv/csv_target_sink.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ def process_batch(self, context: dict) -> None:
delimiter="\t",
quotechar='"',
quoting=csv.QUOTE_NONNUMERIC,
escapechar="\\",
)
for record in records_to_drain:
if newfile and not records_written:
Expand Down
22 changes: 2 additions & 20 deletions samples/sample_target_parquet/parquet_target_sink.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ def _json_schema_to_arrow_fields(schema: dict[str, t.Any]) -> pa.StructType:
for name, property_schema in schema.get("properties", {}).items():
field = pa.field(name, _json_type_to_arrow_field(property_schema))
fields.append(field)
return fields

return fields if fields else [pa.field("dummy", pa.string())]


def _json_type_to_arrow_field( # noqa: PLR0911
Expand Down Expand Up @@ -99,22 +100,3 @@ def process_batch(self, context: dict) -> None:
table = pa.Table.from_pylist(records_to_drain, schema=schema)
writer.write_table(table)
writer.close()

@staticmethod
def translate_data_type(singer_type: str | dict) -> t.Any:
"""Translate from singer_type to a native type."""
if singer_type in ["decimal", "float", "double"]:
return pa.decimal128
if singer_type in ["date-time"]:
return pa.datetime
if singer_type in ["date"]:
return pa.date64
return pa.string

def _get_parquet_schema(self) -> list[tuple[str, t.Any]]:
col_list: list[tuple[str, t.Any]] = []
for prop in self.schema["properties"]:
col_list.append(
(prop["name"], self.translate_data_type(prop["type"])),
)
return col_list
3 changes: 1 addition & 2 deletions singer_sdk/testing/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from __future__ import annotations

from .config import SuiteConfig
from .factory import get_tap_test_class, get_target_test_class, get_test_class
from .factory import get_tap_test_class, get_target_test_class
from .legacy import (
_get_tap_catalog,
_select_all,
Expand All @@ -19,7 +19,6 @@
__all__ = [
"get_tap_test_class",
"get_target_test_class",
"get_test_class",
"_get_tap_catalog",
"_select_all",
"get_standard_tap_tests",
Expand Down
Loading