Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
pre-commit-ci[bot] committed Oct 7, 2023
1 parent 899e698 commit 4437e4e
Showing 2 changed files with 56 additions and 26 deletions.
22 changes: 12 additions & 10 deletions singer_sdk/mapper.py
Original file line number Diff line number Diff line change
@@ -55,23 +55,25 @@ def md5(string: str) -> str:


def compound_eval(
expr: str,
operators: t.Dict[str, t.Callable[[dict], dict | None]] = None,
functions: t.Dict[str, t.Callable[[dict], dict | None]] = None,
names=t.Dict[str, t.Any]
) -> t.Union[str, int, float, t.List, t.Set, t.Dict]:
"""Evaluate inline maps using the `EvalWithCompoundTypes` class
expr: str,
operators: dict[str, t.Callable[[dict], dict | None]] | None = None,
functions: dict[str, t.Callable[[dict], dict | None]] | None = None,
names=t.Dict[str, t.Any],
) -> str | (int | (float | (list | (set | dict)))):
"""Evaluate inline maps using the `EvalWithCompoundTypes` class.
Args:
expr: expression to evaluate
operators: dictionary of operators and the functions they map to in the evaluation context
functions: dictionary of function names and definitions available in the evaluation context
names: dictionary of variable names available in the evaluation context
Returns:
result of the evaluated expression
"""
s = simpleeval.EvalWithCompoundTypes(operators=operators, functions=functions, names=names)
s = simpleeval.EvalWithCompoundTypes(
operators=operators, functions=functions, names=names
)
return s.eval(expr)


@@ -398,7 +400,7 @@ def _eval_type(

if expr.startswith("bool("):
return th.BooleanType()

return th.StringType() if expr[0] == "'" and expr[-1] == "'" else default

def _init_functions_and_schema( # noqa: PLR0912, PLR0915, C901
60 changes: 44 additions & 16 deletions tests/core/test_mapper.py
Original file line number Diff line number Diff line change
@@ -22,14 +22,14 @@
from singer_sdk.tap_base import Tap
from singer_sdk.typing import (
ArrayType,
BooleanType,
IntegerType,
NumberType,
ObjectType,
OneOf,
PropertiesList,
Property,
StringType,
BooleanType,
OneOf,
)

if t.TYPE_CHECKING:
@@ -60,11 +60,15 @@ def sample_catalog_dict() -> dict:
).to_dict()
nested_jellybean_schema = PropertiesList(
Property("id", IntegerType),
Property("custom_fields",
ArrayType(ObjectType(
Property("id", IntegerType),
Property("value", OneOf(StringType, IntegerType, BooleanType))))
)
Property(
"custom_fields",
ArrayType(
ObjectType(
Property("id", IntegerType),
Property("value", OneOf(StringType, IntegerType, BooleanType)),
)
),
),
).to_dict()
return {
"streams": [
@@ -81,8 +85,8 @@ def sample_catalog_dict() -> dict:
{
"stream": "nested_jellybean",
"tap_stream_id": "nested_jellybean",
"schema": nested_jellybean_schema
}
"schema": nested_jellybean_schema,
},
],
}

@@ -126,8 +130,22 @@ def sample_stream():
{"brown": "fox"},
],
"nested_jellybean": [
{"id": 123, "custom_fields": [{"id": 1, "value": "abc"}, {"id": 2, "value": 1212}, {"id": 3, "value": None}]},
{"id": 124, "custom_fields": [{"id": 1, "value": "foo"}, {"id": 2, "value": 9009}, {"id": 3, "value": True}]}
{
"id": 123,
"custom_fields": [
{"id": 1, "value": "abc"},
{"id": 2, "value": 1212},
{"id": 3, "value": None},
],
},
{
"id": 124,
"custom_fields": [
{"id": 1, "value": "foo"},
{"id": 2, "value": 9009},
{"id": 3, "value": True},
],
},
],
}

@@ -153,7 +171,7 @@ def transform_stream_maps():
"custom_field_1": 'dict([(x["id"], x["value"]) for x in custom_fields]).get(1)',
"custom_field_2": 'int(dict([(x["id"], x["value"]) for x in custom_fields]).get(2)) if dict([(x["id"], x["value"]) for x in custom_fields]).get(2) else None',
"custom_field_3": 'bool(dict([(x["id"], x["value"]) for x in custom_fields]).get(3)) if dict([(x["id"], x["value"]) for x in custom_fields]).get(3) else None',
}
},
}


@@ -211,8 +229,18 @@ def transformed_result(stream_map_config):
{"brown": "fox"},
],
"nested_jellybean": [
{"id": 123, "custom_field_1": "abc", "custom_field_2": 1212, "custom_field_3": None},
{"id": 124, "custom_field_1": "foo", "custom_field_2": 9009, "custom_field_3": True}
{
"id": 123,
"custom_field_1": "abc",
"custom_field_2": 1212,
"custom_field_3": None,
},
{
"id": 124,
"custom_field_1": "foo",
"custom_field_2": 9009,
"custom_field_3": True,
},
],
}

@@ -237,8 +265,8 @@ def transformed_schemas():
Property("id", IntegerType),
Property("custom_field_1", StringType),
Property("custom_field_2", IntegerType),
Property("custom_field_3", BooleanType)
).to_dict()
Property("custom_field_3", BooleanType),
).to_dict(),
}


0 comments on commit 4437e4e

Please sign in to comment.