diff --git a/.github/workflows/meltano-run.yml b/.github/workflows/meltano-run.yml index 9ad50d8..0ed41cc 100644 --- a/.github/workflows/meltano-run.yml +++ b/.github/workflows/meltano-run.yml @@ -20,6 +20,10 @@ jobs: mapping: whitelist output_db: tap_smoke_test.db target_table: animals + - tap: people + mapping: flatten + output_db: people.db + target_table: people steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index 012541a..dceef05 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,37 @@ This mapper plugin is fully compliant with the Singer Spec and can be placed in For a collection of examples, take a look at [examples/README.md](examples/README.md). -Here's one of them!: +## Settings + +| Setting | Required | Default | Description | +|:--------------------|:--------:|:-------:|:------------| +| stream_maps | True | None | Stream maps | +| flattening_enabled | False | None | 'True' to enable schema flattening and automatically expand nested properties. | +| flattening_max_depth| False | None | The max depth to flatten schemas. | +| stream_map_config | False | None | User-defined config values to be used within map expressions. | + +A full list of supported settings and capabilities is available by running: `meltano-map-transformer --about` + +## Installation + +We recommend using GitHub release tags when installing with `pip`. We also recommend using `pipx` or `meltano` instead of installing with `pip` directly. + +You can see a full list of published releases [here](https://github.com/MeltanoLabs/meltano-map-transform/releases). + +For example: + +``` +# Update the below with the latest published release: +# https://github.com/MeltanoLabs/meltano-map-transform/releases + +# Install with pip, ideally in a new virtual environment to avoid conflicts: +pip install git+https://github.com/MeltanoLabs/meltano-map-transform.git@v0.4.1 + +# Or better yet, use `pipx` so that virtual environments are managed automatically: +pipx install git+https://github.com/MeltanoLabs/meltano-map-transform.git@v0.4.1 +``` + +### Meltano installation instructions 1. Add this plugin to your Meltano project diff --git a/fixtures/people.singer b/fixtures/people.singer new file mode 100644 index 0000000..3b501d7 --- /dev/null +++ b/fixtures/people.singer @@ -0,0 +1,9 @@ +{"type": "SCHEMA", "stream": "people", "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}, "name": {"type": "string"}, "address": {"type": "object", "properties": {"street": {"type": "string"}, "city": {"type": "string"}, "state": {"type": "string"}, "zip": {"type": "string"}, "coordinates": {"type": "object", "properties": {"latitude": {"type": "number"}, "longitude": {"type": "number"}}}}}}}} +{"type": "RECORD", "stream": "people", "record": {"id": 1, "name": "Jane Doe", "address": {"street": "456 Elm St", "city": "Smallville", "state": "NY", "zip": "67890", "coordinates": {"latitude": 40.758896, "longitude": -73.985130}}}} +{"type": "RECORD", "stream": "people", "record": {"id": 2, "name": "John Smith", "address": {"street": "789 Oak Ave", "city": "Metropolis", "state": "IL", "zip": "54321", "coordinates": {"latitude": 41.878114, "longitude": -87.629799}}}} +{"type": "RECORD", "stream": "people", "record": {"id": 3, "name": "Jane Smith", "address": {"street": "246 Maple St", "city": "Gotham", "state": "NY", "zip": "11111", "coordinates": {"latitude": 40.712776, "longitude": -74.005974}}}} +{"type": "RECORD", "stream": "people", "record": {"id": 4, "name": "John Doe", "address": {"street": "369 Pine Rd", "city": "Central City", "state": "CA", "zip": "22222", "coordinates": {"latitude": 37.7749, "longitude": -122.4194}}}} +{"type": "RECORD", "stream": "people", "record": {"id": 5, "name": "Jane Wilson", "address": {"street": "159 Cedar Blvd", "city": "Star City", "state": "IL", "zip": "33333", "coordinates": {"latitude": 41.872389, "longitude": -87.650047}}}} +{"type": "RECORD", "stream": "people", "record": {"id": 6, "name": "John Wilson", "address": {"street": "753 Maple Ave", "city": "Coast City", "state": "CA", "zip": "44444", "coordinates": {"latitude": 34.052235, "longitude": -118.243683}}}} +{"type": "RECORD", "stream": "people", "record": {"id": 7, "name": "Jane Johnson", "address": {"street": "369 Oak St", "city": "Gotham", "state": "NY", "zip": "55555", "coordinates": {"latitude": 40.712776, "longitude": -74.005974}}}} +{"type": "RECORD", "stream": "people", "record": {"id": 8, "name": "John Johnson", "address": {"street": "159 Elm Rd", "city": "Central City", "state": "CA", "zip": "66666", "coordinates": {"latitude": 37.7749, "longitude": -122.4194}}}} diff --git a/meltano.yml b/meltano.yml index d317fa4..392c41c 100644 --- a/meltano.yml +++ b/meltano.yml @@ -20,6 +20,9 @@ plugins: kind: integer - name: streams kind: array + - name: people + namespace: people + executable: scripts/people.sh loaders: - name: target-sqlite variant: meltanolabs @@ -55,6 +58,11 @@ plugins: id: id description: description __else__: __NULL__ + - name: flatten + config: + stream_maps: {} + flattening_enabled: true + flattening_max_depth: 1 environments: - name: dev config: diff --git a/meltano_map_transform/mapper.py b/meltano_map_transform/mapper.py index 3a861ba..d6ed876 100644 --- a/meltano_map_transform/mapper.py +++ b/meltano_map_transform/mapper.py @@ -45,6 +45,19 @@ class StreamTransform(InlineMapper): required=True, description="Stream maps", ), + th.Property( + "flattening_enabled", + th.BooleanType(), + description=( + "'True' to enable schema flattening and automatically expand nested " + "properties." + ), + ), + th.Property( + "flattening_max_depth", + th.IntegerType(), + description="The max depth to flatten schemas.", + ), ).to_dict() def __init__( @@ -93,13 +106,12 @@ def map_schema_message( message_dict.get("key_properties", []), ) for stream_map in self.mapper.stream_maps[stream_id]: - schema_message = singer.SchemaMessage( + yield singer.SchemaMessage( stream_map.stream_alias, stream_map.transformed_schema, stream_map.transformed_key_properties, message_dict.get("bookmark_keys", []), ) - yield schema_message def map_record_message( self, @@ -119,13 +131,12 @@ def map_record_message( for stream_map in self.mapper.stream_maps[stream_id]: mapped_record = stream_map.transform(message_dict["record"]) if mapped_record is not None: - record_message = singer.RecordMessage( + yield singer.RecordMessage( stream=stream_map.stream_alias, record=mapped_record, version=message_dict.get("version"), time_extracted=utc_now(), ) - yield record_message def map_state_message(self, message_dict: dict) -> list[singer.Message]: """Do nothing to the message. diff --git a/plugins/extractors/tap-smoke-test.lock b/plugins/extractors/tap-smoke-test.lock new file mode 100644 index 0000000..d481007 --- /dev/null +++ b/plugins/extractors/tap-smoke-test.lock @@ -0,0 +1,22 @@ +{ + "plugin_type": "extractors", + "name": "tap-smoke-test", + "namespace": "tap_smoke_test", + "label": "tap-smoke-test", + "pip_url": "git+https://github.com/meltano/tap-smoke-test.git", + "executable": "tap-smoke-test", + "capabilities": [ + "discover", + "catalog" + ], + "settings": [ + { + "name": "schema_inference_record_count", + "kind": "integer" + }, + { + "name": "streams", + "kind": "array" + } + ] +} diff --git a/poetry.lock b/poetry.lock index 5e6efc5..5e355d8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -341,7 +341,6 @@ files = [ {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, - {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"}, {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, @@ -350,7 +349,6 @@ files = [ {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, - {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, @@ -380,7 +378,6 @@ files = [ {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, - {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, @@ -389,7 +386,6 @@ files = [ {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, - {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"}, {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, @@ -945,13 +941,13 @@ files = [ [[package]] name = "singer-sdk" -version = "0.31.1" +version = "0.32.0b1" description = "A framework for building Singer taps" optional = false python-versions = ">=3.7.1,<3.12" files = [ - {file = "singer_sdk-0.31.1-py3-none-any.whl", hash = "sha256:c5e20a1b042ab49f8446c2db150f7f75e3e6a99a0d579f1a7ff9eed9abeda084"}, - {file = "singer_sdk-0.31.1.tar.gz", hash = "sha256:cdd76f05259c1244ae91b4ac6fe44f7e3a182a3c480a29b60dfee9c51d498ea0"}, + {file = "singer_sdk-0.32.0b1-py3-none-any.whl", hash = "sha256:49861ae07337ad2855eeb99c4b7894b8a2b47829b8bf1d825bdcb7ba00282fc0"}, + {file = "singer_sdk-0.32.0b1.tar.gz", hash = "sha256:e6286a4aa3b6821cfce9d18700cd84eb23e5d4ae946ca64d28aa6da20d81afa3"}, ] [package.dependencies] @@ -1125,4 +1121,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "<3.12,>=3.7.1" -content-hash = "44909e588292c6f5cb6380ae947cacf3d388d2e99f45aa96b198732fcadcaccc" +content-hash = "c1a5264d53a83c7b832186563b8a3ca6611162f95001344fa8ea3d64ea9743f7" diff --git a/pyproject.toml b/pyproject.toml index 9c1fdc1..93fcfc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "meltano-map-transform" -version = "0.1.0" +version = "0.4.1" description = "`meltano-map-transform` is a Singer-compatible inline map transformer, built with the Meltano SDK for Singer Taps." authors = [ "Meltano Team and Contributors", @@ -10,7 +10,7 @@ keywords = [ "singer.io", "Inline Transformation", ] -license = "Apache 2.0" +license = "Apache-2.0" readme = "README.md" homepage = "https://github.com/MeltanoLabs/meltano-map-transform" repository = "https://github.com/MeltanoLabs/meltano-map-transform" @@ -18,7 +18,10 @@ documentation = "https://github.com/MeltanoLabs/meltano-map-transform#readme" [tool.poetry.dependencies] python = "<3.12,>=3.7.1" -singer-sdk = "~=0.31.0" + +[tool.poetry.dependencies.singer-sdk] +allow-prereleases = true +version = "~=0.32.0b1" [tool.poetry.dev-dependencies] pytest = "^7.4.2" diff --git a/scripts/people.sh b/scripts/people.sh new file mode 100755 index 0000000..8e621c4 --- /dev/null +++ b/scripts/people.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +cat fixtures/people.singer