diff --git a/pyproject.toml b/pyproject.toml index e0ed8942b..82decc654 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -168,5 +168,8 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] -# Sample CLI declaration: -# sdk-tap-countries-sample = 'singer_sdk.samples.sample_tap_countries.countries_tap:SampleTapCountries.cli' +# These are available while developing the SDK and after installing the SDK using pip/pipx. +singer-tools = 'singer_sdk.dev.cli:dev_cli' + +# Uncomment to test sample connectors in the CLI: +# sample-tap-countries = 'samples.sample_tap_countries.countries_tap:SampleTapCountries.cli' diff --git a/singer_sdk/dev/__init__.py b/singer_sdk/dev/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/singer_sdk/dev/cli.py b/singer_sdk/dev/cli.py new file mode 100644 index 000000000..d876d3501 --- /dev/null +++ b/singer_sdk/dev/cli.py @@ -0,0 +1,57 @@ +"""Dev CLI.""" + +from __future__ import annotations + +from os import mkdir +from pathlib import Path + +import click +import yaml + +from singer_sdk.helpers._meltano import meltano_yaml_str + + +@click.group() +def dev_cli() -> None: + """Run the dev CLI.""" + + +@dev_cli.command() +@click.option("--from-file", type=click.Path(exists=True), required=True) +@click.option("--out-dir", type=click.Path(exists=False), required=False) +def analyze( + from_file: str, + out_dir: str | None = None, +) -> None: + """Print helpful information about a tap or target. + + To use this command, first save the tap or targets `--about` output using: + `my-connector --about --format=json > my-connector.about.json` + + Currently the `analyze` command generates a single ` + + Args: + from_file: Source file, in YAML or JSON format (string). + out_dir: Destination directory for generated files. If not specified, the + working directory will be used. + """ + json_info = yaml.load(Path(from_file).read_text(encoding="utf-8"), yaml.SafeLoader) + connector_name = json_info["name"] + meltano_yml = meltano_yaml_str( + json_info["name"], json_info["capabilities"], json_info["settings"] + ) + out_dir = out_dir or "." + Path(out_dir).mkdir(parents=True, exist_ok=True) + for file_desc, out_path, file_text in [ + ( + "Meltano plugin definition", + Path(out_dir) / f"{connector_name}.meltano.yml", + meltano_yml, + ) + ]: + print(f"{file_desc}: {out_path}") + Path(out_path).write_text(file_text, encoding="utf-8") + + +if __name__ == "__main__": + dev_cli() diff --git a/singer_sdk/helpers/_meltano.py b/singer_sdk/helpers/_meltano.py new file mode 100644 index 000000000..0b397dff0 --- /dev/null +++ b/singer_sdk/helpers/_meltano.py @@ -0,0 +1,107 @@ +"""Helper functions for Meltano and MeltanoHub interop.""" + +from __future__ import annotations + +from ._typing import ( + is_array_type, + is_boolean_type, + is_date_or_datetime_type, + is_integer_type, + is_object_type, + is_secret_type, + is_string_type, +) + + +def _to_meltano_kind(jsonschema_def: dict) -> str | None: + """Returns a Meltano `kind` indicator for the provided JSON Schema property node. + + For reference: + https://docs.meltano.com/reference/plugin-definition-syntax#settingskind + + Args: + jsonschema_type: JSON Schema type to check. + + Returns: + A string representing the meltano 'kind'. + """ + if is_secret_type(jsonschema_def): + return "password" + + if is_date_or_datetime_type(jsonschema_def): + return "date_iso8601" + + if is_string_type(jsonschema_def): + return "string" + + if is_object_type(jsonschema_def): + return "object" + + if is_array_type(jsonschema_def): + return "array" + + if is_boolean_type(jsonschema_def): + return "boolean" + + if is_integer_type(jsonschema_def): + return "integer" + + return None + + +def meltano_yaml_str( + plugin_name: str, + capabilities: list[str], + config_jsonschema: dict, +) -> str: + """Returns a Meltano plugin definition as a yaml string. + + Args: + plugin_name: Name of the plugin. + capabilities: List of capabilities. + config_jsonschema: JSON Schema of the expected config. + + Returns: + A string representing the Meltano plugin Yaml definition. + """ + capabilities_str: str = "\n".join( + [f" - {capability}" for capability in capabilities] + ) + settings_str: str = "\n".join( + [ + f"""- name: {setting_name} + label: {setting_name.replace("_", " ").title()} + kind: {_to_meltano_kind(property_node)} + description: {property_node.get("description", 'null')}""" + for setting_name, property_node in config_jsonschema["properties"].items() + ] + ) + required_settings = [ + setting_name + for setting_name, type_dict in config_jsonschema["properties"].items() + if setting_name in config_jsonschema.get("required", []) + or type_dict.get("required", False) + ] + settings_group_validation_str = " - - " + "\n - ".join(required_settings) + + return f"""name: {plugin_name} +namespace: {plugin_name.replace('-', '_')} + +## The following could not be auto-detected: +# maintenance_status: # +# repo: # +# variant: # +# label: # +# description: # +# pip_url: # +# domain_url: # +# logo_url: # +# keywords: [] # + +capabilities: +{capabilities_str} +settings_group_validation: +{settings_group_validation_str} +settings: +{settings_str} +""" diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index 0d4ba30db..4db07a685 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -341,64 +341,75 @@ def print_about(cls: Type["PluginBase"], format: Optional[str] = None) -> None: if format == "json": print(json.dumps(info, indent=2, default=str)) + return - elif format == "markdown": - max_setting_len = cast( - int, max(len(k) for k in info["settings"]["properties"].keys()) - ) + if format == "markdown": + cls._print_about_markdown(info) + return - # Set table base for markdown - table_base = ( - f"| {'Setting':{max_setting_len}}| Required | Default | Description |\n" - f"|:{'-' * max_setting_len}|:--------:|:-------:|:------------|\n" - ) + formatted = "\n".join([f"{k.title()}: {v}" for k, v in info.items()]) + print(formatted) - # Empty list for string parts - md_list = [] - # Get required settings for table - required_settings = info["settings"].get("required", []) + @classmethod + def _print_about_markdown(cls: Type["PluginBase"], info: dict) -> None: + """Print about info as markdown. - # Iterate over Dict to set md - md_list.append( - f"# `{info['name']}`\n\n" - f"{info['description']}\n\n" - f"Built with the [Meltano Singer SDK](https://sdk.meltano.com).\n\n" - ) - for key, value in info.items(): - - if key == "capabilities": - capabilities = f"## {key.title()}\n\n" - capabilities += "\n".join([f"* `{v}`" for v in value]) - capabilities += "\n\n" - md_list.append(capabilities) - - if key == "settings": - setting = f"## {key.title()}\n\n" - for k, v in info["settings"].get("properties", {}).items(): - md_description = v.get("description", "").replace("\n", "
") - table_base += ( - f"| {k}{' ' * (max_setting_len - len(k))}" - f"| {'True' if k in required_settings else 'False':8} | " - f"{v.get('default', 'None'):7} | " - f"{md_description:11} |\n" - ) - setting += table_base - setting += ( - "\n" - + "\n".join( - [ - "A full list of supported settings and capabilities " - f"is available by running: `{info['name']} --about`" - ] - ) - + "\n" + Args: + info: The collected metadata for the class. + """ + max_setting_len = cast( + int, max(len(k) for k in info["settings"]["properties"].keys()) + ) + + # Set table base for markdown + table_base = ( + f"| {'Setting':{max_setting_len}}| Required | Default | Description |\n" + f"|:{'-' * max_setting_len}|:--------:|:-------:|:------------|\n" + ) + + # Empty list for string parts + md_list = [] + # Get required settings for table + required_settings = info["settings"].get("required", []) + + # Iterate over Dict to set md + md_list.append( + f"# `{info['name']}`\n\n" + f"{info['description']}\n\n" + f"Built with the [Meltano Singer SDK](https://sdk.meltano.com).\n\n" + ) + for key, value in info.items(): + + if key == "capabilities": + capabilities = f"## {key.title()}\n\n" + capabilities += "\n".join([f"* `{v}`" for v in value]) + capabilities += "\n\n" + md_list.append(capabilities) + + if key == "settings": + setting = f"## {key.title()}\n\n" + for k, v in info["settings"].get("properties", {}).items(): + md_description = v.get("description", "").replace("\n", "
") + table_base += ( + f"| {k}{' ' * (max_setting_len - len(k))}" + f"| {'True' if k in required_settings else 'False':8} | " + f"{v.get('default', 'None'):7} | " + f"{md_description:11} |\n" + ) + setting += table_base + setting += ( + "\n" + + "\n".join( + [ + "A full list of supported settings and capabilities " + f"is available by running: `{info['name']} --about`" + ] ) - md_list.append(setting) + + "\n" + ) + md_list.append(setting) - print("".join(md_list)) - else: - formatted = "\n".join([f"{k.title()}: {v}" for k, v in info.items()]) - print(formatted) + print("".join(md_list)) @classproperty def cli(cls) -> Callable: diff --git a/tests/core/test_meltano_helpers.py b/tests/core/test_meltano_helpers.py new file mode 100644 index 000000000..a0df899f8 --- /dev/null +++ b/tests/core/test_meltano_helpers.py @@ -0,0 +1,136 @@ +"""Test sample sync.""" + +from __future__ import annotations + +import pytest + +from singer_sdk.helpers._meltano import _to_meltano_kind, meltano_yaml_str +from singer_sdk.typing import ( + ArrayType, + BooleanType, + DateTimeType, + DateType, + DurationType, + EmailType, + HostnameType, + IntegerType, + IPv4Type, + IPv6Type, + JSONPointerType, + NumberType, + ObjectType, + PropertiesList, + Property, + RegexType, + RelativeJSONPointerType, + StringType, + TimeType, + URIReferenceType, + URITemplateType, + URIType, + UUIDType, +) + + +@pytest.mark.parametrize( + "plugin_name,capabilities,config_jsonschema,expected_yaml", + [ + ( + "tap-from-source", + ["about", "stream-maps"], + PropertiesList( + Property( + "username", + StringType, + required=True, + description="The username to connect with.", + ), + Property( + "password", + StringType, + required=True, + secret=True, + description="The user's password.", + ), + Property("start_date", DateType), + ).to_dict(), + """name: tap-from-source +namespace: tap_from_source + +## The following could not be auto-detected: +# maintenance_status: # +# repo: # +# variant: # +# label: # +# description: # +# pip_url: # +# domain_url: # +# logo_url: # +# keywords: [] # + +capabilities: + - about + - stream-maps +settings_group_validation: + - - username + - password +settings: +- name: username + label: Username + kind: string + description: The username to connect with. +- name: password + label: Password + kind: password + description: The user's password. +- name: start_date + label: Start Date + kind: date_iso8601 + description: null +""", + ) + ], +) +def test_meltano_yml_creation( + plugin_name: str, + capabilities: list[str], + config_jsonschema: dict, + expected_yaml: str, +): + assert expected_yaml == meltano_yaml_str( + plugin_name, capabilities, config_jsonschema + ) + + +@pytest.mark.parametrize( + "type_dict,expected_kindstr", + [ + # Handled Types: + (StringType.type_dict, "string"), + (DateTimeType.type_dict, "date_iso8601"), + (DateType.type_dict, "date_iso8601"), + (BooleanType.type_dict, "boolean"), + (IntegerType.type_dict, "integer"), + ( + DurationType.type_dict, + "string", + ), + # Treat as strings: + (TimeType.type_dict, "string"), + (EmailType.type_dict, "string"), + (HostnameType.type_dict, "string"), + (IPv4Type.type_dict, "string"), + (IPv6Type.type_dict, "string"), + (UUIDType.type_dict, "string"), + (URIType.type_dict, "string"), + (URIReferenceType.type_dict, "string"), + (URITemplateType.type_dict, "string"), + (JSONPointerType.type_dict, "string"), + (RelativeJSONPointerType.type_dict, "string"), + (RegexType.type_dict, "string"), + # No handling and no compatible default: + (NumberType.type_dict, None), + ], +) +def test_meltano_type_to_kind(type_dict: dict, expected_kindstr: str | None) -> None: + assert _to_meltano_kind(type_dict) == expected_kindstr