From c45a206a753cdd5fae0a7a685e6ec546059a5a57 Mon Sep 17 00:00:00 2001 From: Pat Nadolny Date: Tue, 11 Jul 2023 16:07:08 -0400 Subject: [PATCH] feat: `--initialize` command to onboard a new account (#80) Closes https://github.com/MeltanoLabs/target-snowflake/issues/21 This is kind of a new idea that we talked about in office hours a while back for leveraging the target's capabilities to configure the database. Related to https://github.com/meltano/hub/issues/1407. Adds a CLI flag for initializing a new account. It interactively prompts for all the information it needs and will execute it if you want it to, otherwise it will just print the sql and you can run it yourself. Based on https://fivetran.com/docs/destinations/snowflake/setup-guide. I created a personal trial snowflake account and was able to initialize using this script then immediately run a tap using the new user/role/warehouse/database. --------- Co-authored-by: Edgar R. M --- README.md | 31 +++++++++++++++++ target_snowflake/connector.py | 49 ++++++++++++++++++++++++++ target_snowflake/initializer.py | 61 +++++++++++++++++++++++++++++++++ target_snowflake/target.py | 42 +++++++++++++++++++++-- 4 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 target_snowflake/initializer.py diff --git a/README.md b/README.md index 1370776..2924168 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,37 @@ Built with the [Meltano Singer SDK](https://sdk.meltano.com). A full list of supported settings and capabilities is available by running: `target-snowflake --about` +### Initializing a Snowflake Account + +This target has an interactive feature that will help you get a Snowflake account initialized with everything needed to get started loading data. + +- User +- Role +- Warehouse +- Database +- Proper grants + +The CLI will ask you to provide information about the new user/role/etc. you want to create but it will also need SYSADMIN credentials to execute the queries. +You should prepare the following inputs: + +- Account +- User that has SYSADMIN and SECURITYADMIN access. These comes default with the user that created the Snowflake account. +- The password for your SYSADMIN user. + +Run the following command to get started with the interactive CLI. +Note - the CLI will print the SQL queries it is planning to run and confirm with you before it makes any changes. + +```bash +poetry run target-snowflake --initialize + +# Alternatively using Meltano CLI +meltano invoke target-snowflake --initialize +``` + +The CLI also has a "dry run" mode that will print the queries without executing them. + +Check out the demo of this [on YouTube](https://youtu.be/9vEFxw-0nxI). + ### Configure using environment variables This Singer target will automatically import any environment variables within the working directory's diff --git a/target_snowflake/connector.py b/target_snowflake/connector.py index 5f0e4a4..e945f41 100644 --- a/target_snowflake/connector.py +++ b/target_snowflake/connector.py @@ -461,3 +461,52 @@ def remove_staged_files(self, sync_id: str) -> None: ) self.logger.debug(f"Removing staged files with SQL: {remove_statement!s}") conn.execute(remove_statement, **kwargs) + + @staticmethod + def get_initialize_script(role, user, password, warehouse, database): + # https://fivetran.com/docs/destinations/snowflake/setup-guide + return f""" + begin; + + -- change role to securityadmin for user / role steps + use role securityadmin; + + -- create role + create role if not exists {role}; + grant role {role} to role SYSADMIN; + + -- create a user + create user if not exists {user} + password = '{password}' + default_role = {role} + default_warehouse = {warehouse}; + + grant role {role} to user {user}; + + -- change role to sysadmin for warehouse / database steps + use role sysadmin; + + -- create a warehouse + create warehouse if not exists {warehouse} + warehouse_size = xsmall + warehouse_type = standard + auto_suspend = 60 + auto_resume = true + initially_suspended = true; + + -- create database + create database if not exists {database}; + + -- grant role access to warehouse + grant USAGE + on warehouse {warehouse} + to role {role}; + + -- grant access to database + grant CREATE SCHEMA, MONITOR, USAGE + on database {database} + to role {role}; + + commit; + + """ diff --git a/target_snowflake/initializer.py b/target_snowflake/initializer.py new file mode 100644 index 0000000..7504daa --- /dev/null +++ b/target_snowflake/initializer.py @@ -0,0 +1,61 @@ +import click +from target_snowflake.connector import SnowflakeConnector +import sys +from sqlalchemy import text + + +def initializer(): + click.echo("") + click.echo("") + click.echo("✨Initializing Snowflake account.✨") + click.echo("Note: You will always be asked to confirm before anything is executed.") + click.echo("") + click.echo("Additionally you can run in `dry_run` mode which will print the SQL without running it.") + dry_run = click.prompt("Would you like to run in `dry_run` mode?", default=False, type=bool) + click.echo("") + click.echo("We will now interactively create (or the print queries) for all the following objects in your Snowflake account:") + click.echo(" - Role") + click.echo(" - User") + click.echo(" - Warehouse") + click.echo(" - Database") + click.echo("") + role = click.prompt("Meltano Role Name:", type=str, default="MELTANO_ROLE") + user = click.prompt("Meltano User Name:", type=str, default="MELTANO_USER") + password = click.prompt("Meltano Password", type=str, confirmation_prompt=True) + warehouse = click.prompt("Meltano Warehouse Name", type=str, default="MELTANO_WAREHOUSE") + database = click.prompt("Meltano Database Name", type=str, default="MELTANO_DATABASE") + script = SnowflakeConnector.get_initialize_script(role, user, password, warehouse, database) + if dry_run: + click.echo(script) + sys.exit(0) + else: + account = click.prompt("Account (i.e. lqnwlrc-onb17812)", type=str) + admin_user = click.prompt("User w/SYSADMIN access", type=str) + admin_pass = click.prompt("User Password", type=str) + connector = SnowflakeConnector( + { + "account": account, + "database": "SNOWFLAKE", + "password": admin_pass, + "role": "SYSADMIN", + "user": admin_user, + } + ) + connector + try: + click.echo("Initialization Started") + with connector._connect() as conn: + click.echo(f"Executing:") + click.echo(f"{script}") + click.prompt("Confirm?", default=True, type=bool) + click.echo("Initialization Started...") + for statement in script.split(';'): + if len(statement.strip()) > 0: + conn.execute( + text(statement) + ) + click.echo("Success!") + click.echo("Initialization Complete") + except Exception as e: + click.echo(f"Initialization Failed: {e}") + sys.exit(1) diff --git a/target_snowflake/target.py b/target_snowflake/target.py index 7d6352d..e54de1e 100644 --- a/target_snowflake/target.py +++ b/target_snowflake/target.py @@ -2,11 +2,15 @@ from __future__ import annotations +import sys + +import click from singer_sdk import typing as th -from singer_sdk.target_base import SQLTarget +from singer_sdk.target_base import SQLTarget, Target +from target_snowflake.initializer import initializer from target_snowflake.sinks import SnowflakeSink -from singer_sdk.sinks import Sink + class TargetSnowflake(SQLTarget): """Target for Snowflake.""" @@ -69,5 +73,39 @@ class TargetSnowflake(SQLTarget): default_sink_class = SnowflakeSink + @classmethod + def cb_inititalize( + cls: type[TargetSnowflake], + ctx: click.Context, + param: click.Option, # noqa: ARG003 + value: bool, # noqa: FBT001 + ) -> None: + if value: + initializer() + ctx.exit() + + @classmethod + def get_singer_command(cls: type[TargetSnowflake]) -> click.Command: + """Execute standard CLI handler for targets. + + Returns: + A click.Command object. + """ + command = super().get_singer_command() + command.params.extend( + [ + click.Option( + ["--initialize"], + is_flag=True, + help="Interactive Snowflake account initialization.", + callback=cls.cb_inititalize, + expose_value=False, + ), + ], + ) + + return command + + if __name__ == "__main__": TargetSnowflake.cli()