From a48c81bf412a097620d47bd61ea7adb1fc2d09e0 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Tue, 21 Sep 2021 02:51:30 +0530 Subject: [PATCH] update SQLAlchemy docs (#416) * update docs -- SQLAlchemy Signed-off-by: Samhita Alla * resolve conflicts Signed-off-by: Samhita Alla --- .../flytekit_plugins/sql/requirements.txt | 9 ++-- .../sql/sql-alchemy-remote.py | 45 ------------------- .../flytekit_plugins/sql/sql-alchemy.py | 19 ++++---- 3 files changed, 12 insertions(+), 61 deletions(-) delete mode 100644 cookbook/integrations/flytekit_plugins/sql/sql-alchemy-remote.py diff --git a/cookbook/integrations/flytekit_plugins/sql/requirements.txt b/cookbook/integrations/flytekit_plugins/sql/requirements.txt index 21a876e7c..14a9d2448 100644 --- a/cookbook/integrations/flytekit_plugins/sql/requirements.txt +++ b/cookbook/integrations/flytekit_plugins/sql/requirements.txt @@ -2,13 +2,13 @@ # This file is autogenerated by pip-compile with python 3.8 # To update, run: # -# /Applications/Xcode.app/Contents/Developer/usr/bin/make requirements.txt +# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt # attrs==21.2.0 # via scantree certifi==2021.5.30 # via requests -charset-normalizer==2.0.5 +charset-normalizer==2.0.6 # via requests click==7.1.2 # via flytekit @@ -36,7 +36,7 @@ flytekit==0.22.2 # via # -r ../../../common/requirements-common.in # flytekitplugins-sqlalchemy -flytekitplugins-sqlalchemy==0.22.2 +flytekitplugins-sqlalchemy==0.22.3 # via -r requirements.in greenlet==1.1.1 # via sqlalchemy @@ -76,7 +76,7 @@ pathspec==0.9.0 # via scantree pillow==8.3.2 # via matplotlib -protobuf==3.17.3 +protobuf==3.18.0 # via # flyteidl # flytekit @@ -119,7 +119,6 @@ six==1.16.0 # cycler # flytekit # grpcio - # protobuf # python-dateutil # responses # scantree diff --git a/cookbook/integrations/flytekit_plugins/sql/sql-alchemy-remote.py b/cookbook/integrations/flytekit_plugins/sql/sql-alchemy-remote.py deleted file mode 100644 index 5cbfb598e..000000000 --- a/cookbook/integrations/flytekit_plugins/sql/sql-alchemy-remote.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -SQLAlchemy ----------- - -SQLAlchemy is the Python SQL toolkit and Object Relational Mapper that gives application developers the full power and flexibility of SQL. - -That being said, Flyte provides an easy-to-use interface to utilize SQLAlchemy to connect to various SQL Databases. - -The SQLAlchemy task will run with a pre-built container, and thus users needn't build one. -""" - -# %% -# Let's import the libraries. -import pandas -from flytekit import kwtypes, task, workflow -from flytekitplugins.sqlalchemy import SQLAlchemyConfig, SQLAlchemyTask - - -# %% -# We define an SQLAlchemyTask to fetch limited records from a table. Finally, we return the length of the returned DataFrame. -# -# .. note:: -# -# The output of SQLAlchemyTask is a :py:class:`~flytekit.types.schema.FlyteSchema` by default. -@task -def get_length(df: pandas.DataFrame) -> int: - return len(df) - - -sql_task = SQLAlchemyTask( - name="sqlalchemy_task", - query_template="select * from limit {{.inputs.limit}}", - inputs=kwtypes(limit=int), - task_config=SQLAlchemyConfig(uri=""), -) - - -@workflow -def my_wf(limit: int) -> int: - return get_length(df=sql_task(limit=limit)) - - -if __name__ == "__main__": - print(f"Running {__file__} main...") - print(my_wf(limit=3)) diff --git a/cookbook/integrations/flytekit_plugins/sql/sql-alchemy.py b/cookbook/integrations/flytekit_plugins/sql/sql-alchemy.py index a2f6e281b..1ed689383 100644 --- a/cookbook/integrations/flytekit_plugins/sql/sql-alchemy.py +++ b/cookbook/integrations/flytekit_plugins/sql/sql-alchemy.py @@ -4,17 +4,13 @@ SQLAlchemy is the Python SQL toolkit and Object Relational Mapper that gives application developers the full power and flexibility of SQL. -That being said, Flyte provides an easy-to-use interface to utilize SQLAlchemy to connect to various SQL Databases. +Flyte provides an easy-to-use interface to utilize SQLAlchemy to connect to various SQL Databases. In this example, we use a Postgres DB to understand how SQLAlchemy can be used within Flyte. -In this example, we'll use a Postgres DB to understand how you can use SQLAlchemy with Flyte. This task will run with a pre-built container, and thus users needn't build one. You can simply implement the task, then register and execute it immediately. -Install the following packages before running this example (works locally only): - -* `Postgres `__ -* ``pip install flytekitplugins-sqlalchemy`` -* ``pip install psycopg2`` +.. note:: + This example works locally only. For remote, you need to input a valid URI. """ # %% @@ -52,7 +48,7 @@ def create_db( # %% -# We define a ``run_query`` function to create a table and insert data entries into the table. +# We define a ``run_query`` function to create a table and insert data entries into it. def run_query( sql_query: str, conn: psycopg2.extensions.connection, @@ -73,7 +69,7 @@ def run_query( # %% -# We define a helper function to call the required functions. +# We define a helper function. def pg_server() -> str: conn_info = {"database": "flights", "user": "postgres", "password": "postgres"} @@ -131,7 +127,7 @@ def get_length(df: pandas.DataFrame) -> int: and duration <= {{ .inputs.upper_duration_cap }} """, inputs=kwtypes(lower_duration_cap=int, upper_duration_cap=int), - task_config=SQLAlchemyConfig(uri=local_pg_server), + task_config=SQLAlchemyConfig(uri="postgresql://localhost/flights"), ) @@ -147,4 +143,5 @@ def my_wf(lower_duration_cap: int, upper_duration_cap: int) -> int: if __name__ == "__main__": print(f"Starting pg server at {pg_server()}") print(f"Running {__file__} main...") - print(my_wf(lower_duration_cap=600, upper_duration_cap=800)) + print(f"Starting Postgres DB at {pg_server()}") + print(my_wf(lower_duration_cap=300, upper_duration_cap=800))