diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 2e2ef772b2..668f8b25fd 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -7,7 +7,7 @@ on: pull_request: env: - FLYTE_SDK_LOGGING_LEVEL: 10 # debug + FLYTE_SDK_LOGGING_LEVEL: 10 # debug jobs: build: @@ -68,6 +68,7 @@ jobs: - flytekit-aws-sagemaker - flytekit-bigquery - flytekit-data-fsspec + - flytekit-dbt - flytekit-deck-standard - flytekit-dolt - flytekit-greatexpectations @@ -109,8 +110,6 @@ jobs: # Issue tracked: https://github.com/whylabs/whylogs/issues/697 - python-version: 3.10 plugin-names: "flytekit-whylogs" - - steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -136,7 +135,6 @@ jobs: run: | cd plugins/${{ matrix.plugin-names }} coverage run -m pytest tests - lint: runs-on: ubuntu-latest steps: diff --git a/plugins/README.md b/plugins/README.md index 4986e3445f..b60d998524 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -4,22 +4,23 @@ All the Flytekit plugins maintained by the core team are added here. It is not n ## Currently Available Plugins 🔌 -| Plugin | Installation | Description | Version | Type | -|------------------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|---------------| -| AWS Sagemaker Training | ```bash pip install flytekitplugins-awssagemaker ``` | Installs SDK to author Sagemaker built-in and custom training jobs in python | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-awssagemaker/) | Backend | -| Hive Queries | ```bash pip install flytekitplugins-hive ``` | Installs SDK to author Hive Queries that can be executed on a configured hive backend using Flyte backend plugin | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-hive/) | Backend | -| K8s distributed PyTorch Jobs | ```bash pip install flytekitplugins-kfpytorch ``` | Installs SDK to author Distributed pyTorch Jobs in python using Kubeflow PyTorch Operator | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-kfpytorch/) | Backend | -| K8s native tensorflow Jobs | ```bash pip install flytekitplugins-kftensorflow ``` | Installs SDK to author Distributed tensorflow Jobs in python using Kubeflow Tensorflow Operator | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-kftensorflow/) | Backend | -| K8s native MPI Jobs | ```bash pip install flytekitplugins-kfmpi ``` | Installs SDK to author Distributed MPI Jobs in python using Kubeflow MPI Operator | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-kftensorflow/) | Backend | -| Papermill based Tasks | ```bash pip install flytekitplugins-papermill ``` | Execute entire notebooks as Flyte Tasks and pass inputs and outputs between them and python tasks | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-papermill/) | Flytekit-only | -| Pod Tasks | ```bash pip install flytekitplugins-pod ``` | Installs SDK to author Pods in python. These pods can have multiple containers, use volumes and have non exiting side-cars | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-pod/) | Flytekit-only | -| spark | ```bash pip install flytekitplugins-spark ``` | Installs SDK to author Spark jobs that can be executed natively on Kubernetes with a supported backend Flyte plugin | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-spark/) | Backend | -| AWS Athena Queries | ```bash pip install flytekitplugins-athena ``` | Installs SDK to author queries executed on AWS Athena | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-athena/) | Backend | -| DOLT | ```bash pip install flytekitplugins-dolt ``` | Read & write dolt data sets and use dolt tables as native types | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-dolt/) | Flytekit-only | -| Pandera | ```bash pip install flytekitplugins-pandera ``` | Use Pandera schemas as native Flyte types, which enable data quality checks. | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-pandera/) | Flytekit-only | -| SQLAlchemy | ```bash pip install flytekitplugins-sqlalchemy ``` | Write queries for any database that supports SQLAlchemy | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-sqlalchemy/) | Flytekit-only | -| Great Expectations | ```bash pip install flytekitplugins-great-expectations``` | Enforce data quality for various data types within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-great-expectations.svg)](https://pypi.python.org/pypi/flytekitplugins-great-expectations/) | Flytekit-only | -| Snowflake | ```bash pip install flytekitplugins-snowflake``` | Use Snowflake as a 'data warehouse-as-a-service' within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-great-expectations.svg)](https://pypi.python.org/pypi/flytekitplugins-great-expectations/) | Backend | +| Plugin | Installation | Description | Version | Type | +|------------------------------|-----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------| +| AWS Sagemaker Training | ```bash pip install flytekitplugins-awssagemaker ``` | Installs SDK to author Sagemaker built-in and custom training jobs in python | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-awssagemaker.svg)](https://pypi.python.org/pypi/flytekitplugins-awssagemaker/) | Backend | +| Hive Queries | ```bash pip install flytekitplugins-hive ``` | Installs SDK to author Hive Queries that can be executed on a configured hive backend using Flyte backend plugin | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-hive.svg)](https://pypi.python.org/pypi/flytekitplugins-hive/) | Backend | +| K8s distributed PyTorch Jobs | ```bash pip install flytekitplugins-kfpytorch ``` | Installs SDK to author Distributed pyTorch Jobs in python using Kubeflow PyTorch Operator | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-kfpytorch.svg)](https://pypi.python.org/pypi/flytekitplugins-kfpytorch/) | Backend | +| K8s native tensorflow Jobs | ```bash pip install flytekitplugins-kftensorflow ``` | Installs SDK to author Distributed tensorflow Jobs in python using Kubeflow Tensorflow Operator | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-kftensorflow.svg)](https://pypi.python.org/pypi/flytekitplugins-kftensorflow/) | Backend | +| K8s native MPI Jobs | ```bash pip install flytekitplugins-kfmpi ``` | Installs SDK to author Distributed MPI Jobs in python using Kubeflow MPI Operator | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-kfmpi.svg)](https://pypi.python.org/pypi/flytekitplugins-kfmpi/) | Backend | +| Papermill based Tasks | ```bash pip install flytekitplugins-papermill ``` | Execute entire notebooks as Flyte Tasks and pass inputs and outputs between them and python tasks | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-papermill.svg)](https://pypi.python.org/pypi/flytekitplugins-papermill/) | Flytekit-only | +| Pod Tasks | ```bash pip install flytekitplugins-pod ``` | Installs SDK to author Pods in python. These pods can have multiple containers, use volumes and have non exiting side-cars | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-pod.svg)](https://pypi.python.org/pypi/flytekitplugins-pod/) | Flytekit-only | +| spark | ```bash pip install flytekitplugins-spark ``` | Installs SDK to author Spark jobs that can be executed natively on Kubernetes with a supported backend Flyte plugin | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-spark/) | Backend | +| AWS Athena Queries | ```bash pip install flytekitplugins-athena ``` | Installs SDK to author queries executed on AWS Athena | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-athena.svg)](https://pypi.python.org/pypi/flytekitplugins-athena/) | Backend | +| DOLT | ```bash pip install flytekitplugins-dolt ``` | Read & write dolt data sets and use dolt tables as native types | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-dolt.svg)](https://pypi.python.org/pypi/flytekitplugins-dolt/) | Flytekit-only | +| Pandera | ```bash pip install flytekitplugins-pandera ``` | Use Pandera schemas as native Flyte types, which enable data quality checks. | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-pandera.svg)](https://pypi.python.org/pypi/flytekitplugins-pandera/) | Flytekit-only | +| SQLAlchemy | ```bash pip install flytekitplugins-sqlalchemy ``` | Write queries for any database that supports SQLAlchemy | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-sqlalchemy.svg)](https://pypi.python.org/pypi/flytekitplugins-sqlalchemy/) | Flytekit-only | +| Great Expectations | ```bash pip install flytekitplugins-great-expectations``` | Enforce data quality for various data types within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-great-expectations.svg)](https://pypi.python.org/pypi/flytekitplugins-great-expectations/) | Flytekit-only | +| Snowflake | ```bash pip install flytekitplugins-snowflake``` | Use Snowflake as a 'data warehouse-as-a-service' within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-snowflake.svg)](https://pypi.python.org/pypi/flytekitplugins-snowflake/) | Backend | +| dbt | ```bash pip install flytekitplugins-dbt``` | Run dbt within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-dbt.svg)](https://pypi.python.org/pypi/flytekitplugins-dbt/) | Flytekit-only | ## Have a Plugin Idea? 💡 diff --git a/plugins/flytekit-dbt/README.md b/plugins/flytekit-dbt/README.md new file mode 100644 index 0000000000..5801a9ccb8 --- /dev/null +++ b/plugins/flytekit-dbt/README.md @@ -0,0 +1,15 @@ +# Flytekit dbt plugin + +Flytekit plugin for performing DBT tasks. Currently it supports both `dbt run` and `dbt test` tasks. + +To install the plugin, run the following command: + +```bash +pip install flytekitplugins-dbt +``` + +_Example coming soon!_ + +## Contributors + +- [Gojek](https://www.gojek.io/) diff --git a/plugins/flytekit-dbt/flytekitplugins/dbt/__init__.py b/plugins/flytekit-dbt/flytekitplugins/dbt/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/flytekit-dbt/flytekitplugins/dbt/error.py b/plugins/flytekit-dbt/flytekitplugins/dbt/error.py new file mode 100644 index 0000000000..a617b6e505 --- /dev/null +++ b/plugins/flytekit-dbt/flytekitplugins/dbt/error.py @@ -0,0 +1,49 @@ +from typing import List + + +class DBTHandledError(Exception): + """ + DBTHandledError wraps error logs and message from command execution that returns ``exit code 1``. + + Parameters + ---------- + message : str + Error message. + logs : list of str + Logs produced by the command execution. + + Attributes + ---------- + message : str + Error message. + logs : list of str + Logs produced by the command execution. + """ + + def __init__(self, message: str, logs: List[str]): + self.logs = logs + self.message = message + + +class DBTUnhandledError(Exception): + """ + DBTUnhandledError wraps error logs and message from command execution that returns ``exit code 2``. + + Parameters + ---------- + message : str + Error message. + logs : list of str + Logs produced by the command execution. + + Attributes + ---------- + message : str + Error message. + logs : list of str + Logs produced by the command execution. + """ + + def __init__(self, message: str, logs: List[str]): + self.logs = logs + self.message = message diff --git a/plugins/flytekit-dbt/flytekitplugins/dbt/schema.py b/plugins/flytekit-dbt/flytekitplugins/dbt/schema.py new file mode 100644 index 0000000000..9103ac62ae --- /dev/null +++ b/plugins/flytekit-dbt/flytekitplugins/dbt/schema.py @@ -0,0 +1,205 @@ +import json +from dataclasses import dataclass +from typing import List, Optional + +from dataclasses_json import dataclass_json + + +@dataclass_json +@dataclass +class BaseDBTInput: + """ + Base class for DBT Task Input. + + Attributes + ---------- + project_dir : str + Path to directory containing the DBT ``dbt_project.yml``. + profiles_dir : str + Path to directory containing the DBT ``profiles.yml``. + profile : str + Profile name to be used for the DBT task. It will override value in ``dbt_project.yml``. + target : str + Target to load for the given profile (default=None). + output_path : str + Path to directory where compiled files (e.g. models) will be written when running the task (default=target). + ignore_handled_error : bool + Ignore handled error (exit code = 1) returned by DBT, see https://docs.getdbt.com/reference/exit-codes (default=False). + flags : dict + Dictionary containing CLI flags to be added to the ``dbt run`` command (default=False). + """ + + project_dir: str + profiles_dir: str + profile: str + target: str = None + output_path: str = "target" + ignore_handled_error: bool = False + flags: dict = None + + def to_args(self) -> List[str]: + """ + Convert the instance of BaseDBTInput into list of arguments. + + Returns + ------- + List[str] + List of arguments. + """ + + args = [] + args += ["--project-dir", self.project_dir] + args += ["--profiles-dir", self.profiles_dir] + args += ["--profile", self.profile] + if self.target is not None: + args += ["--target", self.target] + + if self.flags is not None: + for flag, value in self.flags.items(): + if not value: + continue + + args.append(f"--{flag}") + if isinstance(value, bool): + continue + + if isinstance(value, list): + args += value + continue + + if isinstance(value, dict): + args.append(json.dumps(value)) + continue + + args.append(str(value)) + + return args + + +@dataclass_json +@dataclass +class BaseDBTOutput: + """ + Base class for output of DBT task. + + Attributes + ---------- + command : str + Complete CLI command and flags that was executed by DBT Task. + exit_code : int + Exit code returned by DBT CLI. + """ + + command: str + exit_code: int + + +@dataclass_json +@dataclass +class DBTRunInput(BaseDBTInput): + """ + Input to DBT Run task. + + Attributes + ---------- + select : List[str] + List of model to be executed (default=None). + exclude : List[str] + List of model to be excluded (default=None). + """ + + select: Optional[List[str]] = None + exclude: Optional[List[str]] = None + + def to_args(self) -> List[str]: + """ + Convert the instance of BaseDBTInput into list of arguments. + + Returns + ------- + List[str] + List of arguments. + """ + + args = BaseDBTInput.to_args(self) + if self.select is not None: + args += ["--select"] + self.select + + if self.exclude is not None: + args += ["--exclude"] + self.exclude + + return args + + +@dataclass_json +@dataclass +class DBTRunOutput(BaseDBTOutput): + """ + Output of DBT run task. + + Attributes + ---------- + raw_run_result : str + Raw value of DBT's ``run_result.json``. + raw_manifest : str + Raw value of DBT's ``manifest.json``. + """ + + raw_run_result: str + raw_manifest: str + + +@dataclass_json +@dataclass +class DBTTestInput(BaseDBTInput): + """ + Input to DBT Test task. + + Attributes + ---------- + select : List[str] + List of model to be executed (default : None). + exclude : List[str] + List of model to be excluded (default : None). + """ + + select: Optional[List[str]] = None + exclude: Optional[List[str]] = None + + def to_args(self) -> List[str]: + """ + Convert the instance of DBTTestInput into list of arguments. + + Returns + ------- + List[str] + List of arguments. + """ + + args = BaseDBTInput.to_args(self) + + if self.select is not None: + args += ["--select"] + self.select + + if self.exclude is not None: + args += ["--exclude"] + self.exclude + + return args + + +@dataclass_json +@dataclass +class DBTTestOutput(BaseDBTOutput): + """ + Output of DBT test task. + + Attributes + ---------- + raw_run_result : str + Raw value of DBT's ``run_result.json``. + raw_manifest : str + Raw value of DBT's ``manifest.json``. + """ + + raw_run_result: str + raw_manifest: str diff --git a/plugins/flytekit-dbt/flytekitplugins/dbt/task.py b/plugins/flytekit-dbt/flytekitplugins/dbt/task.py new file mode 100644 index 0000000000..5ef03cbfe2 --- /dev/null +++ b/plugins/flytekit-dbt/flytekitplugins/dbt/task.py @@ -0,0 +1,239 @@ +import os + +from flytekitplugins.dbt.error import DBTHandledError, DBTUnhandledError +from flytekitplugins.dbt.schema import DBTRunInput, DBTRunOutput, DBTTestInput, DBTTestOutput +from flytekitplugins.dbt.util import run_cli + +from flytekit import kwtypes +from flytekit.core.interface import Interface +from flytekit.core.python_function_task import PythonInstanceTask +from flytekit.loggers import logger + +SUCCESS = 0 +HANDLED_ERROR_CODE = 1 +UNHANDLED_ERROR_CODE = 2 + + +class DBTRun(PythonInstanceTask): + """ + Execute DBT Run CLI command. + + The task will execute ``dbt run`` CLI command in a subprocess. + Input from :class:`flytekitplugins.dbt.schema.DBTRunInput` will be converted into the corresponding CLI flags + and stored in :class:`flytekitplugins.dbt.schema.DBTRunOutput`'s command. + + Parameters + ---------- + name : str + Task name. + """ + + def __init__( + self, + name: str, + **kwargs, + ): + super(DBTRun, self).__init__( + task_type="dbt-run", + name=name, + task_config=None, + interface=Interface(inputs=kwtypes(input=DBTRunInput), outputs=kwtypes(output=DBTRunOutput)), + **kwargs, + ) + + def execute(self, **kwargs) -> DBTRunOutput: + """ + This method will be invoked to execute the task. + + Example + ------- + :: + + dbt_run_task = DBTRun(name="test-task") + + @workflow + def my_workflow() -> DBTRunOutput: + return dbt_run_task( + input=DBTRunInput( + project_dir="tests/jaffle_shop", + profiles_dir="tests/jaffle_shop/profiles", + profile="jaffle_shop", + ) + ) + + + Parameters + ---------- + input : DBTRunInput + DBT run input. + + Returns + ------- + DBTRunOutput + DBT run output. + + Raises + ------ + DBTHandledError + If the ``dbt run`` command returns ``exit code 1``. + DBTUnhandledError + If the ``dbt run`` command returns ``exit code 2``. + """ + + task_input: DBTRunInput = kwargs["input"] + + args = task_input.to_args() + cmd = ["dbt", "--log-format", "json", "run"] + args + full_command = " ".join(cmd) + + logger.info(f"Executing command: {full_command}") + exit_code, logs = run_cli(cmd) + logger.info(f"dbt exited with return code {exit_code}") + + if exit_code == HANDLED_ERROR_CODE and not task_input.ignore_handled_error: + raise DBTHandledError(f"handled error while executing {full_command}", logs) + + if exit_code == UNHANDLED_ERROR_CODE: + raise DBTUnhandledError(f"unhandled error while executing {full_command}", logs) + + output_dir = os.path.join(task_input.project_dir, task_input.output_path) + run_result_path = os.path.join(output_dir, "run_results.json") + with open(run_result_path) as file: + run_result = file.read() + + # read manifest.json + manifest_path = os.path.join(output_dir, "manifest.json") + with open(manifest_path) as file: + manifest = file.read() + + return DBTRunOutput( + command=full_command, + exit_code=exit_code, + raw_run_result=run_result, + raw_manifest=manifest, + ) + + +class DBTTest(PythonInstanceTask): + """Execute DBT Test CLI command + + The task will execute ``dbt test`` CLI command in a subprocess. + Input from :class:`flytekitplugins.dbt.schema.DBTTestInput` will be converted into the corresponding CLI flags + and stored in :class:`flytekitplugins.dbt.schema.DBTTestOutput`'s command. + + Parameters + ---------- + name : str + Task name. + """ + + def __init__( + self, + name: str, + **kwargs, + ): + super(DBTTest, self).__init__( + task_type="dbt-test", + name=name, + task_config=None, + interface=Interface( + inputs={ + "input": DBTTestInput, + }, + outputs={"output": DBTTestOutput}, + ), + **kwargs, + ) + + def execute(self, **kwargs) -> DBTTestOutput: + """ + This method will be invoked to execute the task. + + Example + ------- + :: + + dbt_test_task = DBTTest(name="test-task") + + @workflow + def my_workflow() -> DBTRunOutput: + # run all models + dbt_test_task( + input=DBTTestInput( + project_dir="tests/jaffle_shop", + profiles_dir="tests/jaffle_shop/profiles", + profile="jaffle_shop", + ) + ) + + # run singular test only + dbt_test_task( + input=DBTTestInput( + project_dir="tests/jaffle_shop", + profiles_dir="tests/jaffle_shop/profiles", + profile="jaffle_shop", + select=["test_type:singular"], + ) + ) + + # run both singular and generic test + return dbt_test_task( + input=DBTTestInput( + project_dir="tests/jaffle_shop", + profiles_dir="tests/jaffle_shop/profiles", + profile="jaffle_shop", + select=["test_type:singular", "test_type:generic"], + ) + ) + + + Parameters + ---------- + input : DBTTestInput + DBT test input + + Returns + ------- + DBTTestOutput + DBT test output + + Raises + ------ + DBTHandledError + If the ``dbt run`` command returns ``exit code 1``. + DBTUnhandledError + If the ``dbt run`` command returns ``exit code 2``. + """ + + task_input: DBTTestInput = kwargs["input"] + + args = task_input.to_args() + cmd = ["dbt", "--log-format", "json", "test"] + args + full_command = " ".join(cmd) + + logger.info(f"Executing command: {full_command}") + exit_code, logs = run_cli(cmd) + logger.info(f"dbt exited with return code {exit_code}") + + if exit_code == HANDLED_ERROR_CODE and not task_input.ignore_handled_error: + raise DBTHandledError(f"handled error while executing {full_command}", logs) + + if exit_code == UNHANDLED_ERROR_CODE: + raise DBTUnhandledError(f"unhandled error while executing {full_command}", logs) + + output_dir = os.path.join(task_input.project_dir, task_input.output_path) + run_result_path = os.path.join(output_dir, "run_results.json") + with open(run_result_path) as file: + run_result = file.read() + + # read manifest.json + manifest_path = os.path.join(output_dir, "manifest.json") + with open(manifest_path) as file: + manifest = file.read() + + return DBTTestOutput( + command=full_command, + exit_code=exit_code, + raw_run_result=run_result, + raw_manifest=manifest, + ) diff --git a/plugins/flytekit-dbt/flytekitplugins/dbt/util.py b/plugins/flytekit-dbt/flytekitplugins/dbt/util.py new file mode 100644 index 0000000000..c127c9279c --- /dev/null +++ b/plugins/flytekit-dbt/flytekitplugins/dbt/util.py @@ -0,0 +1,40 @@ +import json +import subprocess +from typing import List + +from flytekit.loggers import logger + + +def run_cli(cmd: List[str]) -> (int, List[str]): + """ + Execute a CLI command in a subprocess + + Parameters + ---------- + cmd : list of str + Command to be executed. + + Returns + ------- + int + Command's exit code. + list of str + Logs produced by the command execution. + """ + + logs = [] + process = subprocess.Popen(cmd, stdout=subprocess.PIPE) + for raw_line in process.stdout or []: + line = raw_line.decode("utf-8") + try: + json_line = json.loads(line) + except json.JSONDecodeError: + logger.info(line.rstrip()) + else: + logs.append(json_line) + # TODO: pluck `levelname` from json_line and choose appropriate level to use + # in flytekit logger instead of defaulting to `info` + logger.info(line.rstrip()) + + process.wait() + return process.returncode, logs diff --git a/plugins/flytekit-dbt/requirements.in b/plugins/flytekit-dbt/requirements.in new file mode 100644 index 0000000000..57d29c207a --- /dev/null +++ b/plugins/flytekit-dbt/requirements.in @@ -0,0 +1,5 @@ +. +-e file:.#egg=flytekitplugins-dbt +# dbt-sqlite and dbt-core should be compatible +dbt-core==1.2.1 +dbt-sqlite==1.2.0a2 diff --git a/plugins/flytekit-dbt/requirements.txt b/plugins/flytekit-dbt/requirements.txt new file mode 100644 index 0000000000..9720798d7b --- /dev/null +++ b/plugins/flytekit-dbt/requirements.txt @@ -0,0 +1,265 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# pip-compile requirements.in +# +-e file:.#egg=flytekitplugins-dbt + # via -r requirements.in +agate==1.6.3 + # via dbt-core +arrow==1.2.2 + # via jinja2-time +attrs==21.4.0 + # via jsonschema +babel==2.9.1 + # via agate +binaryornot==0.4.4 + # via cookiecutter +certifi==2021.10.8 + # via requests +cffi==1.15.0 + # via + # cryptography + # dbt-core +chardet==4.0.0 + # via binaryornot +charset-normalizer==2.0.12 + # via requests +click==8.0.4 + # via + # cookiecutter + # dbt-core + # flytekit +cloudpickle==2.0.0 + # via flytekit +colorama==0.4.4 + # via dbt-core +cookiecutter==1.7.3 + # via flytekit +croniter==1.3.4 + # via flytekit +cryptography==3.4.8 + # via + # pyopenssl + # secretstorage +dataclasses-json==0.5.6 + # via flytekit +dbt-core==1.2.1 + # via + # -r requirements.in + # dbt-sqlite + # flytekitplugins-dbt +dbt-extractor==0.4.1 + # via dbt-core +dbt-sqlite==1.2.0a2 + # via -r requirements.in +decorator==5.1.1 + # via retry +deprecated==1.2.13 + # via flytekit +diskcache==5.4.0 + # via flytekit +docker==5.0.3 + # via flytekit +docker-image-py==0.1.12 + # via flytekit +docstring-parser==0.13 + # via flytekit +flyteidl==1.1.12 + # via flytekit +flytekit==1.1.1 + # via flytekitplugins-dbt +future==0.18.2 + # via parsedatetime +googleapis-common-protos==1.55.0 + # via + # flyteidl + # grpcio-status +grpcio==1.44.0 + # via + # flytekit + # grpcio-status +grpcio-status==1.44.0 + # via flytekit +hologram==0.0.14 + # via dbt-core +idna==3.3 + # via + # dbt-core + # requests +importlib-metadata==4.11.2 + # via + # flytekit + # jsonschema + # keyring +isodate==0.6.1 + # via + # agate + # dbt-core +jeepney==0.8.0 + # via + # keyring + # secretstorage +jinja2==2.11.3 + # via + # cookiecutter + # dbt-core + # jinja2-time +jinja2-time==0.2.0 + # via cookiecutter +jsonschema==3.1.1 + # via hologram +keyring==23.5.0 + # via flytekit +leather==0.3.4 + # via agate +logbook==1.5.3 + # via dbt-core +markupsafe==2.0.1 + # via + # dbt-core + # jinja2 +marshmallow==3.14.1 + # via + # dataclasses-json + # marshmallow-enum + # marshmallow-jsonschema +marshmallow-enum==1.5.1 + # via dataclasses-json +marshmallow-jsonschema==0.13.0 + # via flytekit +mashumaro==2.9 + # via dbt-core +minimal-snowplow-tracker==0.0.2 + # via dbt-core +msgpack==1.0.3 + # via mashumaro +mypy-extensions==0.4.3 + # via typing-inspect +natsort==8.1.0 + # via flytekit +networkx==2.7.1 + # via dbt-core +numpy==1.22.3 + # via + # pandas + # pyarrow +packaging==21.3 + # via dbt-core +pandas==1.4.1 + # via flytekit +parsedatetime==2.4 + # via agate +poyo==0.5.0 + # via cookiecutter +protobuf==3.19.4 + # via + # flyteidl + # flytekit + # googleapis-common-protos + # grpcio-status + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl +py==1.11.0 + # via retry +pyarrow==6.0.1 + # via flytekit +pycparser==2.21 + # via cffi +pyopenssl==21.0.0 + # via flytekit +pyparsing==3.0.7 + # via packaging +pyrsistent==0.18.1 + # via jsonschema +python-dateutil==2.8.2 + # via + # arrow + # croniter + # flytekit + # hologram + # pandas +python-json-logger==2.0.2 + # via flytekit +python-slugify==6.1.1 + # via + # agate + # cookiecutter +pytimeparse==1.1.8 + # via + # agate + # flytekit +pytz==2021.3 + # via + # babel + # flytekit + # pandas +pyyaml==6.0 + # via + # flytekit + # mashumaro +regex==2022.3.2 + # via docker-image-py +requests==2.27.1 + # via + # cookiecutter + # dbt-core + # docker + # flytekit + # minimal-snowplow-tracker + # responses +responses==0.19.0 + # via flytekit +retry==0.9.2 + # via flytekit +secretstorage==3.3.3 + # via keyring +six==1.16.0 + # via + # agate + # cookiecutter + # grpcio + # isodate + # jsonschema + # leather + # minimal-snowplow-tracker + # pyopenssl + # python-dateutil +sortedcontainers==2.4.0 + # via flytekit +sqlparse==0.4.2 + # via dbt-core +statsd==3.3.0 + # via flytekit +text-unidecode==1.3 + # via python-slugify +typing-extensions==3.10.0.2 + # via + # dbt-core + # flytekit + # mashumaro + # typing-inspect +typing-inspect==0.7.1 + # via dataclasses-json +urllib3==1.26.8 + # via + # flytekit + # requests + # responses +websocket-client==1.3.3 + # via docker +werkzeug==2.0.3 + # via dbt-core +wheel==0.37.1 + # via flytekit +wrapt==1.13.3 + # via + # deprecated + # flytekit +zipp==3.7.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/plugins/flytekit-dbt/setup.py b/plugins/flytekit-dbt/setup.py new file mode 100644 index 0000000000..ab325d82a0 --- /dev/null +++ b/plugins/flytekit-dbt/setup.py @@ -0,0 +1,42 @@ +from setuptools import setup + +PLUGIN_NAME = "dbt" + +microlib_name = f"flytekitplugins-{PLUGIN_NAME}" + +plugin_requires = [ + "flytekit>=1.1.0,<2.0.0", + "dbt-core>=1.0.0", +] + +__version__ = "0.0.0+develop" + +setup( + name=microlib_name, + version=__version__, + author="flyteorg", + author_email="admin@flyte.org", + description="DBT Plugin for Flytekit", + url="https://github.com/flyteorg/flytekit/tree/master/plugins/flytekit-dbt", + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + namespace_packages=["flytekitplugins"], + packages=[f"flytekitplugins.{PLUGIN_NAME}"], + install_requires=plugin_requires, + license="apache2", + python_requires=">=3.7", + classifiers=[ + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + ], +) diff --git a/plugins/flytekit-dbt/tests/__init__.py b/plugins/flytekit-dbt/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/flytekit-dbt/tests/test_schema.py b/plugins/flytekit-dbt/tests/test_schema.py new file mode 100644 index 0000000000..400eb132d4 --- /dev/null +++ b/plugins/flytekit-dbt/tests/test_schema.py @@ -0,0 +1,234 @@ +import shlex + +import pytest +from flytekitplugins.dbt.schema import BaseDBTInput, DBTRunInput, DBTTestInput + +project_dir = "." +profiles_dir = "profiles" +profile_name = "development" + + +class TestBaseDBTInput: + @pytest.mark.parametrize( + "task_input,expected", + [ + ( + BaseDBTInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name}", + ), + ( + BaseDBTInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + target="production", + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --target production", + ), + ( + BaseDBTInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + flags={"vars": {"var1": "val1", "var2": 2}}, + ), + f"""--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --vars '{{"var1": "val1", "var2": 2}}'""", + ), + ( + BaseDBTInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + flags={"bool-flag": True}, + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --bool-flag", + ), + ( + BaseDBTInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + flags={"list-flag": ["a", "b", "c"]}, + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --list-flag a b c", + ), + ], + ) + def test_to_args(self, task_input, expected): + assert task_input.to_args() == shlex.split(expected) + + +class TestDBRunTInput: + @pytest.mark.parametrize( + "task_input,expected", + [ + ( + DBTRunInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name}", + ), + ( + DBTRunInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + select=["model_a", "model_b"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --select model_a model_b", + ), + ( + DBTRunInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + select=["tag:nightly", "my_model", "finance.base.*"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --select tag:nightly my_model finance.base.*", + ), + ( + DBTRunInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + select=["path:marts/finance,tag:nightly,config.materialized:table"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --select path:marts/finance,tag:nightly,config.materialized:table", + ), + ( + DBTRunInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + exclude=["model_a", "model_b"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --exclude model_a model_b", + ), + ( + DBTRunInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + exclude=["tag:nightly", "my_model", "finance.base.*"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --exclude tag:nightly my_model finance.base.*", + ), + ( + DBTRunInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + exclude=["path:marts/finance,tag:nightly,config.materialized:table"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --exclude path:marts/finance,tag:nightly,config.materialized:table", + ), + ], + ) + def test_to_args(self, task_input, expected): + assert task_input.to_args() == shlex.split(expected) + + +class TestDBTestInput: + @pytest.mark.parametrize( + "task_input,expected", + [ + ( + DBTTestInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name}", + ), + ( + DBTTestInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + select=["test_type:singular"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --select test_type:singular", + ), + ( + DBTTestInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + select=["model_a", "model_b"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --select model_a model_b", + ), + ( + DBTTestInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + select=["tag:nightly", "my_model", "finance.base.*"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --select tag:nightly my_model finance.base.*", + ), + ( + DBTTestInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + select=["tag:nightly", "my_model", "finance.base.*", "test_type:singular"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --select tag:nightly my_model finance.base.* test_type:singular", + ), + ( + DBTTestInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + select=["path:marts/finance,tag:nightly,config.materialized:table,test_type:singular"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --select path:marts/finance,tag:nightly,config.materialized:table,test_type:singular", + ), + ( + DBTTestInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + exclude=["model_a", "model_b"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --exclude model_a model_b", + ), + ( + DBTTestInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + exclude=["tag:nightly", "my_model", "finance.base.*"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --exclude tag:nightly my_model finance.base.*", + ), + ( + DBTTestInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + exclude=["path:marts/finance,tag:nightly,config.materialized:table"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --exclude path:marts/finance,tag:nightly,config.materialized:table", + ), + ( + DBTTestInput( + project_dir=project_dir, + profiles_dir=profiles_dir, + profile=profile_name, + select=["test_type:singular"], + exclude=["path:marts/finance,tag:nightly,config.materialized:table"], + ), + f"--project-dir {project_dir} --profiles-dir {profiles_dir} --profile {profile_name} --select test_type:singular --exclude path:marts/finance,tag:nightly,config.materialized:table", + ), + ], + ) + def test_to_args(self, task_input, expected): + assert task_input.to_args() == shlex.split(expected) diff --git a/plugins/flytekit-dbt/tests/test_task.py b/plugins/flytekit-dbt/tests/test_task.py new file mode 100644 index 0000000000..a0f11fb9f4 --- /dev/null +++ b/plugins/flytekit-dbt/tests/test_task.py @@ -0,0 +1,159 @@ +import os +import pathlib + +import pytest +from flytekitplugins.dbt.error import DBTUnhandledError +from flytekitplugins.dbt.schema import DBTRunInput, DBTRunOutput, DBTTestInput, DBTTestOutput +from flytekitplugins.dbt.task import DBTRun, DBTTest + +from flytekit import workflow +from flytekit.tools.subprocess import check_call + +DBT_PROJECT_DIR = str(pathlib.Path(os.path.dirname(os.path.realpath(__file__)), "testdata", "jaffle_shop")) +DBT_PROFILES_DIR = str(pathlib.Path(os.path.dirname(os.path.realpath(__file__)), "testdata", "profiles")) +DBT_PROFILE = "jaffle_shop" + + +@pytest.fixture(scope="module", autouse=True) +def prepare_db(): + # Ensure path to sqlite database file exists + dbs_path = pathlib.Path(DBT_PROJECT_DIR, "dbs") + dbs_path.mkdir(exist_ok=True, parents=True) + database_file = pathlib.Path(dbs_path, "database_name.db") + database_file.touch() + + # Seed the database + check_call( + [ + "dbt", + "--log-format", + "json", + "seed", + "--project-dir", + DBT_PROJECT_DIR, + "--profiles-dir", + DBT_PROFILES_DIR, + "--profile", + DBT_PROFILE, + ] + ) + + yield + + # Delete the database file + database_file.unlink() + + +class TestDBTRun: + def test_simple_task(self): + dbt_run_task = DBTRun( + name="test-task", + ) + + @workflow + def my_workflow() -> DBTRunOutput: + # run all models + return dbt_run_task( + input=DBTRunInput( + project_dir=DBT_PROJECT_DIR, + profiles_dir=DBT_PROFILES_DIR, + profile=DBT_PROFILE, + ) + ) + + result = my_workflow() + assert isinstance(result, DBTRunOutput) + + def test_incorrect_project_dir(self): + dbt_run_task = DBTRun( + name="test-task", + ) + + with pytest.raises(DBTUnhandledError): + dbt_run_task( + input=DBTRunInput( + project_dir=".", + profiles_dir=DBT_PROFILES_DIR, + profile=DBT_PROFILE, + ) + ) + + def test_task_output(self): + dbt_run_task = DBTRun( + name="test-task", + ) + + output = dbt_run_task.execute( + input=DBTRunInput(project_dir=DBT_PROJECT_DIR, profiles_dir=DBT_PROFILES_DIR, profile=DBT_PROFILE) + ) + + assert output.exit_code == 0 + assert ( + output.command + == f"dbt --log-format json run --project-dir {DBT_PROJECT_DIR} --profiles-dir {DBT_PROFILES_DIR} --profile {DBT_PROFILE}" + ) + + with open(f"{DBT_PROJECT_DIR}/target/run_results.json", "r") as fp: + exp_run_result = fp.read() + assert output.raw_run_result == exp_run_result + + with open(f"{DBT_PROJECT_DIR}/target/manifest.json", "r") as fp: + exp_manifest = fp.read() + assert output.raw_manifest == exp_manifest + + +class TestDBTTest: + def test_simple_task(self): + dbt_test_task = DBTTest( + name="test-task", + ) + + @workflow + def test_workflow() -> DBTTestOutput: + # run all tests + return dbt_test_task( + input=DBTTestInput( + project_dir=DBT_PROJECT_DIR, + profiles_dir=DBT_PROFILES_DIR, + profile=DBT_PROFILE, + ) + ) + + assert isinstance(test_workflow(), DBTTestOutput) + + def test_incorrect_project_dir(self): + dbt_test_task = DBTTest( + name="test-task", + ) + + with pytest.raises(DBTUnhandledError): + dbt_test_task( + input=DBTTestInput( + project_dir=".", + profiles_dir=DBT_PROFILES_DIR, + profile=DBT_PROFILE, + ) + ) + + def test_task_output(self): + dbt_test_task = DBTTest( + name="test-task", + ) + + output = dbt_test_task.execute( + input=DBTTestInput(project_dir=DBT_PROJECT_DIR, profiles_dir=DBT_PROFILES_DIR, profile=DBT_PROFILE) + ) + + assert output.exit_code == 0 + assert ( + output.command + == f"dbt --log-format json test --project-dir {DBT_PROJECT_DIR} --profiles-dir {DBT_PROFILES_DIR} --profile {DBT_PROFILE}" + ) + + with open(f"{DBT_PROJECT_DIR}/target/run_results.json", "r") as fp: + exp_run_result = fp.read() + assert output.raw_run_result == exp_run_result + + with open(f"{DBT_PROJECT_DIR}/target/manifest.json", "r") as fp: + exp_manifest = fp.read() + assert output.raw_manifest == exp_manifest diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/.gitignore b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/.gitignore new file mode 100644 index 0000000000..7164422079 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/.gitignore @@ -0,0 +1,5 @@ + +target/ +dbt_modules/ +logs/ +**/.DS_Store diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/LICENSE b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/README.md b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/README.md new file mode 100644 index 0000000000..cd94389ceb --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/README.md @@ -0,0 +1,94 @@ +## Testing dbt project: `jaffle_shop` + +`jaffle_shop` is a fictional ecommerce store. This dbt project transforms raw data from an app database into a customers and orders model ready for analytics. + +### What is this repo? + +What this repo _is_: + +- A self-contained playground dbt project, useful for testing out scripts, and communicating some of the core dbt concepts. + +What this repo _is not_: + +- A tutorial — check out the [Getting Started Tutorial](https://docs.getdbt.com/tutorial/setting-up) for that. Notably, this repo contains some anti-patterns to make it self-contained, namely the use of seeds instead of sources. +- A demonstration of best practices — check out the [dbt Learn Demo](https://github.com/dbt-labs/dbt-learn-demo) repo instead. We want to keep this project as simple as possible. As such, we chose not to implement: + - our standard file naming patterns (which make more sense on larger projects, rather than this five-model project) + - a pull request flow + - CI/CD integrations +- A demonstration of using dbt for a high-complex project, or a demo of advanced features (e.g. macros, packages, hooks, operations) — we're just trying to keep things simple here! + +### What's in this repo? + +This repo contains [seeds](https://docs.getdbt.com/docs/building-a-dbt-project/seeds) that includes some (fake) raw data from a fictional app. + +The raw data consists of customers, orders, and payments, with the following entity-relationship diagram: + +![Jaffle Shop ERD](./etc/jaffle_shop_erd.png) + +### Running this project + +To get up and running with this project: + +1. Install dbt using [these instructions](https://docs.getdbt.com/docs/installation). + +2. Clone this repository. + +3. Change into the `jaffle_shop` directory from the command line: + +```bash +$ cd jaffle_shop +``` + +4. Set up a profile called `jaffle_shop` to connect to a data warehouse by following [these instructions](https://docs.getdbt.com/docs/configure-your-profile). If you have access to a data warehouse, you can use those credentials – we recommend setting your [target schema](https://docs.getdbt.com/docs/configure-your-profile#section-populating-your-profile) to be a new schema (dbt will create the schema for you, as long as you have the right privileges). If you don't have access to an existing data warehouse, you can also setup a local postgres database and connect to it in your profile. + +5. Ensure your profile is setup correctly from the command line: + +```bash +$ dbt debug +``` + +6. Load the CSVs with the demo data set. This materializes the CSVs as tables in your target schema. Note that a typical dbt project **does not require this step** since dbt assumes your raw data is already in your warehouse. + +```bash +$ dbt seed +``` + +7. Run the models: + +```bash +$ dbt run +``` + +> **NOTE:** If this steps fails, it might mean that you need to make small changes to the SQL in the models folder to adjust for the flavor of SQL of your target database. Definitely consider this if you are using a community-contributed adapter. + +8. Test the output of the models: + +```bash +$ dbt test +``` + +9. Generate documentation for the project: + +```bash +$ dbt docs generate +``` + +10. View the documentation for the project: + +```bash +$ dbt docs serve +``` + +### What is a jaffle? + +A jaffle is a toasted sandwich with crimped, sealed edges. Invented in Bondi in 1949, the humble jaffle is an Australian classic. The sealed edges allow jaffle-eaters to enjoy liquid fillings inside the sandwich, which reach temperatures close to the core of the earth during cooking. Often consumed at home after a night out, the most classic filling is tinned spaghetti, while my personal favourite is leftover beef stew with melted cheese. + +--- + +For more information on dbt: + +- Read the [introduction to dbt](https://docs.getdbt.com/docs/introduction). +- Read the [dbt viewpoint](https://docs.getdbt.com/docs/about/viewpoint). +- Join the [dbt community](http://community.getdbt.com/). + +--- diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/dbt_project.yml b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/dbt_project.yml new file mode 100644 index 0000000000..acdce4c57c --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/dbt_project.yml @@ -0,0 +1,26 @@ +name: 'jaffle_shop' + +config-version: 2 +version: '0.1' + +profile: 'jaffle_shop' + +model-paths: ["models"] +seed-paths: ["seeds"] +test-paths: ["tests"] +analysis-paths: ["analysis"] +macro-paths: ["macros"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_modules" + - "logs" + +require-dbt-version: [">=1.0.0", "<2.0.0"] + +models: + jaffle_shop: + materialized: table + staging: + materialized: view diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/etc/dbdiagram_definition.txt b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/etc/dbdiagram_definition.txt new file mode 100644 index 0000000000..3a6e12c079 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/etc/dbdiagram_definition.txt @@ -0,0 +1,23 @@ +Table orders { + id int PK + user_id int + order_date date + status varchar +} + +Table payments { + id int + order_id int + payment_method int + amount int +} + +Table customers { + id int PK + first_name varchar + last_name varchar +} + +Ref: orders.user_id > customers.id + +Ref: payments.order_id > orders.id diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/etc/jaffle_shop_erd.png b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/etc/jaffle_shop_erd.png new file mode 100644 index 0000000000..dd14739095 Binary files /dev/null and b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/etc/jaffle_shop_erd.png differ diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/customers.sql b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/customers.sql new file mode 100644 index 0000000000..016a004fe5 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/customers.sql @@ -0,0 +1,69 @@ +with customers as ( + + select * from {{ ref('stg_customers') }} + +), + +orders as ( + + select * from {{ ref('stg_orders') }} + +), + +payments as ( + + select * from {{ ref('stg_payments') }} + +), + +customer_orders as ( + + select + customer_id, + + min(order_date) as first_order, + max(order_date) as most_recent_order, + count(order_id) as number_of_orders + from orders + + group by customer_id + +), + +customer_payments as ( + + select + orders.customer_id, + sum(amount) as total_amount + + from payments + + left join orders on + payments.order_id = orders.order_id + + group by orders.customer_id + +), + +final as ( + + select + customers.customer_id, + customers.first_name, + customers.last_name, + customer_orders.first_order, + customer_orders.most_recent_order, + customer_orders.number_of_orders, + customer_payments.total_amount as customer_lifetime_value + + from customers + + left join customer_orders + on customers.customer_id = customer_orders.customer_id + + left join customer_payments + on customers.customer_id = customer_payments.customer_id + +) + +select * from final diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/docs.md b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/docs.md new file mode 100644 index 0000000000..c6ae93be07 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/docs.md @@ -0,0 +1,14 @@ +{% docs orders_status %} + +Orders can be one of the following statuses: + +| status | description | +|----------------|------------------------------------------------------------------------------------------------------------------------| +| placed | The order has been placed but has not yet left the warehouse | +| shipped | The order has ben shipped to the customer and is currently in transit | +| completed | The order has been received by the customer | +| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse | +| returned | The order has been returned by the customer and received at the warehouse | + + +{% enddocs %} diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/orders.sql b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/orders.sql new file mode 100644 index 0000000000..cbb2934911 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/orders.sql @@ -0,0 +1,56 @@ +{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %} + +with orders as ( + + select * from {{ ref('stg_orders') }} + +), + +payments as ( + + select * from {{ ref('stg_payments') }} + +), + +order_payments as ( + + select + order_id, + + {% for payment_method in payment_methods -%} + sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount, + {% endfor -%} + + sum(amount) as total_amount + + from payments + + group by order_id + +), + +final as ( + + select + orders.order_id, + orders.customer_id, + orders.order_date, + orders.status, + + {% for payment_method in payment_methods -%} + + order_payments.{{ payment_method }}_amount, + + {% endfor -%} + + order_payments.total_amount as amount + + from orders + + + left join order_payments + on orders.order_id = order_payments.order_id + +) + +select * from final diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/overview.md b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/overview.md new file mode 100644 index 0000000000..0544c42b17 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/overview.md @@ -0,0 +1,11 @@ +{% docs __overview__ %} + +## Data Documentation for Jaffle Shop + +`jaffle_shop` is a fictional ecommerce store. + +This [dbt](https://www.getdbt.com/) project is for testing out code. + +The source code can be found [here](https://github.com/clrcrl/jaffle_shop). + +{% enddocs %} diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/schema.yml b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/schema.yml new file mode 100644 index 0000000000..381349cfda --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/schema.yml @@ -0,0 +1,82 @@ +version: 2 + +models: + - name: customers + description: This table has basic information about a customer, as well as some derived facts based on a customer's orders + + columns: + - name: customer_id + description: This is a unique identifier for a customer + tests: + - unique + - not_null + + - name: first_name + description: Customer's first name. PII. + + - name: last_name + description: Customer's last name. PII. + + - name: first_order + description: Date (UTC) of a customer's first order + + - name: most_recent_order + description: Date (UTC) of a customer's most recent order + + - name: number_of_orders + description: Count of the number of orders a customer has placed + + - name: total_order_amount + description: Total value (AUD) of a customer's orders + + - name: orders + description: This table has basic information about orders, as well as some derived facts based on payments + + columns: + - name: order_id + tests: + - unique + - not_null + description: This is a unique identifier for an order + + - name: customer_id + description: Foreign key to the customers table + tests: + - not_null + - relationships: + to: ref('customers') + field: customer_id + + - name: order_date + description: Date (UTC) that the order was placed + + - name: status + description: '{{ doc("orders_status") }}' + tests: + - accepted_values: + values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] + + - name: amount + description: Total amount (AUD) of the order + tests: + - not_null + + - name: credit_card_amount + description: Amount of the order (AUD) paid for by credit card + tests: + - not_null + + - name: coupon_amount + description: Amount of the order (AUD) paid for by coupon + tests: + - not_null + + - name: bank_transfer_amount + description: Amount of the order (AUD) paid for by bank transfer + tests: + - not_null + + - name: gift_card_amount + description: Amount of the order (AUD) paid for by gift card + tests: + - not_null diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/staging/schema.yml b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/staging/schema.yml new file mode 100644 index 0000000000..c207e4cf52 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/staging/schema.yml @@ -0,0 +1,31 @@ +version: 2 + +models: + - name: stg_customers + columns: + - name: customer_id + tests: + - unique + - not_null + + - name: stg_orders + columns: + - name: order_id + tests: + - unique + - not_null + - name: status + tests: + - accepted_values: + values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] + + - name: stg_payments + columns: + - name: payment_id + tests: + - unique + - not_null + - name: payment_method + tests: + - accepted_values: + values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/staging/stg_customers.sql b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/staging/stg_customers.sql new file mode 100644 index 0000000000..cad0472695 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/staging/stg_customers.sql @@ -0,0 +1,22 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_customers') }} + +), + +renamed as ( + + select + id as customer_id, + first_name, + last_name + + from source + +) + +select * from renamed diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/staging/stg_orders.sql b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/staging/stg_orders.sql new file mode 100644 index 0000000000..a654dcb947 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/staging/stg_orders.sql @@ -0,0 +1,23 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_orders') }} + +), + +renamed as ( + + select + id as order_id, + user_id as customer_id, + order_date, + status + + from source + +) + +select * from renamed diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/staging/stg_payments.sql b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/staging/stg_payments.sql new file mode 100644 index 0000000000..f718596ad0 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/models/staging/stg_payments.sql @@ -0,0 +1,25 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_payments') }} + +), + +renamed as ( + + select + id as payment_id, + order_id, + payment_method, + + -- `amount` is currently stored in cents, so we convert it to dollars + amount / 100 as amount + + from source + +) + +select * from renamed diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/seeds/.gitkeep b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/seeds/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/seeds/raw_customers.csv b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/seeds/raw_customers.csv new file mode 100644 index 0000000000..b3e6747d69 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/seeds/raw_customers.csv @@ -0,0 +1,101 @@ +id,first_name,last_name +1,Michael,P. +2,Shawn,M. +3,Kathleen,P. +4,Jimmy,C. +5,Katherine,R. +6,Sarah,R. +7,Martin,M. +8,Frank,R. +9,Jennifer,F. +10,Henry,W. +11,Fred,S. +12,Amy,D. +13,Kathleen,M. +14,Steve,F. +15,Teresa,H. +16,Amanda,H. +17,Kimberly,R. +18,Johnny,K. +19,Virginia,F. +20,Anna,A. +21,Willie,H. +22,Sean,H. +23,Mildred,A. +24,David,G. +25,Victor,H. +26,Aaron,R. +27,Benjamin,B. +28,Lisa,W. +29,Benjamin,K. +30,Christina,W. +31,Jane,G. +32,Thomas,O. +33,Katherine,M. +34,Jennifer,S. +35,Sara,T. +36,Harold,O. +37,Shirley,J. +38,Dennis,J. +39,Louise,W. +40,Maria,A. +41,Gloria,C. +42,Diana,S. +43,Kelly,N. +44,Jane,R. +45,Scott,B. +46,Norma,C. +47,Marie,P. +48,Lillian,C. +49,Judy,N. +50,Billy,L. +51,Howard,R. +52,Laura,F. +53,Anne,B. +54,Rose,M. +55,Nicholas,R. +56,Joshua,K. +57,Paul,W. +58,Kathryn,K. +59,Adam,A. +60,Norma,W. +61,Timothy,R. +62,Elizabeth,P. +63,Edward,G. +64,David,C. +65,Brenda,W. +66,Adam,W. +67,Michael,H. +68,Jesse,E. +69,Janet,P. +70,Helen,F. +71,Gerald,C. +72,Kathryn,O. +73,Alan,B. +74,Harry,A. +75,Andrea,H. +76,Barbara,W. +77,Anne,W. +78,Harry,H. +79,Jack,R. +80,Phillip,H. +81,Shirley,H. +82,Arthur,D. +83,Virginia,R. +84,Christina,R. +85,Theresa,M. +86,Jason,C. +87,Phillip,B. +88,Adam,T. +89,Margaret,J. +90,Paul,P. +91,Todd,W. +92,Willie,O. +93,Frances,R. +94,Gregory,H. +95,Lisa,P. +96,Jacqueline,A. +97,Shirley,D. +98,Nicole,M. +99,Mary,G. +100,Jean,M. diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/seeds/raw_orders.csv b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/seeds/raw_orders.csv new file mode 100644 index 0000000000..7c2be07888 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/seeds/raw_orders.csv @@ -0,0 +1,100 @@ +id,user_id,order_date,status +1,1,2018-01-01,returned +2,3,2018-01-02,completed +3,94,2018-01-04,completed +4,50,2018-01-05,completed +5,64,2018-01-05,completed +6,54,2018-01-07,completed +7,88,2018-01-09,completed +8,2,2018-01-11,returned +9,53,2018-01-12,completed +10,7,2018-01-14,completed +11,99,2018-01-14,completed +12,59,2018-01-15,completed +13,84,2018-01-17,completed +14,40,2018-01-17,returned +15,25,2018-01-17,completed +16,39,2018-01-18,completed +17,71,2018-01-18,completed +18,64,2018-01-20,returned +19,54,2018-01-22,completed +20,20,2018-01-23,completed +21,71,2018-01-23,completed +22,86,2018-01-24,completed +23,22,2018-01-26,return_pending +24,3,2018-01-27,completed +25,51,2018-01-28,completed +26,32,2018-01-28,completed +27,94,2018-01-29,completed +28,8,2018-01-29,completed +29,57,2018-01-31,completed +30,69,2018-02-02,completed +31,16,2018-02-02,completed +32,28,2018-02-04,completed +33,42,2018-02-04,completed +34,38,2018-02-06,completed +35,80,2018-02-08,completed +36,85,2018-02-10,completed +37,1,2018-02-10,completed +38,51,2018-02-10,completed +39,26,2018-02-11,completed +40,33,2018-02-13,completed +41,99,2018-02-14,completed +42,92,2018-02-16,completed +43,31,2018-02-17,completed +44,66,2018-02-17,completed +45,22,2018-02-17,completed +46,6,2018-02-19,completed +47,50,2018-02-20,completed +48,27,2018-02-21,completed +49,35,2018-02-21,completed +50,51,2018-02-23,completed +51,71,2018-02-24,completed +52,54,2018-02-25,return_pending +53,34,2018-02-26,completed +54,54,2018-02-26,completed +55,18,2018-02-27,completed +56,79,2018-02-28,completed +57,93,2018-03-01,completed +58,22,2018-03-01,completed +59,30,2018-03-02,completed +60,12,2018-03-03,completed +61,63,2018-03-03,completed +62,57,2018-03-05,completed +63,70,2018-03-06,completed +64,13,2018-03-07,completed +65,26,2018-03-08,completed +66,36,2018-03-10,completed +67,79,2018-03-11,completed +68,53,2018-03-11,completed +69,3,2018-03-11,completed +70,8,2018-03-12,completed +71,42,2018-03-12,shipped +72,30,2018-03-14,shipped +73,19,2018-03-16,completed +74,9,2018-03-17,shipped +75,69,2018-03-18,completed +76,25,2018-03-20,completed +77,35,2018-03-21,shipped +78,90,2018-03-23,shipped +79,52,2018-03-23,shipped +80,11,2018-03-23,shipped +81,76,2018-03-23,shipped +82,46,2018-03-24,shipped +83,54,2018-03-24,shipped +84,70,2018-03-26,placed +85,47,2018-03-26,shipped +86,68,2018-03-26,placed +87,46,2018-03-27,placed +88,91,2018-03-27,shipped +89,21,2018-03-28,placed +90,66,2018-03-30,shipped +91,47,2018-03-31,placed +92,84,2018-04-02,placed +93,66,2018-04-03,placed +94,63,2018-04-03,placed +95,27,2018-04-04,placed +96,90,2018-04-06,placed +97,89,2018-04-07,placed +98,41,2018-04-07,placed +99,85,2018-04-09,placed diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/seeds/raw_payments.csv b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/seeds/raw_payments.csv new file mode 100644 index 0000000000..a587baab59 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/seeds/raw_payments.csv @@ -0,0 +1,114 @@ +id,order_id,payment_method,amount +1,1,credit_card,1000 +2,2,credit_card,2000 +3,3,coupon,100 +4,4,coupon,2500 +5,5,bank_transfer,1700 +6,6,credit_card,600 +7,7,credit_card,1600 +8,8,credit_card,2300 +9,9,gift_card,2300 +10,9,bank_transfer,0 +11,10,bank_transfer,2600 +12,11,credit_card,2700 +13,12,credit_card,100 +14,13,credit_card,500 +15,13,bank_transfer,1400 +16,14,bank_transfer,300 +17,15,coupon,2200 +18,16,credit_card,1000 +19,17,bank_transfer,200 +20,18,credit_card,500 +21,18,credit_card,800 +22,19,gift_card,600 +23,20,bank_transfer,1500 +24,21,credit_card,1200 +25,22,bank_transfer,800 +26,23,gift_card,2300 +27,24,coupon,2600 +28,25,bank_transfer,2000 +29,25,credit_card,2200 +30,25,coupon,1600 +31,26,credit_card,3000 +32,27,credit_card,2300 +33,28,bank_transfer,1900 +34,29,bank_transfer,1200 +35,30,credit_card,1300 +36,31,credit_card,1200 +37,32,credit_card,300 +38,33,credit_card,2200 +39,34,bank_transfer,1500 +40,35,credit_card,2900 +41,36,bank_transfer,900 +42,37,credit_card,2300 +43,38,credit_card,1500 +44,39,bank_transfer,800 +45,40,credit_card,1400 +46,41,credit_card,1700 +47,42,coupon,1700 +48,43,gift_card,1800 +49,44,gift_card,1100 +50,45,bank_transfer,500 +51,46,bank_transfer,800 +52,47,credit_card,2200 +53,48,bank_transfer,300 +54,49,credit_card,600 +55,49,credit_card,900 +56,50,credit_card,2600 +57,51,credit_card,2900 +58,51,credit_card,100 +59,52,bank_transfer,1500 +60,53,credit_card,300 +61,54,credit_card,1800 +62,54,bank_transfer,1100 +63,55,credit_card,2900 +64,56,credit_card,400 +65,57,bank_transfer,200 +66,58,coupon,1800 +67,58,gift_card,600 +68,59,gift_card,2800 +69,60,credit_card,400 +70,61,bank_transfer,1600 +71,62,gift_card,1400 +72,63,credit_card,2900 +73,64,bank_transfer,2600 +74,65,credit_card,0 +75,66,credit_card,2800 +76,67,bank_transfer,400 +77,67,credit_card,1900 +78,68,credit_card,1600 +79,69,credit_card,1900 +80,70,credit_card,2600 +81,71,credit_card,500 +82,72,credit_card,2900 +83,73,bank_transfer,300 +84,74,credit_card,3000 +85,75,credit_card,1900 +86,76,coupon,200 +87,77,credit_card,0 +88,77,bank_transfer,1900 +89,78,bank_transfer,2600 +90,79,credit_card,1800 +91,79,credit_card,900 +92,80,gift_card,300 +93,81,coupon,200 +94,82,credit_card,800 +95,83,credit_card,100 +96,84,bank_transfer,2500 +97,85,bank_transfer,1700 +98,86,coupon,2300 +99,87,gift_card,3000 +100,87,credit_card,2600 +101,88,credit_card,2900 +102,89,bank_transfer,2200 +103,90,bank_transfer,200 +104,91,credit_card,1900 +105,92,bank_transfer,1500 +106,92,coupon,200 +107,93,gift_card,2600 +108,94,coupon,700 +109,95,coupon,2400 +110,96,gift_card,1700 +111,97,bank_transfer,1400 +112,98,bank_transfer,1000 +113,99,credit_card,2400 diff --git a/plugins/flytekit-dbt/tests/testdata/jaffle_shop/tests/assert_total_payment_amount_is_positive.sql b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/tests/assert_total_payment_amount_is_positive.sql new file mode 100644 index 0000000000..bfd8ee6b5d --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/jaffle_shop/tests/assert_total_payment_amount_is_positive.sql @@ -0,0 +1,6 @@ +select + order_id, + sum(amount) as total_amount +from {{ ref('orders' )}} +group by 1 +having not(sum(amount) >= 0) diff --git a/plugins/flytekit-dbt/tests/testdata/profiles/profiles.yml b/plugins/flytekit-dbt/tests/testdata/profiles/profiles.yml new file mode 100644 index 0000000000..8d90d76768 --- /dev/null +++ b/plugins/flytekit-dbt/tests/testdata/profiles/profiles.yml @@ -0,0 +1,11 @@ +jaffle_shop: + target: dev + outputs: + dev: + type: sqlite + threads: 1 + database: 'database' + schema: 'main' + schemas_and_paths: + main: 'dbs/database_name.db' + schema_directory: 'file_path' diff --git a/plugins/setup.py b/plugins/setup.py index 96072304d5..8cef499a5d 100644 --- a/plugins/setup.py +++ b/plugins/setup.py @@ -13,6 +13,7 @@ "flytekitplugins-awssagemaker": "flytekit-aws-sagemaker", "flytekitplugins-bigquery": "flytekit-bigquery", "flytekitplugins-fsspec": "flytekit-data-fsspec", + "flytekitplugins-dbt": "flytekit-dbt", "flytekitplugins-dolt": "flytekit-dolt", "flytekitplugins-great_expectations": "flytekit-greatexpectations", "flytekitplugins-hive": "flytekit-hive",