-
Notifications
You must be signed in to change notification settings - Fork 300
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* flytekit-dbt plugin * Supports dbt run and dbt test tasks * The plugin includes integration test that need local PostgreSQL database Signed-off-by: ariefrahmansyah <[email protected]> * Revert README.md about unit tests Signed-off-by: ariefrahmansyah <[email protected]> * Merge conflicts Signed-off-by: Eduardo Apolinario <[email protected]> * Update requirements Signed-off-by: Eduardo Apolinario <[email protected]> * Move to dbt-sqlite Signed-off-by: Eduardo Apolinario <[email protected]> * Linting Signed-off-by: Eduardo Apolinario <[email protected]> * Regenerate requirements Signed-off-by: Eduardo Apolinario <[email protected]> * Delete setup_db.sh Signed-off-by: Eduardo Apolinario <[email protected]> * Fix test_task.py tests Signed-off-by: Eduardo Apolinario <[email protected]> * Move testdata to a separate directory Signed-off-by: Eduardo Apolinario <[email protected]> * Delete unused test Signed-off-by: Eduardo Apolinario <[email protected]> * Use flytekit logger Signed-off-by: Eduardo Apolinario <[email protected]> * Use my fork in the plugins tests Signed-off-by: Eduardo Apolinario <[email protected]> * Get string path for call to touch.touch Signed-off-by: Eduardo Apolinario <[email protected]> * Use pathlib.Path.touch Signed-off-by: Eduardo Apolinario <[email protected]> * Remove the touch package Signed-off-by: Eduardo Apolinario <[email protected]> * Simplify CI Signed-off-by: Eduardo Apolinario <[email protected]> * Revert "Simplify CI" This reverts commit 134fa1c. Signed-off-by: Eduardo Apolinario <[email protected]> * Revert "Use my fork in the plugins tests" This reverts commit 02ef380. Signed-off-by: Eduardo Apolinario <[email protected]> * Remove unused file Signed-off-by: Eduardo Apolinario <[email protected]> * Add dbt to README.md Signed-off-by: Eduardo Apolinario <[email protected]> * Remove "Set up postgres" from CI job Signed-off-by: Eduardo Apolinario <[email protected]> * Leave a note in requirements.in Signed-off-by: Eduardo Apolinario <[email protected]> * Remove unused jaffle_shop dir Signed-off-by: Eduardo Apolinario <[email protected]> * Set upperbound flytekit version to 2.0.0 Signed-off-by: Eduardo Apolinario <[email protected]> Signed-off-by: ariefrahmansyah <[email protected]> Signed-off-by: Eduardo Apolinario <[email protected]> Co-authored-by: ariefrahmansyah <[email protected]> Co-authored-by: Eduardo Apolinario <[email protected]>
- Loading branch information
3 people
authored
Sep 17, 2022
1 parent
21ae290
commit cfcccb8
Showing
36 changed files
with
2,287 additions
and
20 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| # Flytekit dbt plugin | ||
|
|
||
| Flytekit plugin for performing DBT tasks. Currently it supports both `dbt run` and `dbt test` tasks. | ||
|
|
||
| To install the plugin, run the following command: | ||
|
|
||
| ```bash | ||
| pip install flytekitplugins-dbt | ||
| ``` | ||
|
|
||
| _Example coming soon!_ | ||
|
|
||
| ## Contributors | ||
|
|
||
| - [Gojek](https://www.gojek.io/) |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| from typing import List | ||
|
|
||
|
|
||
| class DBTHandledError(Exception): | ||
| """ | ||
| DBTHandledError wraps error logs and message from command execution that returns ``exit code 1``. | ||
| Parameters | ||
| ---------- | ||
| message : str | ||
| Error message. | ||
| logs : list of str | ||
| Logs produced by the command execution. | ||
| Attributes | ||
| ---------- | ||
| message : str | ||
| Error message. | ||
| logs : list of str | ||
| Logs produced by the command execution. | ||
| """ | ||
|
|
||
| def __init__(self, message: str, logs: List[str]): | ||
| self.logs = logs | ||
| self.message = message | ||
|
|
||
|
|
||
| class DBTUnhandledError(Exception): | ||
| """ | ||
| DBTUnhandledError wraps error logs and message from command execution that returns ``exit code 2``. | ||
| Parameters | ||
| ---------- | ||
| message : str | ||
| Error message. | ||
| logs : list of str | ||
| Logs produced by the command execution. | ||
| Attributes | ||
| ---------- | ||
| message : str | ||
| Error message. | ||
| logs : list of str | ||
| Logs produced by the command execution. | ||
| """ | ||
|
|
||
| def __init__(self, message: str, logs: List[str]): | ||
| self.logs = logs | ||
| self.message = message |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,205 @@ | ||
| import json | ||
| from dataclasses import dataclass | ||
| from typing import List, Optional | ||
|
|
||
| from dataclasses_json import dataclass_json | ||
|
|
||
|
|
||
| @dataclass_json | ||
| @dataclass | ||
| class BaseDBTInput: | ||
| """ | ||
| Base class for DBT Task Input. | ||
| Attributes | ||
| ---------- | ||
| project_dir : str | ||
| Path to directory containing the DBT ``dbt_project.yml``. | ||
| profiles_dir : str | ||
| Path to directory containing the DBT ``profiles.yml``. | ||
| profile : str | ||
| Profile name to be used for the DBT task. It will override value in ``dbt_project.yml``. | ||
| target : str | ||
| Target to load for the given profile (default=None). | ||
| output_path : str | ||
| Path to directory where compiled files (e.g. models) will be written when running the task (default=target). | ||
| ignore_handled_error : bool | ||
| Ignore handled error (exit code = 1) returned by DBT, see https://docs.getdbt.com/reference/exit-codes (default=False). | ||
| flags : dict | ||
| Dictionary containing CLI flags to be added to the ``dbt run`` command (default=False). | ||
| """ | ||
|
|
||
| project_dir: str | ||
| profiles_dir: str | ||
| profile: str | ||
| target: str = None | ||
| output_path: str = "target" | ||
| ignore_handled_error: bool = False | ||
| flags: dict = None | ||
|
|
||
| def to_args(self) -> List[str]: | ||
| """ | ||
| Convert the instance of BaseDBTInput into list of arguments. | ||
| Returns | ||
| ------- | ||
| List[str] | ||
| List of arguments. | ||
| """ | ||
|
|
||
| args = [] | ||
| args += ["--project-dir", self.project_dir] | ||
| args += ["--profiles-dir", self.profiles_dir] | ||
| args += ["--profile", self.profile] | ||
| if self.target is not None: | ||
| args += ["--target", self.target] | ||
|
|
||
| if self.flags is not None: | ||
| for flag, value in self.flags.items(): | ||
| if not value: | ||
| continue | ||
|
|
||
| args.append(f"--{flag}") | ||
| if isinstance(value, bool): | ||
| continue | ||
|
|
||
| if isinstance(value, list): | ||
| args += value | ||
| continue | ||
|
|
||
| if isinstance(value, dict): | ||
| args.append(json.dumps(value)) | ||
| continue | ||
|
|
||
| args.append(str(value)) | ||
|
|
||
| return args | ||
|
|
||
|
|
||
| @dataclass_json | ||
| @dataclass | ||
| class BaseDBTOutput: | ||
| """ | ||
| Base class for output of DBT task. | ||
| Attributes | ||
| ---------- | ||
| command : str | ||
| Complete CLI command and flags that was executed by DBT Task. | ||
| exit_code : int | ||
| Exit code returned by DBT CLI. | ||
| """ | ||
|
|
||
| command: str | ||
| exit_code: int | ||
|
|
||
|
|
||
| @dataclass_json | ||
| @dataclass | ||
| class DBTRunInput(BaseDBTInput): | ||
| """ | ||
| Input to DBT Run task. | ||
| Attributes | ||
| ---------- | ||
| select : List[str] | ||
| List of model to be executed (default=None). | ||
| exclude : List[str] | ||
| List of model to be excluded (default=None). | ||
| """ | ||
|
|
||
| select: Optional[List[str]] = None | ||
| exclude: Optional[List[str]] = None | ||
|
|
||
| def to_args(self) -> List[str]: | ||
| """ | ||
| Convert the instance of BaseDBTInput into list of arguments. | ||
| Returns | ||
| ------- | ||
| List[str] | ||
| List of arguments. | ||
| """ | ||
|
|
||
| args = BaseDBTInput.to_args(self) | ||
| if self.select is not None: | ||
| args += ["--select"] + self.select | ||
|
|
||
| if self.exclude is not None: | ||
| args += ["--exclude"] + self.exclude | ||
|
|
||
| return args | ||
|
|
||
|
|
||
| @dataclass_json | ||
| @dataclass | ||
| class DBTRunOutput(BaseDBTOutput): | ||
| """ | ||
| Output of DBT run task. | ||
| Attributes | ||
| ---------- | ||
| raw_run_result : str | ||
| Raw value of DBT's ``run_result.json``. | ||
| raw_manifest : str | ||
| Raw value of DBT's ``manifest.json``. | ||
| """ | ||
|
|
||
| raw_run_result: str | ||
| raw_manifest: str | ||
|
|
||
|
|
||
| @dataclass_json | ||
| @dataclass | ||
| class DBTTestInput(BaseDBTInput): | ||
| """ | ||
| Input to DBT Test task. | ||
| Attributes | ||
| ---------- | ||
| select : List[str] | ||
| List of model to be executed (default : None). | ||
| exclude : List[str] | ||
| List of model to be excluded (default : None). | ||
| """ | ||
|
|
||
| select: Optional[List[str]] = None | ||
| exclude: Optional[List[str]] = None | ||
|
|
||
| def to_args(self) -> List[str]: | ||
| """ | ||
| Convert the instance of DBTTestInput into list of arguments. | ||
| Returns | ||
| ------- | ||
| List[str] | ||
| List of arguments. | ||
| """ | ||
|
|
||
| args = BaseDBTInput.to_args(self) | ||
|
|
||
| if self.select is not None: | ||
| args += ["--select"] + self.select | ||
|
|
||
| if self.exclude is not None: | ||
| args += ["--exclude"] + self.exclude | ||
|
|
||
| return args | ||
|
|
||
|
|
||
| @dataclass_json | ||
| @dataclass | ||
| class DBTTestOutput(BaseDBTOutput): | ||
| """ | ||
| Output of DBT test task. | ||
| Attributes | ||
| ---------- | ||
| raw_run_result : str | ||
| Raw value of DBT's ``run_result.json``. | ||
| raw_manifest : str | ||
| Raw value of DBT's ``manifest.json``. | ||
| """ | ||
|
|
||
| raw_run_result: str | ||
| raw_manifest: str |
Oops, something went wrong.