From a8d3dd22a030d165439adc4dd584469fb809b564 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Fri, 15 Nov 2024 13:58:17 +0530 Subject: [PATCH] Add integration test for source node rendering --- dev/dags/example_source_redering.py | 43 +++++++++++++++++++++++++++++ tests/test_example_dags.py | 2 ++ 2 files changed, 45 insertions(+) create mode 100644 dev/dags/example_source_redering.py diff --git a/dev/dags/example_source_redering.py b/dev/dags/example_source_redering.py new file mode 100644 index 000000000..91607909d --- /dev/null +++ b/dev/dags/example_source_redering.py @@ -0,0 +1,43 @@ +""" +An example DAG that uses Cosmos to render a dbt project into an Airflow DAG using Cosmos source rendering. +""" + +import os +from datetime import datetime +from pathlib import Path + +from cosmos import DbtDag, ProfileConfig, ProjectConfig, RenderConfig +from cosmos.constants import SourceRenderingBehavior +from cosmos.profiles import PostgresUserPasswordProfileMapping + +DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" +DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH)) + +profile_config = ProfileConfig( + profile_name="default", + target_name="dev", + profile_mapping=PostgresUserPasswordProfileMapping( + conn_id="example_conn", + profile_args={"schema": "public"}, + disable_event_tracking=True, + ), +) + +source_rendering_dag = DbtDag( + # dbt/cosmos-specific parameters + project_config=ProjectConfig( + DBT_ROOT_PATH / "jaffle_shop", + ), + profile_config=profile_config, + operator_args={ + "install_deps": True, # install any necessary dependencies before running any dbt command + "full_refresh": True, # used only in dbt commands that support this flag + }, + render_config=RenderConfig(source_rendering_behavior=SourceRenderingBehavior.ALL), + # normal dag parameters + schedule_interval="@daily", + start_date=datetime(2024, 1, 1), + catchup=False, + dag_id="source_rendering_dag", + default_args={"retries": 2}, +) diff --git a/tests/test_example_dags.py b/tests/test_example_dags.py index 2a66faa49..4be51a176 100644 --- a/tests/test_example_dags.py +++ b/tests/test_example_dags.py @@ -79,6 +79,8 @@ def get_dag_bag() -> DagBag: file.writelines(["example_cosmos_sources.py\n"]) if DBT_VERSION < Version("1.6.0"): file.writelines(["example_model_version.py\n"]) + if DBT_VERSION < Version("1.5.0"): + file.writelines(["example_source_rendering.py\n"]) print(".airflowignore contents: ") print(AIRFLOW_IGNORE_FILE.read_text())