From 998683ef26b44bf318ef6628ca00cdd50658e419 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Mon, 22 Jul 2024 14:32:37 +0530 Subject: [PATCH 01/64] Run some example in Kubernetes execution mode in CI --- .github/workflows/test_kubernetes.yml | 19 ++++++ dev/Dockerfile.postgres_profile_docker_k8s | 17 +++++ dev/dags/dbt/jaffle_shop/profiles.yml | 12 ++++ dev/dags/jaffle_shop_kubernetes.py | 67 +++++++++++++++++++ pyproject.toml | 1 + scripts/test/integration-kubernetes.sh | 9 +++ scripts/test/kubernetes-setup.sh | 77 ++++++++++++++++++++++ tests/test_example_dags.py | 11 ++-- tests/test_example_k8s_dags.py | 44 +++++++++++++ 9 files changed, 252 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/test_kubernetes.yml create mode 100644 dev/Dockerfile.postgres_profile_docker_k8s create mode 100644 dev/dags/jaffle_shop_kubernetes.py create mode 100644 scripts/test/integration-kubernetes.sh create mode 100644 scripts/test/kubernetes-setup.sh create mode 100644 tests/test_example_k8s_dags.py diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml new file mode 100644 index 000000000..0f0adf8fa --- /dev/null +++ b/.github/workflows/test_kubernetes.yml @@ -0,0 +1,19 @@ +name: Create Cluster + +on: push + +jobs: + create-cluster: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Kubernetes KinD Cluster + uses: container-tools/kind-action@v1 + - name: Run setup + run: | + sh ./scripts/test/kubernetes-setup.sh + pip install hatch + hatch -e tests.py3.9-2.9 run pip freeze + hatch run tests.py3.9-2.9:test-kubernetes + kubectl get pods -o wide + kubectl logs postgres-postgresql-0 diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s new file mode 100644 index 000000000..e7af7cfc2 --- /dev/null +++ b/dev/Dockerfile.postgres_profile_docker_k8s @@ -0,0 +1,17 @@ +FROM python:3.9 + +RUN pip install dbt-postgres==1.3.1 psycopg2==2.9.3 pytz + +ENV POSTGRES_DATABASE=postgres +ENV POSTGRES_HOST=postgres-postgresql.default.svc.cluster.local +ENV POSTGRES_PASSWORD= +ENV POSTGRES_PORT=5432 +ENV POSTGRES_SCHEMA=public +ENV POSTGRES_USER=postgres + +RUN mkdir /root/.dbt +COPY dags/dbt/jaffle_shop/profiles.yml /root/.dbt/profiles.yml + +RUN mkdir dags +COPY dags dags +RUN rm dags/dbt/jaffle_shop/packages.yml diff --git a/dev/dags/dbt/jaffle_shop/profiles.yml b/dev/dags/dbt/jaffle_shop/profiles.yml index 224f565f4..43f0aae7d 100644 --- a/dev/dags/dbt/jaffle_shop/profiles.yml +++ b/dev/dags/dbt/jaffle_shop/profiles.yml @@ -10,3 +10,15 @@ default: dbname: "{{ env_var('POSTGRES_DB') }}" schema: "{{ env_var('POSTGRES_SCHEMA') }}" threads: 4 + +postgres_profile: + target: dev + outputs: + dev: + type: postgres + dbname: '{{ env_var(''POSTGRES_DATABASE'') }}' + host: '{{ env_var(''POSTGRES_HOST'') }}' + pass: '{{ env_var(''POSTGRES_PASSWORD'') }}' + port: '{{ env_var(''POSTGRES_PORT'') | as_number }}' + schema: '{{ env_var(''POSTGRES_SCHEMA'') }}' + user: '{{ env_var(''POSTGRES_USER'') }}' diff --git a/dev/dags/jaffle_shop_kubernetes.py b/dev/dags/jaffle_shop_kubernetes.py new file mode 100644 index 000000000..10877f5e6 --- /dev/null +++ b/dev/dags/jaffle_shop_kubernetes.py @@ -0,0 +1,67 @@ +""" +## Jaffle Shop DAG +[Jaffle Shop](https://github.com/dbt-labs/jaffle_shop) is a fictional eCommerce store. This dbt project originates from +dbt labs as an example project with dummy data to demonstrate a working dbt core project. This DAG uses the cosmos dbt +parser to generate an Airflow TaskGroup from the dbt project folder. + + +The step-by-step to run this DAG are described in: +https://astronomer.github.io/astronomer-cosmos/getting_started/kubernetes.html#kubernetes + +""" +from airflow import DAG +from airflow.providers.cncf.kubernetes.secret import Secret +from pendulum import datetime + +from cosmos import ( + ProfileConfig, + DbtSeedKubernetesOperator, +) +from cosmos.profiles import PostgresUserPasswordProfileMapping + +DBT_IMAGE = "dbt-jaffle-shop:1.0.0" + + +project_seeds = [ + {"project": "jaffle_shop", "seeds": ["raw_customers", "raw_payments", "raw_orders"]} +] + +postgres_password_secret = Secret( + deploy_type="env", + deploy_target="POSTGRES_PASSWORD", + secret="postgres-secrets", + key="password", +) + +postgres_host_secret = Secret( + deploy_type="env", + deploy_target="POSTGRES_HOST", + secret="postgres-secrets", + key="host", +) + +with DAG( + dag_id="jaffle_shop_kubernetes", + start_date=datetime(2022, 11, 27), + doc_md=__doc__, + catchup=False, +) as dag: + load_seeds = DbtSeedKubernetesOperator( + task_id="load_seeds", + project_dir="dags/dbt/jaffle_shop", + get_logs=True, + schema="public", + image=DBT_IMAGE, + is_delete_operator_pod=False, + secrets=[postgres_password_secret, postgres_host_secret], + profile_config=ProfileConfig( + profile_name="postgres_profile", + target_name="dev", + profile_mapping=PostgresUserPasswordProfileMapping( + conn_id="postgres_default", + profile_args={ + "schema": "public", + } + ) + ) + ) diff --git a/pyproject.toml b/pyproject.toml index a257fa860..4bd09d666 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -168,6 +168,7 @@ freeze = "pip freeze" test = 'sh scripts/test/unit.sh' test-cov = 'sh scripts/test/unit-cov.sh' test-integration = 'sh scripts/test/integration.sh' +test-kubernetes = "sh scripts/test/integration-kubernetes.sh" test-integration-dbt-1-5-4 = 'sh scripts/test/integration-dbt-1-5-4.sh' test-integration-expensive = 'sh scripts/test/integration-expensive.sh' test-integration-setup = 'sh scripts/test/integration-setup.sh' diff --git a/scripts/test/integration-kubernetes.sh b/scripts/test/integration-kubernetes.sh new file mode 100644 index 000000000..2ef58db99 --- /dev/null +++ b/scripts/test/integration-kubernetes.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -x +set -e + + +airflow db reset -y + +pytest tests/test_example_k8s_dags.py diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh new file mode 100644 index 000000000..d37ee71f0 --- /dev/null +++ b/scripts/test/kubernetes-setup.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +set -x +set -e + +check_nodes_ready() { + # Get the list of node statuses + node_statuses=$(kubectl get nodes --no-headers | awk '{print $2}') + # Check if all nodes are in the "Ready" state + for status in $node_statuses; do + if [ "$status" != "Ready" ]; then + return 1 + fi + done + return 0 +} + +wait_for_nodes_ready() { + local max_attempts=60 + local interval=5 + local attempt=0 + + echo "Waiting for nodes in the kind cluster to be in 'Ready' state..." + + while [ $attempt -lt $max_attempts ]; do + if check_nodes_ready; then + echo "All nodes in the kind cluster are in 'Ready' state." + return 0 + else + echo "Nodes are not yet ready. Checking again in $interval seconds..." + sleep $interval + attempt=$((attempt + 1)) + fi + done + + echo "Timeout waiting for nodes in the kind cluster to be in 'Ready' state." + return 1 +} + +kubectl config set-context default + +# Deploy a Postgres pod to Kind +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo update +helm upgrade --install postgres bitnami/postgresql + +# Retrieve the Postgres password and set it as an environment variable +POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) +export POSTGRES_PASSWORD + +# Expose the Postgres to the host running Docker/Kind +kubectl port-forward --namespace default postgres-postgresql-0 5432:5432 & +kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD + +# Create a docker image containing the dbt project files and dbt profile +cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . + +wait_for_nodes_ready + +# Make the build image available in the Kind K8s cluster +kind load docker-image dbt-jaffle-shop:1.0.0 + +# Wait for the kind cluster to be in 'Ready' state +wait_for_nodes_ready + +# For Debugging +echo "nodes" +kubectl get nodes +echo "helm" +helm list +echo "pod service" +kubectl get pods --namespace default +kubectl get svc --namespace default +echo "pg log" +kubectl logs postgres-postgresql-0 -c postgresql +kubectl describe pod postgres-postgresql-0 + diff --git a/tests/test_example_dags.py b/tests/test_example_dags.py index e10093bb5..d25950124 100644 --- a/tests/test_example_dags.py +++ b/tests/test_example_dags.py @@ -7,7 +7,6 @@ except ImportError: from functools import lru_cache as cache - import airflow import pytest from airflow.models.dagbag import DagBag @@ -32,7 +31,6 @@ IGNORED_DAG_FILES = ["performance_dag.py"] - # Sort descending based on Versions and convert string to an actual version MIN_VER_DAG_FILE_VER: dict[Version, list[str]] = { Version(version): MIN_VER_DAG_FILE[version] for version in sorted(MIN_VER_DAG_FILE, key=Version, reverse=True) @@ -51,7 +49,7 @@ def session(): @cache -def get_dag_bag() -> DagBag: +def get_dag_bag(in_kube: bool = False) -> DagBag: """Create a DagBag by adding the files that are not supported to .airflowignore""" if AIRFLOW_VERSION in PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS: return DagBag(dag_folder=None, include_examples=False) @@ -72,6 +70,9 @@ def get_dag_bag() -> DagBag: if DBT_VERSION < Version("1.6.0"): file.writelines(["example_model_version.py\n"]) + if not in_kube: + file.writelines(["jaffle_shop_kubernetes.py\n"]) + print(".airflowignore contents: ") print(AIRFLOW_IGNORE_FILE.read_text()) db = DagBag(EXAMPLE_DAGS_DIR, include_examples=False) @@ -80,8 +81,8 @@ def get_dag_bag() -> DagBag: return db -def get_dag_ids() -> list[str]: - dag_bag = get_dag_bag() +def get_dag_ids(in_kube: bool = False) -> list[str]: + dag_bag = get_dag_bag(in_kube) return dag_bag.dag_ids diff --git a/tests/test_example_k8s_dags.py b/tests/test_example_k8s_dags.py new file mode 100644 index 000000000..213b03950 --- /dev/null +++ b/tests/test_example_k8s_dags.py @@ -0,0 +1,44 @@ +import os +from pathlib import Path + +import pytest +from airflow.models.dagbag import DagBag + +from . import utils as test_utils +from airflow.utils.db import create_default_connections +from airflow.utils.session import provide_session + +EXAMPLE_DAGS_DIR = Path(__file__).parent.parent / "dev/dags" +AIRFLOW_IGNORE_FILE = EXAMPLE_DAGS_DIR / ".airflowignore" + +KUBERNETES_DAG_FILES = ["jaffle_shop_kubernetes.py"] + + +@provide_session +def get_session(session=None): + create_default_connections(session) + return session + + +@pytest.fixture() +def session(): + return get_session() + + +def get_all_dag_files(): + python_files = [] + for file in os.listdir(EXAMPLE_DAGS_DIR): + if file.endswith(".py") and file not in KUBERNETES_DAG_FILES: + python_files.append(file) + + with open(AIRFLOW_IGNORE_FILE, "w+") as dag_ignorefile: + dag_ignorefile.writelines([f"{file}\n" for file in python_files]) + + +@pytest.mark.integration +def test_example_dag_kubernetes(session): + get_all_dag_files() + db = DagBag(EXAMPLE_DAGS_DIR, include_examples=False) + #for dag_id in KUBERNETES_DAG_FILES: + dag = db.get_dag("jaffle_shop_kubernetes") + test_utils.run_dag(dag) From 8c1899a07781fc6e46c3df5ec9f7662db4b383c7 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Tue, 30 Jul 2024 01:58:11 +0530 Subject: [PATCH 02/64] Fix static check --- dev/dags/jaffle_shop_kubernetes.py | 13 ++++++------- scripts/test/kubernetes-setup.sh | 1 - tests/test_example_dags.py | 11 +++++------ tests/test_example_k8s_dags.py | 6 +++--- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/dev/dags/jaffle_shop_kubernetes.py b/dev/dags/jaffle_shop_kubernetes.py index 10877f5e6..a50a8e454 100644 --- a/dev/dags/jaffle_shop_kubernetes.py +++ b/dev/dags/jaffle_shop_kubernetes.py @@ -9,22 +9,21 @@ https://astronomer.github.io/astronomer-cosmos/getting_started/kubernetes.html#kubernetes """ + from airflow import DAG from airflow.providers.cncf.kubernetes.secret import Secret from pendulum import datetime from cosmos import ( - ProfileConfig, DbtSeedKubernetesOperator, + ProfileConfig, ) from cosmos.profiles import PostgresUserPasswordProfileMapping DBT_IMAGE = "dbt-jaffle-shop:1.0.0" -project_seeds = [ - {"project": "jaffle_shop", "seeds": ["raw_customers", "raw_payments", "raw_orders"]} -] +project_seeds = [{"project": "jaffle_shop", "seeds": ["raw_customers", "raw_payments", "raw_orders"]}] postgres_password_secret = Secret( deploy_type="env", @@ -61,7 +60,7 @@ conn_id="postgres_default", profile_args={ "schema": "public", - } - ) - ) + }, + ), + ), ) diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index d37ee71f0..bce7fb58a 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -74,4 +74,3 @@ kubectl get svc --namespace default echo "pg log" kubectl logs postgres-postgresql-0 -c postgresql kubectl describe pod postgres-postgresql-0 - diff --git a/tests/test_example_dags.py b/tests/test_example_dags.py index d25950124..e10093bb5 100644 --- a/tests/test_example_dags.py +++ b/tests/test_example_dags.py @@ -7,6 +7,7 @@ except ImportError: from functools import lru_cache as cache + import airflow import pytest from airflow.models.dagbag import DagBag @@ -31,6 +32,7 @@ IGNORED_DAG_FILES = ["performance_dag.py"] + # Sort descending based on Versions and convert string to an actual version MIN_VER_DAG_FILE_VER: dict[Version, list[str]] = { Version(version): MIN_VER_DAG_FILE[version] for version in sorted(MIN_VER_DAG_FILE, key=Version, reverse=True) @@ -49,7 +51,7 @@ def session(): @cache -def get_dag_bag(in_kube: bool = False) -> DagBag: +def get_dag_bag() -> DagBag: """Create a DagBag by adding the files that are not supported to .airflowignore""" if AIRFLOW_VERSION in PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS: return DagBag(dag_folder=None, include_examples=False) @@ -70,9 +72,6 @@ def get_dag_bag(in_kube: bool = False) -> DagBag: if DBT_VERSION < Version("1.6.0"): file.writelines(["example_model_version.py\n"]) - if not in_kube: - file.writelines(["jaffle_shop_kubernetes.py\n"]) - print(".airflowignore contents: ") print(AIRFLOW_IGNORE_FILE.read_text()) db = DagBag(EXAMPLE_DAGS_DIR, include_examples=False) @@ -81,8 +80,8 @@ def get_dag_bag(in_kube: bool = False) -> DagBag: return db -def get_dag_ids(in_kube: bool = False) -> list[str]: - dag_bag = get_dag_bag(in_kube) +def get_dag_ids() -> list[str]: + dag_bag = get_dag_bag() return dag_bag.dag_ids diff --git a/tests/test_example_k8s_dags.py b/tests/test_example_k8s_dags.py index 213b03950..2b24fd466 100644 --- a/tests/test_example_k8s_dags.py +++ b/tests/test_example_k8s_dags.py @@ -3,11 +3,11 @@ import pytest from airflow.models.dagbag import DagBag - -from . import utils as test_utils from airflow.utils.db import create_default_connections from airflow.utils.session import provide_session +from . import utils as test_utils + EXAMPLE_DAGS_DIR = Path(__file__).parent.parent / "dev/dags" AIRFLOW_IGNORE_FILE = EXAMPLE_DAGS_DIR / ".airflowignore" @@ -39,6 +39,6 @@ def get_all_dag_files(): def test_example_dag_kubernetes(session): get_all_dag_files() db = DagBag(EXAMPLE_DAGS_DIR, include_examples=False) - #for dag_id in KUBERNETES_DAG_FILES: + # for dag_id in KUBERNETES_DAG_FILES: dag = db.get_dag("jaffle_shop_kubernetes") test_utils.run_dag(dag) From 8f333f2cc00a6c5557dc78254cc6605ff4d8514f Mon Sep 17 00:00:00 2001 From: Pankaj Date: Tue, 30 Jul 2024 02:11:18 +0530 Subject: [PATCH 03/64] Update job name --- .github/workflows/test_kubernetes.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index 0f0adf8fa..f8fad55ec 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -1,4 +1,4 @@ -name: Create Cluster +name: Kubernetes Integration Tests on: push @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v3 - name: Kubernetes KinD Cluster uses: container-tools/kind-action@v1 - - name: Run setup + - name: Run tests run: | sh ./scripts/test/kubernetes-setup.sh pip install hatch From 954beed3bbd93d5fc7d05533a192dbf36f2443e1 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Tue, 30 Jul 2024 02:11:48 +0530 Subject: [PATCH 04/64] Update job name --- .github/workflows/test_kubernetes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index f8fad55ec..d8cb96c05 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -3,7 +3,7 @@ name: Kubernetes Integration Tests on: push jobs: - create-cluster: + run-kubernets-tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From a800d4042659fda818165b9e9e260d0c755cdb68 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Tue, 30 Jul 2024 18:03:11 +0530 Subject: [PATCH 05/64] Ignore kubernetes tests in other tests --- scripts/test/integration-dbt-1-5-4.sh | 1 + scripts/test/integration-expensive.sh | 1 + scripts/test/integration-sqlite.sh | 1 + scripts/test/integration.sh | 1 + scripts/test/performance.sh | 3 ++- scripts/test/unit-cov.sh | 3 ++- scripts/test/unit.sh | 3 ++- tests/test_example_dags.py | 5 +++++ 8 files changed, 15 insertions(+), 3 deletions(-) diff --git a/scripts/test/integration-dbt-1-5-4.sh b/scripts/test/integration-dbt-1-5-4.sh index 284f60517..bb936fc21 100644 --- a/scripts/test/integration-dbt-1-5-4.sh +++ b/scripts/test/integration-dbt-1-5-4.sh @@ -10,4 +10,5 @@ pytest -vv \ --durations=0 \ -m integration \ --ignore=tests/perf \ + --ignore=tests/test_example_k8s_dags.py \ -k 'basic_cosmos_task_group' diff --git a/scripts/test/integration-expensive.sh b/scripts/test/integration-expensive.sh index 96c2388cf..a8d94546c 100644 --- a/scripts/test/integration-expensive.sh +++ b/scripts/test/integration-expensive.sh @@ -6,4 +6,5 @@ pytest -vv \ --durations=0 \ -m integration \ --ignore=tests/perf \ + --ignore=tests/test_example_k8s_dags.py \ -k 'example_cosmos_python_models or example_virtualenv' diff --git a/scripts/test/integration-sqlite.sh b/scripts/test/integration-sqlite.sh index dc32324d4..dab70efb4 100644 --- a/scripts/test/integration-sqlite.sh +++ b/scripts/test/integration-sqlite.sh @@ -5,4 +5,5 @@ pytest -vv \ --durations=0 \ -m integration \ --ignore=tests/perf \ + --ignore=tests/test_example_k8s_dags.py \ -k 'example_cosmos_sources or sqlite' diff --git a/scripts/test/integration.sh b/scripts/test/integration.sh index a39ef63b1..3a8a41511 100644 --- a/scripts/test/integration.sh +++ b/scripts/test/integration.sh @@ -20,4 +20,5 @@ pytest -vv \ --durations=0 \ -m integration \ --ignore=tests/perf \ + --ignore=tests/test_example_k8s_dags.py \ -k 'not (sqlite or example_cosmos_sources or example_cosmos_python_models or example_virtualenv)' diff --git a/scripts/test/performance.sh b/scripts/test/performance.sh index ea58c1960..2023026d3 100644 --- a/scripts/test/performance.sh +++ b/scripts/test/performance.sh @@ -2,4 +2,5 @@ pytest -vv \ -s \ -m 'perf' \ --ignore=tests/test_example_dags.py \ - --ignore=tests/test_example_dags_no_connections.py + --ignore=tests/test_example_dags_no_connections.py \ + --ignore=tests/test_example_k8s_dags.py diff --git a/scripts/test/unit-cov.sh b/scripts/test/unit-cov.sh index 89a6244ba..d134c7494 100644 --- a/scripts/test/unit-cov.sh +++ b/scripts/test/unit-cov.sh @@ -7,4 +7,5 @@ pytest \ -m "not (integration or perf)" \ --ignore=tests/perf \ --ignore=tests/test_example_dags.py \ - --ignore=tests/test_example_dags_no_connections.py + --ignore=tests/test_example_dags_no_connections.py \ + --ignore=tests/test_example_k8s_dags.py diff --git a/scripts/test/unit.sh b/scripts/test/unit.sh index ecc1a049a..a60f3c85b 100644 --- a/scripts/test/unit.sh +++ b/scripts/test/unit.sh @@ -4,4 +4,5 @@ pytest \ -m "not (integration or perf)" \ --ignore=tests/perf \ --ignore=tests/test_example_dags.py \ - --ignore=tests/test_example_dags_no_connections.py + --ignore=tests/test_example_dags_no_connections.py \ + --ignore=tests/test_example_k8s_dags.py diff --git a/tests/test_example_dags.py b/tests/test_example_dags.py index e10093bb5..cb246fbf9 100644 --- a/tests/test_example_dags.py +++ b/tests/test_example_dags.py @@ -24,6 +24,7 @@ AIRFLOW_IGNORE_FILE = EXAMPLE_DAGS_DIR / ".airflowignore" DBT_VERSION = Version(get_dbt_version().to_version_string()[1:]) AIRFLOW_VERSION = Version(airflow.__version__) +KUBERNETES_DAG_FILES = ["jaffle_shop_kubernetes.py"] MIN_VER_DAG_FILE: dict[str, list[str]] = { "2.4": ["cosmos_seed_dag.py"], @@ -53,6 +54,10 @@ def session(): @cache def get_dag_bag() -> DagBag: """Create a DagBag by adding the files that are not supported to .airflowignore""" + + with open(AIRFLOW_IGNORE_FILE, "w+") as file: + file.writelines([f"{dag_file}\n" for dag_file in KUBERNETES_DAG_FILES]) + if AIRFLOW_VERSION in PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS: return DagBag(dag_folder=None, include_examples=False) From 2f847f21695c25b90140371ae90a93b2da569978 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Tue, 30 Jul 2024 18:05:56 +0530 Subject: [PATCH 06/64] Update hash --- tests/dbt/test_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 8b526a8ab..70e594761 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -1518,7 +1518,7 @@ def test_save_dbt_ls_cache(mock_variable_set, mock_datetime, tmp_dbt_project_dir hash_dir, hash_args = version.split(",") assert hash_args == "d41d8cd98f00b204e9800998ecf8427e" if sys.platform == "darwin": - assert hash_dir == "a9879ec2ec503b0fe023d059caf50d41" + assert hash_dir == "27970d3415be1f40a2ad43e46436c42d" else: assert hash_dir == "9001bedf4aa8a329f7b669c89f337c24" From bff6f107597332640abafd897eafa71730efea50 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Wed, 31 Jul 2024 02:49:48 +0530 Subject: [PATCH 07/64] Ignore kubernetes dag --- tests/test_example_dags.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_example_dags.py b/tests/test_example_dags.py index cb246fbf9..50e294bd6 100644 --- a/tests/test_example_dags.py +++ b/tests/test_example_dags.py @@ -24,7 +24,7 @@ AIRFLOW_IGNORE_FILE = EXAMPLE_DAGS_DIR / ".airflowignore" DBT_VERSION = Version(get_dbt_version().to_version_string()[1:]) AIRFLOW_VERSION = Version(airflow.__version__) -KUBERNETES_DAG_FILES = ["jaffle_shop_kubernetes.py"] +KUBERNETES_DAGS = ["jaffle_shop_kubernetes"] MIN_VER_DAG_FILE: dict[str, list[str]] = { "2.4": ["cosmos_seed_dag.py"], @@ -55,9 +55,6 @@ def session(): def get_dag_bag() -> DagBag: """Create a DagBag by adding the files that are not supported to .airflowignore""" - with open(AIRFLOW_IGNORE_FILE, "w+") as file: - file.writelines([f"{dag_file}\n" for dag_file in KUBERNETES_DAG_FILES]) - if AIRFLOW_VERSION in PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS: return DagBag(dag_folder=None, include_examples=False) @@ -97,6 +94,8 @@ def get_dag_ids() -> list[str]: @pytest.mark.integration @pytest.mark.parametrize("dag_id", get_dag_ids()) def test_example_dag(session, dag_id: str): + if dag_id in KUBERNETES_DAGS: + return dag_bag = get_dag_bag() dag = dag_bag.get_dag(dag_id) test_utils.run_dag(dag) From 7bb3a70f6e111cec1c47d02e19392d6443c1ff00 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 1 Aug 2024 01:41:09 +0530 Subject: [PATCH 08/64] Add custom liveness param --- scripts/test/kubernetes-setup.sh | 2 +- scripts/test/values.yaml | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 scripts/test/values.yaml diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index bce7fb58a..3aec3dd17 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -42,7 +42,7 @@ kubectl config set-context default # Deploy a Postgres pod to Kind helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update -helm upgrade --install postgres bitnami/postgresql +helm upgrade --install postgres bitnami/postgresql -f scripts/test/values.yaml # Retrieve the Postgres password and set it as an environment variable POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) diff --git a/scripts/test/values.yaml b/scripts/test/values.yaml new file mode 100644 index 000000000..495760f34 --- /dev/null +++ b/scripts/test/values.yaml @@ -0,0 +1,11 @@ +postgresql: + livenessProbe: + exec: + command: + - /bin/sh + - -c + - exec pg_isready -U "postgres" -h 127.0.0.1 -p 5432 + initialDelaySeconds: 120 + timeoutSeconds: 120 + periodSeconds: 120 + successThreshold: 1 From 21a3f71ebb52df5871b8b41bc39c78d9d4c7bcc0 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 1 Aug 2024 01:49:05 +0530 Subject: [PATCH 09/64] Add custom liveness param --- scripts/test/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/values.yaml b/scripts/test/values.yaml index 495760f34..6ef26b4aa 100644 --- a/scripts/test/values.yaml +++ b/scripts/test/values.yaml @@ -4,7 +4,7 @@ postgresql: command: - /bin/sh - -c - - exec pg_isready -U "postgres" -h 127.0.0.1 -p 5432 + - exec pg_isready -U "postgres" -h postgres-postgresql.default.svc.cluster.local -p 5432 initialDelaySeconds: 120 timeoutSeconds: 120 periodSeconds: 120 From a8828a5d065ef577bf24b2f2e8e7c30784867913 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 1 Aug 2024 01:55:48 +0530 Subject: [PATCH 10/64] Adjust port-forwarding --- scripts/test/kubernetes-setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 3aec3dd17..a53313db3 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -49,7 +49,8 @@ POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql - export POSTGRES_PASSWORD # Expose the Postgres to the host running Docker/Kind -kubectl port-forward --namespace default postgres-postgresql-0 5432:5432 & +#kubectl port-forward --namespace default postgres-postgresql-0 5432:5432 & +kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 & kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD # Create a docker image containing the dbt project files and dbt profile From 6f53c99f03c7e60097b2630002c12de1ee8ee0fd Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 1 Aug 2024 02:11:54 +0530 Subject: [PATCH 11/64] Adjust host --- dev/Dockerfile.postgres_profile_docker_k8s | 2 +- scripts/test/kubernetes-setup.sh | 2 +- scripts/test/values.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s index e7af7cfc2..3a718b7e7 100644 --- a/dev/Dockerfile.postgres_profile_docker_k8s +++ b/dev/Dockerfile.postgres_profile_docker_k8s @@ -3,7 +3,7 @@ FROM python:3.9 RUN pip install dbt-postgres==1.3.1 psycopg2==2.9.3 pytz ENV POSTGRES_DATABASE=postgres -ENV POSTGRES_HOST=postgres-postgresql.default.svc.cluster.local +ENV POSTGRES_HOST=127.0.0.1 ENV POSTGRES_PASSWORD= ENV POSTGRES_PORT=5432 ENV POSTGRES_SCHEMA=public diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index a53313db3..7dffb26cb 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -51,7 +51,7 @@ export POSTGRES_PASSWORD # Expose the Postgres to the host running Docker/Kind #kubectl port-forward --namespace default postgres-postgresql-0 5432:5432 & kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 & -kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD +kubectl create secret generic postgres-secrets --from-literal=host=127.0.0.1 --from-literal=password=$POSTGRES_PASSWORD # Create a docker image containing the dbt project files and dbt profile cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . diff --git a/scripts/test/values.yaml b/scripts/test/values.yaml index 6ef26b4aa..495760f34 100644 --- a/scripts/test/values.yaml +++ b/scripts/test/values.yaml @@ -4,7 +4,7 @@ postgresql: command: - /bin/sh - -c - - exec pg_isready -U "postgres" -h postgres-postgresql.default.svc.cluster.local -p 5432 + - exec pg_isready -U "postgres" -h 127.0.0.1 -p 5432 initialDelaySeconds: 120 timeoutSeconds: 120 periodSeconds: 120 From eb681a7f621f317b5c40bf978fcbcf60cb4705a4 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 1 Aug 2024 02:21:24 +0530 Subject: [PATCH 12/64] Add sleep --- scripts/test/kubernetes-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 7dffb26cb..905669808 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -48,6 +48,7 @@ helm upgrade --install postgres bitnami/postgresql -f scripts/test/values.yaml POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) export POSTGRES_PASSWORD +sleep 60 # Expose the Postgres to the host running Docker/Kind #kubectl port-forward --namespace default postgres-postgresql-0 5432:5432 & kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 & From 7c21be6ac8ef24e76471d6502bb576dee773ab10 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 1 Aug 2024 02:38:09 +0530 Subject: [PATCH 13/64] Fix livenessProbe --- scripts/test/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/test/values.yaml b/scripts/test/values.yaml index 495760f34..eb976fc14 100644 --- a/scripts/test/values.yaml +++ b/scripts/test/values.yaml @@ -5,7 +5,7 @@ postgresql: - /bin/sh - -c - exec pg_isready -U "postgres" -h 127.0.0.1 -p 5432 - initialDelaySeconds: 120 - timeoutSeconds: 120 - periodSeconds: 120 + initialDelaySeconds: 60 + timeoutSeconds: 5 + periodSeconds: 10 successThreshold: 1 From e0dd8c15d337f029f80cf1f9457e100166891d0f Mon Sep 17 00:00:00 2001 From: Pankaj Date: Fri, 2 Aug 2024 02:40:13 +0530 Subject: [PATCH 14/64] Fix values.yaml --- dev/Dockerfile.postgres_profile_docker_k8s | 2 +- scripts/test/kubernetes-setup.sh | 7 ++++--- scripts/test/values.yaml | 9 ++------- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s index 3a718b7e7..e7af7cfc2 100644 --- a/dev/Dockerfile.postgres_profile_docker_k8s +++ b/dev/Dockerfile.postgres_profile_docker_k8s @@ -3,7 +3,7 @@ FROM python:3.9 RUN pip install dbt-postgres==1.3.1 psycopg2==2.9.3 pytz ENV POSTGRES_DATABASE=postgres -ENV POSTGRES_HOST=127.0.0.1 +ENV POSTGRES_HOST=postgres-postgresql.default.svc.cluster.local ENV POSTGRES_PASSWORD= ENV POSTGRES_PORT=5432 ENV POSTGRES_SCHEMA=public diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 905669808..24763ddf9 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -42,17 +42,18 @@ kubectl config set-context default # Deploy a Postgres pod to Kind helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update -helm upgrade --install postgres bitnami/postgresql -f scripts/test/values.yaml +helm install postgres bitnami/postgresql -f scripts/test/values.yaml + +sleep 60 # Retrieve the Postgres password and set it as an environment variable POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) export POSTGRES_PASSWORD -sleep 60 # Expose the Postgres to the host running Docker/Kind #kubectl port-forward --namespace default postgres-postgresql-0 5432:5432 & kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 & -kubectl create secret generic postgres-secrets --from-literal=host=127.0.0.1 --from-literal=password=$POSTGRES_PASSWORD +kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD # Create a docker image containing the dbt project files and dbt profile cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . diff --git a/scripts/test/values.yaml b/scripts/test/values.yaml index eb976fc14..ed81866db 100644 --- a/scripts/test/values.yaml +++ b/scripts/test/values.yaml @@ -1,11 +1,6 @@ -postgresql: +primary: livenessProbe: - exec: - command: - - /bin/sh - - -c - - exec pg_isready -U "postgres" -h 127.0.0.1 -p 5432 - initialDelaySeconds: 60 + initialDelaySeconds: 120 timeoutSeconds: 5 periodSeconds: 10 successThreshold: 1 From 6ae63ffca940008f2f34e74fb61732bd590b4a70 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Fri, 2 Aug 2024 02:49:00 +0530 Subject: [PATCH 15/64] Update values.yaml --- scripts/test/values.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/test/values.yaml b/scripts/test/values.yaml index ed81866db..44fcfa83d 100644 --- a/scripts/test/values.yaml +++ b/scripts/test/values.yaml @@ -4,3 +4,15 @@ primary: timeoutSeconds: 5 periodSeconds: 10 successThreshold: 1 + readinessProbe: + enabled: true + initialDelaySeconds: 120 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + startupProbe: + enabled: true + initialDelaySeconds: 120 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 From d4be6711cbdb726a4162eee7a9167abb7428b026 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Fri, 9 Aug 2024 19:29:36 +0530 Subject: [PATCH 16/64] More experiment --- .github/workflows/postgres-deployment.yaml | 22 ++++++++ .github/workflows/postgres-service.yaml | 11 ++++ .github/workflows/test_kubernetes.yml | 60 +++++++++++++++++----- 3 files changed, 79 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/postgres-deployment.yaml create mode 100644 .github/workflows/postgres-service.yaml diff --git a/.github/workflows/postgres-deployment.yaml b/.github/workflows/postgres-deployment.yaml new file mode 100644 index 000000000..85def64af --- /dev/null +++ b/.github/workflows/postgres-deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:latest + env: + - name: POSTGRES_PASSWORD + value: postgres + ports: + - containerPort: 5432 diff --git a/.github/workflows/postgres-service.yaml b/.github/workflows/postgres-service.yaml new file mode 100644 index 000000000..c12c0fed1 --- /dev/null +++ b/.github/workflows/postgres-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres +spec: + selector: + app: postgres + ports: + - protocol: TCP + port: 5432 + targetPort: 5432   diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index d8cb96c05..799274020 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -3,17 +3,49 @@ name: Kubernetes Integration Tests on: push jobs: - run-kubernets-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Kubernetes KinD Cluster - uses: container-tools/kind-action@v1 - - name: Run tests - run: | - sh ./scripts/test/kubernetes-setup.sh - pip install hatch - hatch -e tests.py3.9-2.9 run pip freeze - hatch run tests.py3.9-2.9:test-kubernetes - kubectl get pods -o wide - kubectl logs postgres-postgresql-0 + name: Spin Kind Cluster, Run Postgres, and Expose + + on: + push: + branches: [ main ] + + jobs: + spin-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Kind + uses: actions/setup-docker@v3 + - name: Create Kind Cluster + run: | + kind create cluster + - name: Create PostgreSQL Deployment + run: | + kubectl apply -f postgres-deployment.yaml + - name: Expose PostgreSQL Service + run: | + kubectl apply -f postgres-service.yaml + - name: Port Forward (or Load Balancer Setup) + run: | + kubectl port-forward service/postgres 5432:5432 + - name: Test PostgreSQL Connection + run: | + kubectl get pods -o wide + - name: Tear Down Cluster + run: | + kind delete cluster --name my-cluster +# run-kubernets-tests: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - name: Kubernetes KinD Cluster +# uses: container-tools/kind-action@v1 +# - name: Run tests +# run: | +# sh ./scripts/test/kubernetes-setup.sh +# pip install hatch +# hatch -e tests.py3.9-2.9 run pip freeze +# hatch run tests.py3.9-2.9:test-kubernetes +# kubectl get pods -o wide +# kubectl logs postgres-postgresql-0 + From ae242e73917a9e42b0b0437444b085a286ca34ec Mon Sep 17 00:00:00 2001 From: Pankaj Date: Fri, 9 Aug 2024 19:32:11 +0530 Subject: [PATCH 17/64] Fix --- .github/workflows/test_kubernetes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index 799274020..995f14b91 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -7,7 +7,7 @@ jobs: on: push: - branches: [ main ] + branches: [ kube_mode_ci ] jobs: spin-and-test: From 2a59f347d4927998560961e1396e94718bed5dce Mon Sep 17 00:00:00 2001 From: Pankaj Date: Fri, 9 Aug 2024 19:34:35 +0530 Subject: [PATCH 18/64] Fix --- .github/workflows/test_kubernetes.yml | 61 +++++++++++++-------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index 995f14b91..a0122bb1d 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -1,39 +1,38 @@ name: Kubernetes Integration Tests -on: push -jobs: - name: Spin Kind Cluster, Run Postgres, and Expose +on: + push: # Run on pushes to the default branch + branches: [kube_mode_ci] + pull_request_target: # Also run on pull requests originated from forks + branches: [kube_mode_ci] - on: - push: - branches: [ kube_mode_ci ] - jobs: - spin-and-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Kind - uses: actions/setup-docker@v3 - - name: Create Kind Cluster - run: | - kind create cluster - - name: Create PostgreSQL Deployment - run: | - kubectl apply -f postgres-deployment.yaml - - name: Expose PostgreSQL Service - run: | - kubectl apply -f postgres-service.yaml - - name: Port Forward (or Load Balancer Setup) - run: | - kubectl port-forward service/postgres 5432:5432 - - name: Test PostgreSQL Connection - run: | - kubectl get pods -o wide - - name: Tear Down Cluster - run: | - kind delete cluster --name my-cluster +jobs: + spin-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Kind + uses: actions/setup-docker@v3 + - name: Create Kind Cluster + run: | + kind create cluster + - name: Create PostgreSQL Deployment + run: | + kubectl apply -f postgres-deployment.yaml + - name: Expose PostgreSQL Service + run: | + kubectl apply -f postgres-service.yaml + - name: Port Forward (or Load Balancer Setup) + run: | + kubectl port-forward service/postgres 5432:5432 + - name: Test PostgreSQL Connection + run: | + kubectl get pods -o wide + - name: Tear Down Cluster + run: | + kind delete cluster --name my-cluster # run-kubernets-tests: # runs-on: ubuntu-latest # steps: From b9be16ec154bdf7f4ce1ebcee954590e11f412e5 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Fri, 9 Aug 2024 19:37:43 +0530 Subject: [PATCH 19/64] Fix --- .github/workflows/test_kubernetes.yml | 43 ++++++--------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index a0122bb1d..0ac147c59 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -9,42 +9,17 @@ on: jobs: - spin-and-test: + run-kubernets-tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Kind - uses: actions/setup-docker@v3 - - name: Create Kind Cluster - run: | - kind create cluster - - name: Create PostgreSQL Deployment - run: | - kubectl apply -f postgres-deployment.yaml - - name: Expose PostgreSQL Service - run: | - kubectl apply -f postgres-service.yaml - - name: Port Forward (or Load Balancer Setup) - run: | - kubectl port-forward service/postgres 5432:5432 - - name: Test PostgreSQL Connection + - name: Kubernetes KinD Cluster + uses: container-tools/kind-action@v1 + - name: Run tests run: | + sh ./scripts/test/kubernetes-setup.sh + pip install hatch + hatch -e tests.py3.9-2.9 run pip freeze + hatch run tests.py3.9-2.9:test-kubernetes kubectl get pods -o wide - - name: Tear Down Cluster - run: | - kind delete cluster --name my-cluster -# run-kubernets-tests: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v3 -# - name: Kubernetes KinD Cluster -# uses: container-tools/kind-action@v1 -# - name: Run tests -# run: | -# sh ./scripts/test/kubernetes-setup.sh -# pip install hatch -# hatch -e tests.py3.9-2.9 run pip freeze -# hatch run tests.py3.9-2.9:test-kubernetes -# kubectl get pods -o wide -# kubectl logs postgres-postgresql-0 - + kubectl logs postgres-postgresql-0 From 1c31bbec596ebe7de47f3b7c225f38c2258abe0c Mon Sep 17 00:00:00 2001 From: Pankaj Date: Mon, 12 Aug 2024 18:42:34 +0530 Subject: [PATCH 20/64] Upgrade postgres adapter --- dev/Dockerfile.postgres_profile_docker_k8s | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s index e7af7cfc2..9b4768a7a 100644 --- a/dev/Dockerfile.postgres_profile_docker_k8s +++ b/dev/Dockerfile.postgres_profile_docker_k8s @@ -1,6 +1,6 @@ FROM python:3.9 -RUN pip install dbt-postgres==1.3.1 psycopg2==2.9.3 pytz +RUN pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz ENV POSTGRES_DATABASE=postgres ENV POSTGRES_HOST=postgres-postgresql.default.svc.cluster.local From 385d747780fbada469ef2eee925c0ce8d09fa173 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Mon, 12 Aug 2024 19:18:07 +0530 Subject: [PATCH 21/64] Refactor startup script --- .github/workflows/test_kubernetes.yml | 4 ++-- scripts/test/kubernetes-setup.sh | 21 +++++++++------------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index 0ac147c59..eb92d2b1b 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -2,8 +2,8 @@ name: Kubernetes Integration Tests on: - push: # Run on pushes to the default branch - branches: [kube_mode_ci] +# push: # Run on pushes to the default branch +# branches: [kube_mode_ci] pull_request_target: # Also run on pull requests originated from forks branches: [kube_mode_ci] diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 24763ddf9..9f951be2a 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -39,13 +39,16 @@ wait_for_nodes_ready() { kubectl config set-context default +# Create a docker image containing the dbt project files and dbt profile +cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . +# Make the build image available in the Kind K8s cluster +kind load docker-image dbt-jaffle-shop:1.0.0 + # Deploy a Postgres pod to Kind helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update helm install postgres bitnami/postgresql -f scripts/test/values.yaml -sleep 60 - # Retrieve the Postgres password and set it as an environment variable POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) export POSTGRES_PASSWORD @@ -55,16 +58,10 @@ export POSTGRES_PASSWORD kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 & kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD -# Create a docker image containing the dbt project files and dbt profile -cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . - -wait_for_nodes_ready - -# Make the build image available in the Kind K8s cluster -kind load docker-image dbt-jaffle-shop:1.0.0 - -# Wait for the kind cluster to be in 'Ready' state -wait_for_nodes_ready +#wait_for_nodes_ready +# +## Wait for the kind cluster to be in 'Ready' state +#wait_for_nodes_ready # For Debugging echo "nodes" From ce98d26a0503edbe296502c3be3fd95a192f1259 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Mon, 12 Aug 2024 19:19:53 +0530 Subject: [PATCH 22/64] Fix workflow yml --- .github/workflows/test_kubernetes.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index eb92d2b1b..df37faae5 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -2,10 +2,10 @@ name: Kubernetes Integration Tests on: -# push: # Run on pushes to the default branch -# branches: [kube_mode_ci] - pull_request_target: # Also run on pull requests originated from forks + push: # Run on pushes to the default branch branches: [kube_mode_ci] +# pull_request_target: # Also run on pull requests originated from forks +# branches: [kube_mode_ci] jobs: From e90831b5bbc10330bdb0612fabf321abd9a52067 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Mon, 12 Aug 2024 19:27:29 +0530 Subject: [PATCH 23/64] disable custom config --- scripts/test/kubernetes-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 9f951be2a..a058ce241 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -47,7 +47,7 @@ kind load docker-image dbt-jaffle-shop:1.0.0 # Deploy a Postgres pod to Kind helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update -helm install postgres bitnami/postgresql -f scripts/test/values.yaml +helm install postgres bitnami/postgresql # -f scripts/test/values.yaml # Retrieve the Postgres password and set it as an environment variable POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) From dfc55cb30fc38e64b65b70037890f6502500aba7 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Mon, 12 Aug 2024 19:44:59 +0530 Subject: [PATCH 24/64] Add sleep before debug log --- scripts/test/kubernetes-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index a058ce241..248ecb2ac 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -62,6 +62,7 @@ kubectl create secret generic postgres-secrets --from-literal=host=postgres-post # ## Wait for the kind cluster to be in 'Ready' state #wait_for_nodes_ready +sleep 120 # For Debugging echo "nodes" From fe9da13ed79842235bc9e3b37b0cb9ede52cbf35 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Mon, 12 Aug 2024 19:50:21 +0530 Subject: [PATCH 25/64] Add sleep before port forward --- scripts/test/kubernetes-setup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 248ecb2ac..5c6a7e447 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -53,16 +53,16 @@ helm install postgres bitnami/postgresql # -f scripts/test/values.yaml POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) export POSTGRES_PASSWORD +kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD + +sleep 120 # Expose the Postgres to the host running Docker/Kind #kubectl port-forward --namespace default postgres-postgresql-0 5432:5432 & kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 & -kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD - #wait_for_nodes_ready # ## Wait for the kind cluster to be in 'Ready' state #wait_for_nodes_ready -sleep 120 # For Debugging echo "nodes" From d5d6665743ce51f60babde1c525b44a790afe4ee Mon Sep 17 00:00:00 2001 From: Pankaj Date: Mon, 12 Aug 2024 20:00:56 +0530 Subject: [PATCH 26/64] Add postgres service --- .github/workflows/test_kubernetes.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index df37faae5..35ff2fbfe 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -11,6 +11,23 @@ on: jobs: run-kubernets-tests: runs-on: ubuntu-latest + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:543 steps: - uses: actions/checkout@v3 - name: Kubernetes KinD Cluster From 72d3b1e5a96c62c6cffb975c68eb92584aec46ef Mon Sep 17 00:00:00 2001 From: Pankaj Date: Mon, 12 Aug 2024 20:40:06 +0530 Subject: [PATCH 27/64] disable postgresqlExtendedConf.huge_pages --- .github/workflows/test_kubernetes.yml | 34 +++++++++++++-------------- scripts/test/kubernetes-setup.sh | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index 35ff2fbfe..18e41ad6f 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -11,23 +11,23 @@ on: jobs: run-kubernets-tests: runs-on: ubuntu-latest - services: - # Label used to access the service container - postgres: - # Docker Hub image - image: postgres - # Provide the password for postgres - env: - POSTGRES_PASSWORD: postgres - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - # Maps tcp port 5432 on service container to the host - - 5432:543 +# services: +# # Label used to access the service container +# postgres: +# # Docker Hub image +# image: postgres +# # Provide the password for postgres +# env: +# POSTGRES_PASSWORD: postgres +# # Set health checks to wait until postgres has started +# options: >- +# --health-cmd pg_isready +# --health-interval 10s +# --health-timeout 5s +# --health-retries 5 +# ports: +# # Maps tcp port 5432 on service container to the host +# - 5432:543 steps: - uses: actions/checkout@v3 - name: Kubernetes KinD Cluster diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 5c6a7e447..989e6e402 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -47,7 +47,7 @@ kind load docker-image dbt-jaffle-shop:1.0.0 # Deploy a Postgres pod to Kind helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update -helm install postgres bitnami/postgresql # -f scripts/test/values.yaml +helm install postgres bitnami/postgresql --set postgresqlExtendedConf.huge_pages="off" # -f scripts/test/values.yaml # Retrieve the Postgres password and set it as an environment variable POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) From a07b661b7073d76f91d71fd8345f7f67a72c81f3 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Mon, 12 Aug 2024 21:08:52 +0530 Subject: [PATCH 28/64] Add postgres deployment yml --- .github/workflows/test_kubernetes.yml | 35 ++++++++++++++++++++------- scripts/test/postgres-deployment.yaml | 0 2 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 scripts/test/postgres-deployment.yaml diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index 18e41ad6f..5418a88a1 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -27,16 +27,33 @@ jobs: # --health-retries 5 # ports: # # Maps tcp port 5432 on service container to the host -# - 5432:543 +# - 5432:5432 steps: - uses: actions/checkout@v3 - - name: Kubernetes KinD Cluster + - name: Create Kind Cluster uses: container-tools/kind-action@v1 - - name: Run tests + - name: Create PostgreSQL Deployment + run: kubectl apply -f scripts/test/postgres-deployment.yaml + - name: Wait for PostgreSQL to be ready + run: kubectl wait --for=condition=Available --timeout=300s deployment/postgres + - name: Test PostgreSQL Connection run: | - sh ./scripts/test/kubernetes-setup.sh - pip install hatch - hatch -e tests.py3.9-2.9 run pip freeze - hatch run tests.py3.9-2.9:test-kubernetes - kubectl get pods -o wide - kubectl logs postgres-postgresql-0 + kubectl run postgres-client --rm --tty -i --restart='Never' --image docker.io/bitnami/postgresql:16.4.0-debian-12-r0 --env="PGPASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d)" \ + --command -- psql --host postgres-postgresql -U postgres -d postgres -p 5432 + kubectl get nodes + kubectl get pods --namespace default + kubectl get svc --namespace default + kubectl logs postgres-postgresql-0 -c postgresql + kubectl describe pod postgres-postgresql-0 +# - uses: actions/checkout@v3 +# - name: Kubernetes KinD Cluster +# uses: container-tools/kind-action@v1 +# - name: Run tests +# run: | +# sh ./scripts/test/kubernetes-setup.sh +# pip install hatch +# hatch -e tests.py3.9-2.9 run pip freeze +# hatch run tests.py3.9-2.9:test-kubernetes +# kubectl get pods -o wide +# kubectl logs postgres-postgresql-0 + diff --git a/scripts/test/postgres-deployment.yaml b/scripts/test/postgres-deployment.yaml new file mode 100644 index 000000000..e69de29bb From 68399988e990fb9ebcd64b47e61e37641f8688b7 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Mon, 12 Aug 2024 21:12:30 +0530 Subject: [PATCH 29/64] Fix postgres-deployment.yaml --- scripts/test/postgres-deployment.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scripts/test/postgres-deployment.yaml b/scripts/test/postgres-deployment.yaml index e69de29bb..05aa006f4 100644 --- a/scripts/test/postgres-deployment.yaml +++ b/scripts/test/postgres-deployment.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + + - name: postgres + image: bitnami/postgres:latest # Replace with desired Bitnami Postgres image + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-postgresql + key: postgres-password + ports: + - containerPort: 5432 From ab5a5cffddbd1728aeb727c30fe1ab04c3e292ab Mon Sep 17 00:00:00 2001 From: Pankaj Date: Mon, 12 Aug 2024 21:21:40 +0530 Subject: [PATCH 30/64] Fix postgres port --- .github/workflows/test_kubernetes.yml | 67 ++++++++++----------------- 1 file changed, 25 insertions(+), 42 deletions(-) diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index 5418a88a1..e341a3232 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -11,49 +11,32 @@ on: jobs: run-kubernets-tests: runs-on: ubuntu-latest -# services: -# # Label used to access the service container -# postgres: -# # Docker Hub image -# image: postgres -# # Provide the password for postgres -# env: -# POSTGRES_PASSWORD: postgres -# # Set health checks to wait until postgres has started -# options: >- -# --health-cmd pg_isready -# --health-interval 10s -# --health-timeout 5s -# --health-retries 5 -# ports: -# # Maps tcp port 5432 on service container to the host -# - 5432:5432 + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 steps: - uses: actions/checkout@v3 - - name: Create Kind Cluster + - name: Kubernetes KinD Cluster uses: container-tools/kind-action@v1 - - name: Create PostgreSQL Deployment - run: kubectl apply -f scripts/test/postgres-deployment.yaml - - name: Wait for PostgreSQL to be ready - run: kubectl wait --for=condition=Available --timeout=300s deployment/postgres - - name: Test PostgreSQL Connection + - name: Run tests run: | - kubectl run postgres-client --rm --tty -i --restart='Never' --image docker.io/bitnami/postgresql:16.4.0-debian-12-r0 --env="PGPASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d)" \ - --command -- psql --host postgres-postgresql -U postgres -d postgres -p 5432 - kubectl get nodes - kubectl get pods --namespace default - kubectl get svc --namespace default - kubectl logs postgres-postgresql-0 -c postgresql - kubectl describe pod postgres-postgresql-0 -# - uses: actions/checkout@v3 -# - name: Kubernetes KinD Cluster -# uses: container-tools/kind-action@v1 -# - name: Run tests -# run: | -# sh ./scripts/test/kubernetes-setup.sh -# pip install hatch -# hatch -e tests.py3.9-2.9 run pip freeze -# hatch run tests.py3.9-2.9:test-kubernetes -# kubectl get pods -o wide -# kubectl logs postgres-postgresql-0 - + sh ./scripts/test/kubernetes-setup.sh + pip install hatch + hatch -e tests.py3.9-2.9 run pip freeze + hatch run tests.py3.9-2.9:test-kubernetes + kubectl get pods -o wide + kubectl logs postgres-postgresql-0 From d04b99739c8fc800c5c6637569093a91c961604d Mon Sep 17 00:00:00 2001 From: Pankaj Date: Tue, 13 Aug 2024 01:58:26 +0530 Subject: [PATCH 31/64] backup --- .github/workflows/test_kubernetes.yml | 19 --- dev/Dockerfile.postgres_profile_docker_k8s | 2 +- dev/dags/dbt/jaffle_shop/profiles.yml | 12 +- scripts/test/integration-kubernetes.sh | 1 + scripts/test/kubernetes-setup.sh | 140 +++++++++++---------- scripts/test/postgres-deployment.yaml | 39 +++++- 6 files changed, 118 insertions(+), 95 deletions(-) diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index e341a3232..3987752b5 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -11,23 +11,6 @@ on: jobs: run-kubernets-tests: runs-on: ubuntu-latest - services: - # Label used to access the service container - postgres: - # Docker Hub image - image: postgres - # Provide the password for postgres - env: - POSTGRES_PASSWORD: postgres - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - # Maps tcp port 5432 on service container to the host - - 5432:5432 steps: - uses: actions/checkout@v3 - name: Kubernetes KinD Cluster @@ -38,5 +21,3 @@ jobs: pip install hatch hatch -e tests.py3.9-2.9 run pip freeze hatch run tests.py3.9-2.9:test-kubernetes - kubectl get pods -o wide - kubectl logs postgres-postgresql-0 diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s index 9b4768a7a..c369f144b 100644 --- a/dev/Dockerfile.postgres_profile_docker_k8s +++ b/dev/Dockerfile.postgres_profile_docker_k8s @@ -3,7 +3,7 @@ FROM python:3.9 RUN pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz ENV POSTGRES_DATABASE=postgres -ENV POSTGRES_HOST=postgres-postgresql.default.svc.cluster.local +ENV POSTGRES_HOST=127.0.0.1 ENV POSTGRES_PASSWORD= ENV POSTGRES_PORT=5432 ENV POSTGRES_SCHEMA=public diff --git a/dev/dags/dbt/jaffle_shop/profiles.yml b/dev/dags/dbt/jaffle_shop/profiles.yml index 43f0aae7d..b5fa5caa2 100644 --- a/dev/dags/dbt/jaffle_shop/profiles.yml +++ b/dev/dags/dbt/jaffle_shop/profiles.yml @@ -16,9 +16,9 @@ postgres_profile: outputs: dev: type: postgres - dbname: '{{ env_var(''POSTGRES_DATABASE'') }}' - host: '{{ env_var(''POSTGRES_HOST'') }}' - pass: '{{ env_var(''POSTGRES_PASSWORD'') }}' - port: '{{ env_var(''POSTGRES_PORT'') | as_number }}' - schema: '{{ env_var(''POSTGRES_SCHEMA'') }}' - user: '{{ env_var(''POSTGRES_USER'') }}' + dbname: postgres #'{{ env_var(''POSTGRES_DATABASE'') }}' + host: "0.0.0.0" #'{{ env_var(''POSTGRES_HOST'') }}' + pass: postgres #'{{ env_var(''POSTGRES_PASSWORD'') }}' + port: 5432 #'{{ env_var(''POSTGRES_PORT'') | as_number }}' + schema: postgres # '{{ env_var(''POSTGRES_SCHEMA'') }}' + user: "postgres" #'{{ env_var(''POSTGRES_USER'') }}' diff --git a/scripts/test/integration-kubernetes.sh b/scripts/test/integration-kubernetes.sh index 2ef58db99..de7b46675 100644 --- a/scripts/test/integration-kubernetes.sh +++ b/scripts/test/integration-kubernetes.sh @@ -3,6 +3,7 @@ set -x set -e +# export POSTGRES_DB=postgres airflow db reset -y diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 989e6e402..a8727cea1 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -3,75 +3,85 @@ set -x set -e -check_nodes_ready() { - # Get the list of node statuses - node_statuses=$(kubectl get nodes --no-headers | awk '{print $2}') - # Check if all nodes are in the "Ready" state - for status in $node_statuses; do - if [ "$status" != "Ready" ]; then - return 1 - fi - done - return 0 -} - -wait_for_nodes_ready() { - local max_attempts=60 - local interval=5 - local attempt=0 - - echo "Waiting for nodes in the kind cluster to be in 'Ready' state..." +#check_nodes_ready() { +# # Get the list of node statuses +# node_statuses=$(kubectl get nodes --no-headers | awk '{print $2}') +# # Check if all nodes are in the "Ready" state +# for status in $node_statuses; do +# if [ "$status" != "Ready" ]; then +# return 1 +# fi +# done +# return 0 +#} +# +#wait_for_nodes_ready() { +# local max_attempts=60 +# local interval=5 +# local attempt=0 +# +# echo "Waiting for nodes in the kind cluster to be in 'Ready' state..." +# +# while [ $attempt -lt $max_attempts ]; do +# if check_nodes_ready; then +# echo "All nodes in the kind cluster are in 'Ready' state." +# return 0 +# else +# echo "Nodes are not yet ready. Checking again in $interval seconds..." +# sleep $interval +# attempt=$((attempt + 1)) +# fi +# done +# +# echo "Timeout waiting for nodes in the kind cluster to be in 'Ready' state." +# return 1 +#} +# +#kubectl config set-context default +# +## Create a docker image containing the dbt project files and dbt profile +#cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . +## Make the build image available in the Kind K8s cluster +#kind load docker-image dbt-jaffle-shop:1.0.0 +# +## Deploy a Postgres pod to Kind +##helm repo add bitnami https://charts.bitnami.com/bitnami +##helm repo update +##helm install postgres bitnami/postgresql --set postgresqlExtendedConf.huge_pages="off" # -f scripts/test/values.yaml +# +## Retrieve the Postgres password and set it as an environment variable +##POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) +##export POSTGRES_PASSWORD +# +#kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD +# +#sleep 120 +## Expose the Postgres to the host running Docker/Kind +##kubectl port-forward --namespace default postgres-postgresql-0 5432:5432 & +##kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 & +##wait_for_nodes_ready +## +### Wait for the kind cluster to be in 'Ready' state +##wait_for_nodes_ready +# +## For Debugging +#echo "nodes" +#kubectl get nodes +#echo "helm" +#helm list +#echo "pod service" +#kubectl get pods --namespace default +#kubectl get svc --namespace default +#echo "pg log" +#kubectl logs postgres-postgresql-0 -c postgresql +#kubectl describe pod postgres-postgresql-0 - while [ $attempt -lt $max_attempts ]; do - if check_nodes_ready; then - echo "All nodes in the kind cluster are in 'Ready' state." - return 0 - else - echo "Nodes are not yet ready. Checking again in $interval seconds..." - sleep $interval - attempt=$((attempt + 1)) - fi - done - echo "Timeout waiting for nodes in the kind cluster to be in 'Ready' state." - return 1 -} +kubectl create secret generic postgres-secrets --from-literal=host=0.0.0.0 --from-literal=password=postgres -kubectl config set-context default +kubectl apply -f scripts/test/postgres-deployment.yaml -# Create a docker image containing the dbt project files and dbt profile cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . -# Make the build image available in the Kind K8s cluster kind load docker-image dbt-jaffle-shop:1.0.0 -# Deploy a Postgres pod to Kind -helm repo add bitnami https://charts.bitnami.com/bitnami -helm repo update -helm install postgres bitnami/postgresql --set postgresqlExtendedConf.huge_pages="off" # -f scripts/test/values.yaml - -# Retrieve the Postgres password and set it as an environment variable -POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) -export POSTGRES_PASSWORD - -kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD - -sleep 120 -# Expose the Postgres to the host running Docker/Kind -#kubectl port-forward --namespace default postgres-postgresql-0 5432:5432 & -kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 & -#wait_for_nodes_ready -# -## Wait for the kind cluster to be in 'Ready' state -#wait_for_nodes_ready - -# For Debugging -echo "nodes" -kubectl get nodes -echo "helm" -helm list -echo "pod service" -kubectl get pods --namespace default -kubectl get svc --namespace default -echo "pg log" -kubectl logs postgres-postgresql-0 -c postgresql -kubectl describe pod postgres-postgresql-0 +kubectl get pod \ No newline at end of file diff --git a/scripts/test/postgres-deployment.yaml b/scripts/test/postgres-deployment.yaml index 05aa006f4..473439cff 100644 --- a/scripts/test/postgres-deployment.yaml +++ b/scripts/test/postgres-deployment.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: postgres + namespace: default spec: replicas: 1 selector: @@ -13,14 +14,44 @@ spec: app: postgres spec: containers: - - name: postgres - image: bitnami/postgres:latest # Replace with desired Bitnami Postgres image + image: bitnami/postgresql:latest env: + - name: POSTGRES_DB + value: postgres + - name: POSTGRES_HOST + valueFrom: + secretKeyRef: + name: postgres-secrets + key: host # Adjust according to your secret's key - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: postgres-postgresql - key: postgres-password + name: postgres-secrets + key: password ports: - containerPort: 5432 + +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secrets +type: Opaque +data: + host: MC4wLjAuMA== + password: cG9zdGdyZXM= + +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: default # Ensure this matches the namespace of the Deployment +spec: + type: LoadBalancer + ports: + - port: 5432 + targetPort: 5432 + selector: + app: postgres From 6751282976e319328b550f3b95fbbec02d681fce Mon Sep 17 00:00:00 2001 From: Pankaj Date: Tue, 13 Aug 2024 02:06:35 +0530 Subject: [PATCH 32/64] add port-forward --- scripts/test/kubernetes-setup.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index a8727cea1..76f489244 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -84,4 +84,10 @@ kubectl apply -f scripts/test/postgres-deployment.yaml cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . kind load docker-image dbt-jaffle-shop:1.0.0 +POD_NAME=$(kubectl get pods -n default -l app=postgres -o jsonpath='{.items[0].metadata.name}') + +echo "$POD_NAME" + +kubectl port-forward --namespace default "$POD_NAME" 5432:5432 & + kubectl get pod \ No newline at end of file From 9a657420085b27ad71d28e2d36d78d8899ace368 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 00:40:46 +0530 Subject: [PATCH 33/64] local working version --- dev/Dockerfile.postgres_profile_docker_k8s | 2 +- dev/dags/dbt/jaffle_shop/profiles.yml | 4 ++-- scripts/test/kubernetes-setup.sh | 4 ++-- scripts/test/postgres-deployment.yaml | 10 ++++++---- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s index c369f144b..1634616c5 100644 --- a/dev/Dockerfile.postgres_profile_docker_k8s +++ b/dev/Dockerfile.postgres_profile_docker_k8s @@ -3,7 +3,7 @@ FROM python:3.9 RUN pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz ENV POSTGRES_DATABASE=postgres -ENV POSTGRES_HOST=127.0.0.1 +ENV POSTGRES_HOST=postgres.default.svc.cluster.local ENV POSTGRES_PASSWORD= ENV POSTGRES_PORT=5432 ENV POSTGRES_SCHEMA=public diff --git a/dev/dags/dbt/jaffle_shop/profiles.yml b/dev/dags/dbt/jaffle_shop/profiles.yml index b5fa5caa2..db1f34acd 100644 --- a/dev/dags/dbt/jaffle_shop/profiles.yml +++ b/dev/dags/dbt/jaffle_shop/profiles.yml @@ -17,8 +17,8 @@ postgres_profile: dev: type: postgres dbname: postgres #'{{ env_var(''POSTGRES_DATABASE'') }}' - host: "0.0.0.0" #'{{ env_var(''POSTGRES_HOST'') }}' + host: postgres.default.svc.cluster.local #'{{ env_var(''POSTGRES_HOST'') }}' pass: postgres #'{{ env_var(''POSTGRES_PASSWORD'') }}' port: 5432 #'{{ env_var(''POSTGRES_PORT'') | as_number }}' schema: postgres # '{{ env_var(''POSTGRES_SCHEMA'') }}' - user: "postgres" #'{{ env_var(''POSTGRES_USER'') }}' + user: postgres #'{{ env_var(''POSTGRES_USER'') }}' diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 76f489244..ce5899d72 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -77,7 +77,7 @@ set -e #kubectl describe pod postgres-postgresql-0 -kubectl create secret generic postgres-secrets --from-literal=host=0.0.0.0 --from-literal=password=postgres +kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=postgres kubectl apply -f scripts/test/postgres-deployment.yaml @@ -90,4 +90,4 @@ echo "$POD_NAME" kubectl port-forward --namespace default "$POD_NAME" 5432:5432 & -kubectl get pod \ No newline at end of file +kubectl get pod diff --git a/scripts/test/postgres-deployment.yaml b/scripts/test/postgres-deployment.yaml index 473439cff..8939d7f36 100644 --- a/scripts/test/postgres-deployment.yaml +++ b/scripts/test/postgres-deployment.yaml @@ -23,7 +23,7 @@ spec: valueFrom: secretKeyRef: name: postgres-secrets - key: host # Adjust according to your secret's key + key: host - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: @@ -39,7 +39,7 @@ metadata: name: postgres-secrets type: Opaque data: - host: MC4wLjAuMA== + host: cG9zdGdyZXMuZGVmYXVsdC5zdmMuY2x1c3Rlci5sb2NhbA== password: cG9zdGdyZXM= --- @@ -47,11 +47,13 @@ apiVersion: v1 kind: Service metadata: name: postgres - namespace: default # Ensure this matches the namespace of the Deployment + namespace: default spec: - type: LoadBalancer + type: NodePort ports: - port: 5432 targetPort: 5432 + protocol: TCP + nodePort: 32583 selector: app: postgres From 7a55a175e5c296f363f45d11d0678c99c6e6b562 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 01:17:24 +0530 Subject: [PATCH 34/64] cleanup --- .github/workflows/test.yml | 70 ++++++++++++++-- .github/workflows/test_kubernetes.yml | 23 ------ dev/Dockerfile.postgres_profile_docker_k8s | 2 +- dev/dags/dbt/jaffle_shop/profiles.yml | 12 +-- scripts/test/kubernetes-setup.sh | 93 ++++------------------ scripts/test/values.yaml | 18 ----- 6 files changed, 89 insertions(+), 129 deletions(-) delete mode 100644 .github/workflows/test_kubernetes.yml delete mode 100644 scripts/test/values.yaml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 219ea2019..9fde47aae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,9 +2,9 @@ name: test on: push: # Run on pushes to the default branch - branches: [main] + branches: [main, kube_mode_ci] pull_request_target: # Also run on pull requests originated from forks - branches: [main] + branches: [main, kube_mode_ci] concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -163,7 +163,6 @@ jobs: POSTGRES_DB: postgres POSTGRES_SCHEMA: public POSTGRES_PORT: 5432 - SOURCE_RENDERING_BEHAVIOR: all - name: Upload coverage to Github uses: actions/upload-artifact@v2 @@ -235,7 +234,6 @@ jobs: POSTGRES_DB: postgres POSTGRES_SCHEMA: public POSTGRES_PORT: 5432 - SOURCE_RENDERING_BEHAVIOR: all - name: Upload coverage to Github uses: actions/upload-artifact@v2 @@ -379,7 +377,6 @@ jobs: POSTGRES_DB: postgres POSTGRES_SCHEMA: public POSTGRES_PORT: 5432 - SOURCE_RENDERING_BEHAVIOR: all - name: Upload coverage to Github uses: actions/upload-artifact@v2 @@ -461,12 +458,75 @@ jobs: AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + Run-Kubernetes-Tests: + needs: Authorize + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.11" ] + airflow-version: [ "2.8" ] + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .local/share/hatch/ + key: coverage-integration-kubernetes-test-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Create KinD cluster + uses: container-tools/kind-action@v1 + + - name: Install packages and dependencies + run: | + python -m pip install uv + uv pip install --system hatch + hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + + - name: Run kubernetes tests agains + run: | + sh ./scripts/test/kubernetes-setup.sh + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-kubernetes + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} + AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }} + AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }} + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} + DATABRICKS_CLUSTER_ID: mock + DATABRICKS_HOST: mock + DATABRICKS_WAREHOUSE_ID: mock + DATABRICKS_TOKEN: mock + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + POSTGRES_SCHEMA: public + POSTGRES_PORT: 5432 + + - name: Upload coverage to Github + uses: actions/upload-artifact@v2 + with: + name: coverage-integration-kubernetes-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} + path: .coverage + Code-Coverage: if: github.event.action != 'labeled' needs: - Run-Unit-Tests - Run-Integration-Tests - Run-Integration-Tests-Expensive + - Run-Kubernetes-Tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml deleted file mode 100644 index 3987752b5..000000000 --- a/.github/workflows/test_kubernetes.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Kubernetes Integration Tests - - -on: - push: # Run on pushes to the default branch - branches: [kube_mode_ci] -# pull_request_target: # Also run on pull requests originated from forks -# branches: [kube_mode_ci] - - -jobs: - run-kubernets-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Kubernetes KinD Cluster - uses: container-tools/kind-action@v1 - - name: Run tests - run: | - sh ./scripts/test/kubernetes-setup.sh - pip install hatch - hatch -e tests.py3.9-2.9 run pip freeze - hatch run tests.py3.9-2.9:test-kubernetes diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s index 1634616c5..7375851c6 100644 --- a/dev/Dockerfile.postgres_profile_docker_k8s +++ b/dev/Dockerfile.postgres_profile_docker_k8s @@ -4,7 +4,7 @@ RUN pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz ENV POSTGRES_DATABASE=postgres ENV POSTGRES_HOST=postgres.default.svc.cluster.local -ENV POSTGRES_PASSWORD= +ENV POSTGRES_PASSWORD=postgres ENV POSTGRES_PORT=5432 ENV POSTGRES_SCHEMA=public ENV POSTGRES_USER=postgres diff --git a/dev/dags/dbt/jaffle_shop/profiles.yml b/dev/dags/dbt/jaffle_shop/profiles.yml index db1f34acd..43f0aae7d 100644 --- a/dev/dags/dbt/jaffle_shop/profiles.yml +++ b/dev/dags/dbt/jaffle_shop/profiles.yml @@ -16,9 +16,9 @@ postgres_profile: outputs: dev: type: postgres - dbname: postgres #'{{ env_var(''POSTGRES_DATABASE'') }}' - host: postgres.default.svc.cluster.local #'{{ env_var(''POSTGRES_HOST'') }}' - pass: postgres #'{{ env_var(''POSTGRES_PASSWORD'') }}' - port: 5432 #'{{ env_var(''POSTGRES_PORT'') | as_number }}' - schema: postgres # '{{ env_var(''POSTGRES_SCHEMA'') }}' - user: postgres #'{{ env_var(''POSTGRES_USER'') }}' + dbname: '{{ env_var(''POSTGRES_DATABASE'') }}' + host: '{{ env_var(''POSTGRES_HOST'') }}' + pass: '{{ env_var(''POSTGRES_PASSWORD'') }}' + port: '{{ env_var(''POSTGRES_PORT'') | as_number }}' + schema: '{{ env_var(''POSTGRES_SCHEMA'') }}' + user: '{{ env_var(''POSTGRES_USER'') }}' diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index ce5899d72..b0e37d0bb 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -1,93 +1,34 @@ #!/bin/bash +# Print each command before executing it +# Exit the script immediately if any command exits with a non-zero status (for debugging purposes) set -x set -e -#check_nodes_ready() { -# # Get the list of node statuses -# node_statuses=$(kubectl get nodes --no-headers | awk '{print $2}') -# # Check if all nodes are in the "Ready" state -# for status in $node_statuses; do -# if [ "$status" != "Ready" ]; then -# return 1 -# fi -# done -# return 0 -#} -# -#wait_for_nodes_ready() { -# local max_attempts=60 -# local interval=5 -# local attempt=0 -# -# echo "Waiting for nodes in the kind cluster to be in 'Ready' state..." -# -# while [ $attempt -lt $max_attempts ]; do -# if check_nodes_ready; then -# echo "All nodes in the kind cluster are in 'Ready' state." -# return 0 -# else -# echo "Nodes are not yet ready. Checking again in $interval seconds..." -# sleep $interval -# attempt=$((attempt + 1)) -# fi -# done -# -# echo "Timeout waiting for nodes in the kind cluster to be in 'Ready' state." -# return 1 -#} -# -#kubectl config set-context default -# -## Create a docker image containing the dbt project files and dbt profile -#cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . -## Make the build image available in the Kind K8s cluster -#kind load docker-image dbt-jaffle-shop:1.0.0 -# -## Deploy a Postgres pod to Kind -##helm repo add bitnami https://charts.bitnami.com/bitnami -##helm repo update -##helm install postgres bitnami/postgresql --set postgresqlExtendedConf.huge_pages="off" # -f scripts/test/values.yaml -# -## Retrieve the Postgres password and set it as an environment variable -##POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) -##export POSTGRES_PASSWORD -# -#kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD -# -#sleep 120 -## Expose the Postgres to the host running Docker/Kind -##kubectl port-forward --namespace default postgres-postgresql-0 5432:5432 & -##kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 & -##wait_for_nodes_ready -## -### Wait for the kind cluster to be in 'Ready' state -##wait_for_nodes_ready -# -## For Debugging -#echo "nodes" -#kubectl get nodes -#echo "helm" -#helm list -#echo "pod service" -#kubectl get pods --namespace default -#kubectl get svc --namespace default -#echo "pg log" -#kubectl logs postgres-postgresql-0 -c postgresql -#kubectl describe pod postgres-postgresql-0 - - -kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=postgres +# Create a Kubernetes secret named 'postgres-secrets' with the specified literals for host and password +kubectl create secret generic postgres-secrets \ + --from-literal=host=postgres-postgresql.default.svc.cluster.local \ + --from-literal=password=postgres +# Apply the PostgreSQL deployment configuration from the specified YAML file kubectl apply -f scripts/test/postgres-deployment.yaml +# Build the Docker image with tag 'dbt-jaffle-shop:1.0.0' using the specified Dockerfile cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . + +# Load the Docker image into the local KIND cluster kind load docker-image dbt-jaffle-shop:1.0.0 +# Retrieve the name of the PostgreSQL pod using the label selector 'app=postgres' +# The output is filtered to get the first pod's name POD_NAME=$(kubectl get pods -n default -l app=postgres -o jsonpath='{.items[0].metadata.name}') +# Print the name of the PostgreSQL pod echo "$POD_NAME" -kubectl port-forward --namespace default "$POD_NAME" 5432:5432 & +# Forward port 5432 from the PostgreSQL pod to the local machine's port 5432 +# This allows local access to the PostgreSQL instance running in the pod +kubectl port-forward --namespace default "$POD_NAME" 5432:5432 & +# List all pods in the default namespace to verify the status of pods kubectl get pod diff --git a/scripts/test/values.yaml b/scripts/test/values.yaml deleted file mode 100644 index 44fcfa83d..000000000 --- a/scripts/test/values.yaml +++ /dev/null @@ -1,18 +0,0 @@ -primary: - livenessProbe: - initialDelaySeconds: 120 - timeoutSeconds: 5 - periodSeconds: 10 - successThreshold: 1 - readinessProbe: - enabled: true - initialDelaySeconds: 120 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - startupProbe: - enabled: true - initialDelaySeconds: 120 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 From 0601a1dffab104f589bf89ed43577a4c9853e3ba Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 01:28:08 +0530 Subject: [PATCH 35/64] cleanup --- .github/workflows/postgres-deployment.yaml | 22 ---------------------- .github/workflows/postgres-service.yaml | 11 ----------- 2 files changed, 33 deletions(-) delete mode 100644 .github/workflows/postgres-deployment.yaml delete mode 100644 .github/workflows/postgres-service.yaml diff --git a/.github/workflows/postgres-deployment.yaml b/.github/workflows/postgres-deployment.yaml deleted file mode 100644 index 85def64af..000000000 --- a/.github/workflows/postgres-deployment.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgres -spec: - replicas: 1 - selector: - matchLabels: - app: postgres - template: - metadata: - labels: - app: postgres - spec: - containers: - - name: postgres - image: postgres:latest - env: - - name: POSTGRES_PASSWORD - value: postgres - ports: - - containerPort: 5432 diff --git a/.github/workflows/postgres-service.yaml b/.github/workflows/postgres-service.yaml deleted file mode 100644 index c12c0fed1..000000000 --- a/.github/workflows/postgres-service.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: postgres -spec: - selector: - app: postgres - ports: - - protocol: TCP - port: 5432 - targetPort: 5432   From d0300a4a3352c82fa0b93b54ed2a6ceccf535fb1 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 01:32:05 +0530 Subject: [PATCH 36/64] Fix tests --- .github/workflows/test.yml | 2 +- tests/dbt/test_graph.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9fde47aae..b106a1bac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -490,7 +490,7 @@ jobs: uv pip install --system hatch hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze - - name: Run kubernetes tests agains + - name: Run kubernetes tests run: | sh ./scripts/test/kubernetes-setup.sh hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-kubernetes diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 70e594761..bffee7b8d 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -1520,7 +1520,7 @@ def test_save_dbt_ls_cache(mock_variable_set, mock_datetime, tmp_dbt_project_dir if sys.platform == "darwin": assert hash_dir == "27970d3415be1f40a2ad43e46436c42d" else: - assert hash_dir == "9001bedf4aa8a329f7b669c89f337c24" + assert hash_dir == "93f820b07ddc095daeb56894735d4c28" @pytest.mark.integration From 4483f1cdb71a7a3c68ce7e275182edb2ca538ded Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 01:34:52 +0530 Subject: [PATCH 37/64] Fix pre-commit check --- .github/workflows/test.yml | 118 ++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b106a1bac..a5dfd9869 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -460,65 +460,65 @@ jobs: Run-Kubernetes-Tests: needs: Authorize - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ "3.11" ] - airflow-version: [ "2.8" ] - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} - - uses: actions/cache@v3 - with: - path: | - ~/.cache/pip - .local/share/hatch/ - key: coverage-integration-kubernetes-test-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Create KinD cluster - uses: container-tools/kind-action@v1 - - - name: Install packages and dependencies - run: | - python -m pip install uv - uv pip install --system hatch - hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze - - - name: Run kubernetes tests - run: | - sh ./scripts/test/kubernetes-setup.sh - hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-kubernetes - env: - AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ - AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres - AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} - AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }} - AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }} - AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 - PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH - COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} - DATABRICKS_CLUSTER_ID: mock - DATABRICKS_HOST: mock - DATABRICKS_WAREHOUSE_ID: mock - DATABRICKS_TOKEN: mock - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - POSTGRES_SCHEMA: public - POSTGRES_PORT: 5432 - - - name: Upload coverage to Github - uses: actions/upload-artifact@v2 - with: - name: coverage-integration-kubernetes-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} - path: .coverage + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.11" ] + airflow-version: [ "2.8" ] + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .local/share/hatch/ + key: coverage-integration-kubernetes-test-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Create KinD cluster + uses: container-tools/kind-action@v1 + + - name: Install packages and dependencies + run: | + python -m pip install uv + uv pip install --system hatch + hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + + - name: Run kubernetes tests + run: | + sh ./scripts/test/kubernetes-setup.sh + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-kubernetes + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} + AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }} + AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }} + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} + DATABRICKS_CLUSTER_ID: mock + DATABRICKS_HOST: mock + DATABRICKS_WAREHOUSE_ID: mock + DATABRICKS_TOKEN: mock + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + POSTGRES_SCHEMA: public + POSTGRES_PORT: 5432 + + - name: Upload coverage to Github + uses: actions/upload-artifact@v2 + with: + name: coverage-integration-kubernetes-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} + path: .coverage Code-Coverage: if: github.event.action != 'labeled' From 123591b3d6b1eda330debb74f4c13f81ce8917d3 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 02:14:36 +0530 Subject: [PATCH 38/64] Fix pre-commit check --- dev/Dockerfile.postgres_profile_docker_k8s | 1 + dev/dags/dbt/jaffle_shop/profiles.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s index 7375851c6..b803b792d 100644 --- a/dev/Dockerfile.postgres_profile_docker_k8s +++ b/dev/Dockerfile.postgres_profile_docker_k8s @@ -3,6 +3,7 @@ FROM python:3.9 RUN pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz ENV POSTGRES_DATABASE=postgres +ENV POSTGRES_DB=postgres ENV POSTGRES_HOST=postgres.default.svc.cluster.local ENV POSTGRES_PASSWORD=postgres ENV POSTGRES_PORT=5432 diff --git a/dev/dags/dbt/jaffle_shop/profiles.yml b/dev/dags/dbt/jaffle_shop/profiles.yml index 43f0aae7d..3960ca84d 100644 --- a/dev/dags/dbt/jaffle_shop/profiles.yml +++ b/dev/dags/dbt/jaffle_shop/profiles.yml @@ -16,9 +16,9 @@ postgres_profile: outputs: dev: type: postgres - dbname: '{{ env_var(''POSTGRES_DATABASE'') }}' - host: '{{ env_var(''POSTGRES_HOST'') }}' - pass: '{{ env_var(''POSTGRES_PASSWORD'') }}' - port: '{{ env_var(''POSTGRES_PORT'') | as_number }}' - schema: '{{ env_var(''POSTGRES_SCHEMA'') }}' - user: '{{ env_var(''POSTGRES_USER'') }}' + dbname: "{{ env_var('POSTGRES_DATABASE') }}" + host: "{{ env_var('POSTGRES_HOST') }}" + pass: "{{ env_var('POSTGRES_PASSWORD') }}" + port: 5432 # "{{ env_var('POSTGRES_PORT') | as_number }}" + schema: "{{ env_var('POSTGRES_SCHEMA') }}" + user: "{{ env_var('POSTGRES_USER') }}" From 5e42afb3ae35f5aa5650a942900217a65ab3241b Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 02:16:13 +0530 Subject: [PATCH 39/64] More env and string issue fix --- dev/dags/jaffle_shop_kubernetes.py | 29 ++++++++++++++++++++++++++ scripts/test/integration-kubernetes.sh | 4 ++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/dev/dags/jaffle_shop_kubernetes.py b/dev/dags/jaffle_shop_kubernetes.py index a50a8e454..4352c79bf 100644 --- a/dev/dags/jaffle_shop_kubernetes.py +++ b/dev/dags/jaffle_shop_kubernetes.py @@ -16,7 +16,11 @@ from cosmos import ( DbtSeedKubernetesOperator, + DbtTaskGroup, + ExecutionConfig, + ExecutionMode, ProfileConfig, + ProjectConfig, ) from cosmos.profiles import PostgresUserPasswordProfileMapping @@ -64,3 +68,28 @@ ), ), ) + + run_models = DbtTaskGroup( + profile_config=ProfileConfig( + profile_name="postgres_profile", + target_name="dev", + profile_mapping=PostgresUserPasswordProfileMapping( + conn_id="postgres_default", + profile_args={ + "schema": "public", + }, + ), + ), + project_config=ProjectConfig("dags/dbt/jaffle_shop"), + execution_config=ExecutionConfig( + execution_mode=ExecutionMode.KUBERNETES, + ), + operator_args={ + "image": DBT_IMAGE, + "get_logs": True, + "is_delete_operator_pod": False, + "secrets": [postgres_password_secret, postgres_host_secret], + }, + ) + + load_seeds >> run_models diff --git a/scripts/test/integration-kubernetes.sh b/scripts/test/integration-kubernetes.sh index de7b46675..b7666e639 100644 --- a/scripts/test/integration-kubernetes.sh +++ b/scripts/test/integration-kubernetes.sh @@ -3,8 +3,8 @@ set -x set -e -# export POSTGRES_DB=postgres - +# Reset the Airflow database to its initial state airflow db reset -y +# Run tests using pytest pytest tests/test_example_k8s_dags.py From b75d29a5190a482059d2603a71a484ceb86a7562 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 02:36:06 +0530 Subject: [PATCH 40/64] Disable some task and add docs --- dev/dags/jaffle_shop_kubernetes.py | 50 ++++++++++++------------ docs/getting_started/execution-modes.rst | 24 ++---------- 2 files changed, 30 insertions(+), 44 deletions(-) diff --git a/dev/dags/jaffle_shop_kubernetes.py b/dev/dags/jaffle_shop_kubernetes.py index 4352c79bf..981fb7372 100644 --- a/dev/dags/jaffle_shop_kubernetes.py +++ b/dev/dags/jaffle_shop_kubernetes.py @@ -24,9 +24,9 @@ ) from cosmos.profiles import PostgresUserPasswordProfileMapping +# [START kubernetes_example] DBT_IMAGE = "dbt-jaffle-shop:1.0.0" - project_seeds = [{"project": "jaffle_shop", "seeds": ["raw_customers", "raw_payments", "raw_orders"]}] postgres_password_secret = Secret( @@ -69,27 +69,29 @@ ), ) - run_models = DbtTaskGroup( - profile_config=ProfileConfig( - profile_name="postgres_profile", - target_name="dev", - profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="postgres_default", - profile_args={ - "schema": "public", - }, - ), - ), - project_config=ProjectConfig("dags/dbt/jaffle_shop"), - execution_config=ExecutionConfig( - execution_mode=ExecutionMode.KUBERNETES, - ), - operator_args={ - "image": DBT_IMAGE, - "get_logs": True, - "is_delete_operator_pod": False, - "secrets": [postgres_password_secret, postgres_host_secret], - }, - ) + # TODO: Enable it + # run_models = DbtTaskGroup( + # profile_config=ProfileConfig( + # profile_name="postgres_profile", + # target_name="dev", + # profile_mapping=PostgresUserPasswordProfileMapping( + # conn_id="postgres_default", + # profile_args={ + # "schema": "public", + # }, + # ), + # ), + # project_config=ProjectConfig("dags/dbt/jaffle_shop"), + # execution_config=ExecutionConfig( + # execution_mode=ExecutionMode.KUBERNETES, + # ), + # operator_args={ + # "image": DBT_IMAGE, + # "get_logs": True, + # "is_delete_operator_pod": False, + # "secrets": [postgres_password_secret, postgres_host_secret], + # }, + # ) - load_seeds >> run_models + # load_seeds >> run_models + # [END kubernetes_example] diff --git a/docs/getting_started/execution-modes.rst b/docs/getting_started/execution-modes.rst index 266b40f32..02267f17d 100644 --- a/docs/getting_started/execution-modes.rst +++ b/docs/getting_started/execution-modes.rst @@ -144,27 +144,11 @@ Check the step-by-step guide on using the ``kubernetes`` execution mode at :ref: Example DAG: -.. code-block:: python - - postgres_password_secret = Secret( - deploy_type="env", - deploy_target="POSTGRES_PASSWORD", - secret="postgres-secrets", - key="password", - ) +.. literalinclude:: ../../dev/dags/jaffle_shop_kubernetes.py + :language: python + :start-after: [START kubernetes_example] + :end-before: [END kubernetes_example] - docker_cosmos_dag = DbtDag( - # ... - execution_config=ExecutionConfig( - execution_mode=ExecutionMode.KUBERNETES, - ), - operator_args={ - "image": "dbt-jaffle-shop:1.0.0", - "get_logs": True, - "is_delete_operator_pod": False, - "secrets": [postgres_password_secret], - }, - ) AWS_EKS ---------- From a71a9f9ae2c511efad8ca53640a51894e5f1d876 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 03:06:19 +0530 Subject: [PATCH 41/64] Update docs --- dev/dags/jaffle_shop_kubernetes.py | 55 +++++++++++++----------- docs/getting_started/execution-modes.rst | 4 +- docs/getting_started/kubernetes.rst | 28 ++---------- 3 files changed, 35 insertions(+), 52 deletions(-) diff --git a/dev/dags/jaffle_shop_kubernetes.py b/dev/dags/jaffle_shop_kubernetes.py index 981fb7372..ab6f57420 100644 --- a/dev/dags/jaffle_shop_kubernetes.py +++ b/dev/dags/jaffle_shop_kubernetes.py @@ -9,6 +9,7 @@ https://astronomer.github.io/astronomer-cosmos/getting_started/kubernetes.html#kubernetes """ +from pathlib import Path from airflow import DAG from airflow.providers.cncf.kubernetes.secret import Secret @@ -24,7 +25,7 @@ ) from cosmos.profiles import PostgresUserPasswordProfileMapping -# [START kubernetes_example] + DBT_IMAGE = "dbt-jaffle-shop:1.0.0" project_seeds = [{"project": "jaffle_shop", "seeds": ["raw_customers", "raw_payments", "raw_orders"]}] @@ -49,6 +50,7 @@ doc_md=__doc__, catchup=False, ) as dag: + # [START kubernetes_seed_example] load_seeds = DbtSeedKubernetesOperator( task_id="load_seeds", project_dir="dags/dbt/jaffle_shop", @@ -68,30 +70,31 @@ ), ), ) + # [END kubernetes_seed_example] - # TODO: Enable it - # run_models = DbtTaskGroup( - # profile_config=ProfileConfig( - # profile_name="postgres_profile", - # target_name="dev", - # profile_mapping=PostgresUserPasswordProfileMapping( - # conn_id="postgres_default", - # profile_args={ - # "schema": "public", - # }, - # ), - # ), - # project_config=ProjectConfig("dags/dbt/jaffle_shop"), - # execution_config=ExecutionConfig( - # execution_mode=ExecutionMode.KUBERNETES, - # ), - # operator_args={ - # "image": DBT_IMAGE, - # "get_logs": True, - # "is_delete_operator_pod": False, - # "secrets": [postgres_password_secret, postgres_host_secret], - # }, - # ) + # [START kubernetes_tg_example] + run_models = DbtTaskGroup( + profile_config=ProfileConfig( + profile_name="postgres_profile", + target_name="dev", + profile_mapping=PostgresUserPasswordProfileMapping( + conn_id="postgres_default", + profile_args={ + "schema": "public", + }, + ), + ), + project_config=ProjectConfig(dbt_project_path="dags/dbt/jaffle_shop"), + execution_config=ExecutionConfig( + execution_mode=ExecutionMode.KUBERNETES, + ), + operator_args={ + "image": DBT_IMAGE, + "get_logs": True, + "is_delete_operator_pod": False, + "secrets": [postgres_password_secret, postgres_host_secret], + }, + ) + # [END kubernetes_tg_example] - # load_seeds >> run_models - # [END kubernetes_example] + load_seeds >> run_models diff --git a/docs/getting_started/execution-modes.rst b/docs/getting_started/execution-modes.rst index 02267f17d..df9b0f26d 100644 --- a/docs/getting_started/execution-modes.rst +++ b/docs/getting_started/execution-modes.rst @@ -146,8 +146,8 @@ Example DAG: .. literalinclude:: ../../dev/dags/jaffle_shop_kubernetes.py :language: python - :start-after: [START kubernetes_example] - :end-before: [END kubernetes_example] + :start-after: [START kubernetes_seed_example] + :end-before: [END kubernetes_seed_example] AWS_EKS ---------- diff --git a/docs/getting_started/kubernetes.rst b/docs/getting_started/kubernetes.rst index 1ae918a9a..bfa02023d 100644 --- a/docs/getting_started/kubernetes.rst +++ b/docs/getting_started/kubernetes.rst @@ -28,30 +28,10 @@ Additional KubernetesPodOperator parameters can be added on the operator_args pa For instance, -.. code-block:: python - - run_models = DbtTaskGroup( - profile_config=ProfileConfig( - profile_name="postgres_profile", - target_name="dev", - profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="postgres_default", - profile_args={ - "schema": "public", - }, - ), - ), - project_config=ProjectConfig(PROJECT_DIR), - execution_config=ExecutionConfig( - execution_mode=ExecutionMode.KUBERNETES, - ), - operator_args={ - "image": DBT_IMAGE, - "get_logs": True, - "is_delete_operator_pod": False, - "secrets": [postgres_password_secret, postgres_host_secret], - }, - ) +.. literalinclude:: ../../dev/dags/jaffle_shop_kubernetes.py + :language: python + :start-after: [START kubernetes_tg_example] + :end-before: [END kubernetes_tg_example] Step-by-step instructions +++++++++++++++++++++++++ From 1238cc706f2ffc70a479392979edea2ef48b055c Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 03:07:00 +0530 Subject: [PATCH 42/64] Update docs --- dev/dags/jaffle_shop_kubernetes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev/dags/jaffle_shop_kubernetes.py b/dev/dags/jaffle_shop_kubernetes.py index ab6f57420..aaca09ad9 100644 --- a/dev/dags/jaffle_shop_kubernetes.py +++ b/dev/dags/jaffle_shop_kubernetes.py @@ -9,7 +9,6 @@ https://astronomer.github.io/astronomer-cosmos/getting_started/kubernetes.html#kubernetes """ -from pathlib import Path from airflow import DAG from airflow.providers.cncf.kubernetes.secret import Secret @@ -25,7 +24,6 @@ ) from cosmos.profiles import PostgresUserPasswordProfileMapping - DBT_IMAGE = "dbt-jaffle-shop:1.0.0" project_seeds = [{"project": "jaffle_shop", "seeds": ["raw_customers", "raw_payments", "raw_orders"]}] From 36f004460957fa85f823a58ea5554f39771751f3 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 12:35:32 +0530 Subject: [PATCH 43/64] try different project path --- dev/dags/jaffle_shop_kubernetes.py | 7 +++++-- scripts/test/integration-kubernetes.sh | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dev/dags/jaffle_shop_kubernetes.py b/dev/dags/jaffle_shop_kubernetes.py index aaca09ad9..d8ee83883 100644 --- a/dev/dags/jaffle_shop_kubernetes.py +++ b/dev/dags/jaffle_shop_kubernetes.py @@ -9,6 +9,7 @@ https://astronomer.github.io/astronomer-cosmos/getting_started/kubernetes.html#kubernetes """ +from pathlib import Path from airflow import DAG from airflow.providers.cncf.kubernetes.secret import Secret @@ -26,6 +27,8 @@ DBT_IMAGE = "dbt-jaffle-shop:1.0.0" +PROJECT_PATH = Path("/home/runner/work/astronomer-cosmos/astronomer-cosmos/dev/dags/dbt/jaffle_shop") + project_seeds = [{"project": "jaffle_shop", "seeds": ["raw_customers", "raw_payments", "raw_orders"]}] postgres_password_secret = Secret( @@ -51,7 +54,7 @@ # [START kubernetes_seed_example] load_seeds = DbtSeedKubernetesOperator( task_id="load_seeds", - project_dir="dags/dbt/jaffle_shop", + project_dir=PROJECT_PATH, get_logs=True, schema="public", image=DBT_IMAGE, @@ -82,7 +85,7 @@ }, ), ), - project_config=ProjectConfig(dbt_project_path="dags/dbt/jaffle_shop"), + project_config=ProjectConfig(dbt_project_path=PROJECT_PATH), execution_config=ExecutionConfig( execution_mode=ExecutionMode.KUBERNETES, ), diff --git a/scripts/test/integration-kubernetes.sh b/scripts/test/integration-kubernetes.sh index b7666e639..402da1437 100644 --- a/scripts/test/integration-kubernetes.sh +++ b/scripts/test/integration-kubernetes.sh @@ -7,4 +7,10 @@ set -e airflow db reset -y # Run tests using pytest -pytest tests/test_example_k8s_dags.py +pytest -vv \ + --cov=cosmos \ + --cov-report=term-missing \ + --cov-report=xml \ + --durations=0 \ + -m integration \ + tests/test_example_k8s_dags.py From 0875d2d33f93f084caf0dfd5202a3791dda5ba3c Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 13:42:39 +0530 Subject: [PATCH 44/64] Add executable path --- dev/dags/jaffle_shop_kubernetes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dev/dags/jaffle_shop_kubernetes.py b/dev/dags/jaffle_shop_kubernetes.py index d8ee83883..2f581bdb0 100644 --- a/dev/dags/jaffle_shop_kubernetes.py +++ b/dev/dags/jaffle_shop_kubernetes.py @@ -9,6 +9,7 @@ https://astronomer.github.io/astronomer-cosmos/getting_started/kubernetes.html#kubernetes """ + from pathlib import Path from airflow import DAG @@ -28,7 +29,7 @@ DBT_IMAGE = "dbt-jaffle-shop:1.0.0" PROJECT_PATH = Path("/home/runner/work/astronomer-cosmos/astronomer-cosmos/dev/dags/dbt/jaffle_shop") - +# PROJECT_PATH = Path("/Users/pankaj/Documents/astro_code/astronomer-cosmos/dev/dags/dbt/jaffle_shop") project_seeds = [{"project": "jaffle_shop", "seeds": ["raw_customers", "raw_payments", "raw_orders"]}] postgres_password_secret = Secret( @@ -54,7 +55,7 @@ # [START kubernetes_seed_example] load_seeds = DbtSeedKubernetesOperator( task_id="load_seeds", - project_dir=PROJECT_PATH, + project_dir="dags/dbt/jaffle_shop", get_logs=True, schema="public", image=DBT_IMAGE, @@ -88,6 +89,7 @@ project_config=ProjectConfig(dbt_project_path=PROJECT_PATH), execution_config=ExecutionConfig( execution_mode=ExecutionMode.KUBERNETES, + dbt_executable_path="/usr/local/bin/dbt", ), operator_args={ "image": DBT_IMAGE, From 407045849ddae648184bd07e3cabf0813bb55f3d Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 13:59:38 +0530 Subject: [PATCH 45/64] Add debug stmt --- scripts/test/kubernetes-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index b0e37d0bb..6ec8fc690 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -32,3 +32,4 @@ kubectl port-forward --namespace default "$POD_NAME" 5432:5432 & # List all pods in the default namespace to verify the status of pods kubectl get pod +docker run --rm -it dbt-jaffle-shop:1.0.0 /bin/sh -c "which dbt" \ No newline at end of file From c1b140305a0afc0a45fb53084b98cc2b27c9ada9 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 14:35:37 +0530 Subject: [PATCH 46/64] add more debgu stmpt --- dev/Dockerfile.postgres_profile_docker_k8s | 2 ++ scripts/test/kubernetes-setup.sh | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s index b803b792d..133051842 100644 --- a/dev/Dockerfile.postgres_profile_docker_k8s +++ b/dev/Dockerfile.postgres_profile_docker_k8s @@ -1,6 +1,8 @@ FROM python:3.9 RUN pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz +RUN bash -c 'echo "Path to dbt: $(which dbt)"' +RUN pip show dbt-postgres ENV POSTGRES_DATABASE=postgres ENV POSTGRES_DB=postgres diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 6ec8fc690..2abe8ef9e 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -14,7 +14,7 @@ kubectl create secret generic postgres-secrets \ kubectl apply -f scripts/test/postgres-deployment.yaml # Build the Docker image with tag 'dbt-jaffle-shop:1.0.0' using the specified Dockerfile -cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . +cd dev && docker build --progress=plain --no-cache -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . # Load the Docker image into the local KIND cluster kind load docker-image dbt-jaffle-shop:1.0.0 @@ -32,4 +32,5 @@ kubectl port-forward --namespace default "$POD_NAME" 5432:5432 & # List all pods in the default namespace to verify the status of pods kubectl get pod -docker run --rm -it dbt-jaffle-shop:1.0.0 /bin/sh -c "which dbt" \ No newline at end of file +bash -c 'echo "Path to dbt: $(which dbt)"' +pip show dbt-postgres \ No newline at end of file From 69846521d92b5eaec5e17f5c212d2f044c5c1790 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 14:35:58 +0530 Subject: [PATCH 47/64] add more debgu stmpt --- scripts/test/kubernetes-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 2abe8ef9e..3f5914191 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -33,4 +33,4 @@ kubectl port-forward --namespace default "$POD_NAME" 5432:5432 & # List all pods in the default namespace to verify the status of pods kubectl get pod bash -c 'echo "Path to dbt: $(which dbt)"' -pip show dbt-postgres \ No newline at end of file +pip show dbt-postgres From 9fc125979bcdb2952d5effebe8cd76f7b545951e Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 16:26:03 +0530 Subject: [PATCH 48/64] Run tests from dev dir --- .github/workflows/test.yml | 4 +++- dev/Dockerfile.postgres_profile_docker_k8s | 2 -- dev/dags/jaffle_shop_kubernetes.py | 7 +------ scripts/test/integration-kubernetes.sh | 2 +- scripts/test/kubernetes-setup.sh | 1 - 5 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a5dfd9869..8a1471f91 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -493,7 +493,9 @@ jobs: - name: Run kubernetes tests run: | sh ./scripts/test/kubernetes-setup.sh - hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-kubernetes + cd dev && ../scripts/test/kubernetes-setup.sh + # TODO: Fix hatch taget it failing with incorrect dbt project path error + # hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-kubernetes env: AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s index 133051842..b803b792d 100644 --- a/dev/Dockerfile.postgres_profile_docker_k8s +++ b/dev/Dockerfile.postgres_profile_docker_k8s @@ -1,8 +1,6 @@ FROM python:3.9 RUN pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz -RUN bash -c 'echo "Path to dbt: $(which dbt)"' -RUN pip show dbt-postgres ENV POSTGRES_DATABASE=postgres ENV POSTGRES_DB=postgres diff --git a/dev/dags/jaffle_shop_kubernetes.py b/dev/dags/jaffle_shop_kubernetes.py index 2f581bdb0..aaca09ad9 100644 --- a/dev/dags/jaffle_shop_kubernetes.py +++ b/dev/dags/jaffle_shop_kubernetes.py @@ -10,8 +10,6 @@ """ -from pathlib import Path - from airflow import DAG from airflow.providers.cncf.kubernetes.secret import Secret from pendulum import datetime @@ -28,8 +26,6 @@ DBT_IMAGE = "dbt-jaffle-shop:1.0.0" -PROJECT_PATH = Path("/home/runner/work/astronomer-cosmos/astronomer-cosmos/dev/dags/dbt/jaffle_shop") -# PROJECT_PATH = Path("/Users/pankaj/Documents/astro_code/astronomer-cosmos/dev/dags/dbt/jaffle_shop") project_seeds = [{"project": "jaffle_shop", "seeds": ["raw_customers", "raw_payments", "raw_orders"]}] postgres_password_secret = Secret( @@ -86,10 +82,9 @@ }, ), ), - project_config=ProjectConfig(dbt_project_path=PROJECT_PATH), + project_config=ProjectConfig(dbt_project_path="dags/dbt/jaffle_shop"), execution_config=ExecutionConfig( execution_mode=ExecutionMode.KUBERNETES, - dbt_executable_path="/usr/local/bin/dbt", ), operator_args={ "image": DBT_IMAGE, diff --git a/scripts/test/integration-kubernetes.sh b/scripts/test/integration-kubernetes.sh index 402da1437..1cc305c62 100644 --- a/scripts/test/integration-kubernetes.sh +++ b/scripts/test/integration-kubernetes.sh @@ -13,4 +13,4 @@ pytest -vv \ --cov-report=xml \ --durations=0 \ -m integration \ - tests/test_example_k8s_dags.py + ../tests/test_example_k8s_dags.py diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 3f5914191..706e37b2e 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -33,4 +33,3 @@ kubectl port-forward --namespace default "$POD_NAME" 5432:5432 & # List all pods in the default namespace to verify the status of pods kubectl get pod bash -c 'echo "Path to dbt: $(which dbt)"' -pip show dbt-postgres From d16b67d2c1b53ef95248a7fdcb9ba37269a7d20c Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 16:26:57 +0530 Subject: [PATCH 49/64] Fix pre-commit --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8a1471f91..d5dada73f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -494,7 +494,7 @@ jobs: run: | sh ./scripts/test/kubernetes-setup.sh cd dev && ../scripts/test/kubernetes-setup.sh - # TODO: Fix hatch taget it failing with incorrect dbt project path error + # TODO: Fix hatch target it failing with incorrect dbt project path error # hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-kubernetes env: AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ From 54991f8942888e9212ebcb3dd0aa60c3e9785f83 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 16:30:17 +0530 Subject: [PATCH 50/64] Update hash --- tests/dbt/test_graph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index bffee7b8d..d1891169d 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -1518,9 +1518,9 @@ def test_save_dbt_ls_cache(mock_variable_set, mock_datetime, tmp_dbt_project_dir hash_dir, hash_args = version.split(",") assert hash_args == "d41d8cd98f00b204e9800998ecf8427e" if sys.platform == "darwin": - assert hash_dir == "27970d3415be1f40a2ad43e46436c42d" + assert hash_dir == "b556a0a268e28868971d14c98548c349" else: - assert hash_dir == "93f820b07ddc095daeb56894735d4c28" + assert hash_dir == "6f63493009733a7be34364a6ea3ffd3c" @pytest.mark.integration From 035d7dbf3fc7089e99c5fda2705cd00f61537ad1 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 16:52:24 +0530 Subject: [PATCH 51/64] Fix command --- .github/workflows/test.yml | 2 +- scripts/test/kubernetes-setup.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d5dada73f..2aba9f8fd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -493,7 +493,7 @@ jobs: - name: Run kubernetes tests run: | sh ./scripts/test/kubernetes-setup.sh - cd dev && ../scripts/test/kubernetes-setup.sh + cd dev && sh ../scripts/test/integration-kubernetes.sh # TODO: Fix hatch target it failing with incorrect dbt project path error # hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-kubernetes env: diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 706e37b2e..d40aaa187 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -32,4 +32,3 @@ kubectl port-forward --namespace default "$POD_NAME" 5432:5432 & # List all pods in the default namespace to verify the status of pods kubectl get pod -bash -c 'echo "Path to dbt: $(which dbt)"' From 19773af255a4a34a1c47b02ba614903c18cf643b Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 16:59:03 +0530 Subject: [PATCH 52/64] Ignore tests --- tests/test_example_dags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_example_dags.py b/tests/test_example_dags.py index 50e294bd6..2adf9b30d 100644 --- a/tests/test_example_dags.py +++ b/tests/test_example_dags.py @@ -31,7 +31,7 @@ "2.8": ["cosmos_manifest_example.py"], } -IGNORED_DAG_FILES = ["performance_dag.py"] +IGNORED_DAG_FILES = ["performance_dag.py", "test_example_k8s_dags.py"] # Sort descending based on Versions and convert string to an actual version From 338da1fe222d5f6741cfc9d7ac49ec356647f5ac Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 17:05:53 +0530 Subject: [PATCH 53/64] Ignore test --- tests/test_example_dags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_example_dags.py b/tests/test_example_dags.py index 2adf9b30d..9f8601156 100644 --- a/tests/test_example_dags.py +++ b/tests/test_example_dags.py @@ -31,7 +31,7 @@ "2.8": ["cosmos_manifest_example.py"], } -IGNORED_DAG_FILES = ["performance_dag.py", "test_example_k8s_dags.py"] +IGNORED_DAG_FILES = ["performance_dag.py", "jaffle_shop_kubernetes.py"] # Sort descending based on Versions and convert string to an actual version From 0a032c9b14d1c8442b130726ddda881b2bb136dd Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 17:57:39 +0530 Subject: [PATCH 54/64] try virtual env --- .github/workflows/test.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2aba9f8fd..108eaf6df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -486,15 +486,18 @@ jobs: - name: Install packages and dependencies run: | - python -m pip install uv - uv pip install --system hatch - hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + python -m venv venv + source venv/bin/activate + pip install --upgrade pip + pip install -e . + pip install apache-airflow==${{ matrix.airflow-version }} + # hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze - name: Run kubernetes tests run: | + source venv/bin/activate sh ./scripts/test/kubernetes-setup.sh cd dev && sh ../scripts/test/integration-kubernetes.sh - # TODO: Fix hatch target it failing with incorrect dbt project path error # hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-kubernetes env: AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ From 640185759087996eff0d83ce4f22c73c1ab957ba Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 18:03:55 +0530 Subject: [PATCH 55/64] Update airflow version --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 108eaf6df..dffa5903f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -464,7 +464,7 @@ jobs: strategy: matrix: python-version: [ "3.11" ] - airflow-version: [ "2.8" ] + airflow-version: [ "2.9" ] steps: - uses: actions/checkout@v3 with: From 5927f3f3d696a52e8b6fcf06ca48828b6cee9284 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 18:14:09 +0530 Subject: [PATCH 56/64] install tests --- .github/workflows/test.yml | 2 +- dev/Dockerfile.postgres_profile_docker_k8s | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dffa5903f..728580c83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -489,7 +489,7 @@ jobs: python -m venv venv source venv/bin/activate pip install --upgrade pip - pip install -e . + pip install -e ".[tests]" pip install apache-airflow==${{ matrix.airflow-version }} # hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s index b803b792d..ae81a5b9f 100644 --- a/dev/Dockerfile.postgres_profile_docker_k8s +++ b/dev/Dockerfile.postgres_profile_docker_k8s @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.11 RUN pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz From 96f1510d08b3b5cd19a4535416a13b2aa2168cd3 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 18:20:56 +0530 Subject: [PATCH 57/64] install apache-airflow-providers-cncf-kubernetes --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 728580c83..3442bd6ab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -490,6 +490,7 @@ jobs: source venv/bin/activate pip install --upgrade pip pip install -e ".[tests]" + pip install apache-airflow-providers-cncf-kubernetes pip install apache-airflow==${{ matrix.airflow-version }} # hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze From 048acc7a54508766f45e4f5df6bbe0977feae28e Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 18:25:40 +0530 Subject: [PATCH 58/64] Install dbt --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3442bd6ab..1a249f6fb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -491,6 +491,7 @@ jobs: pip install --upgrade pip pip install -e ".[tests]" pip install apache-airflow-providers-cncf-kubernetes + pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz pip install apache-airflow==${{ matrix.airflow-version }} # hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze From 151e9dc383260956022732d2120d1680dc0886e7 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 19:34:16 +0530 Subject: [PATCH 59/64] Remove ignore --- scripts/test/integration.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/test/integration.sh b/scripts/test/integration.sh index 3a8a41511..a39ef63b1 100644 --- a/scripts/test/integration.sh +++ b/scripts/test/integration.sh @@ -20,5 +20,4 @@ pytest -vv \ --durations=0 \ -m integration \ --ignore=tests/perf \ - --ignore=tests/test_example_k8s_dags.py \ -k 'not (sqlite or example_cosmos_sources or example_cosmos_python_models or example_virtualenv)' From b5d43aa15800c36569d1b284a2299b1a9276818a Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 19:45:10 +0530 Subject: [PATCH 60/64] Extend ignore --- scripts/test/integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/integration.sh b/scripts/test/integration.sh index a39ef63b1..20e8de728 100644 --- a/scripts/test/integration.sh +++ b/scripts/test/integration.sh @@ -20,4 +20,4 @@ pytest -vv \ --durations=0 \ -m integration \ --ignore=tests/perf \ - -k 'not (sqlite or example_cosmos_sources or example_cosmos_python_models or example_virtualenv)' + -k 'not (sqlite or example_cosmos_sources or example_cosmos_python_models or example_virtualenv or jaffle_shop_kubernetes)' From 7defc5d897218dd9ebab50082e5d4deb4bce05f6 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 19:53:21 +0530 Subject: [PATCH 61/64] Ignore More --- tests/test_example_dags_no_connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_example_dags_no_connections.py b/tests/test_example_dags_no_connections.py index d075a389d..0cc560ecc 100644 --- a/tests/test_example_dags_no_connections.py +++ b/tests/test_example_dags_no_connections.py @@ -22,7 +22,7 @@ "2.8": ["cosmos_manifest_example.py"], } -IGNORED_DAG_FILES = ["performance_dag.py"] +IGNORED_DAG_FILES = ["performance_dag.py", "jaffle_shop_kubernetes.py"] # Sort descending based on Versions and convert string to an actual version MIN_VER_DAG_FILE_VER: dict[Version, list[str]] = { From 82b8345daba89725676780584184160ebd7947c5 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Thu, 15 Aug 2024 20:09:42 +0530 Subject: [PATCH 62/64] Ignore more :( --- scripts/test/integration.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test/integration.sh b/scripts/test/integration.sh index 20e8de728..db11bff6c 100644 --- a/scripts/test/integration.sh +++ b/scripts/test/integration.sh @@ -20,4 +20,5 @@ pytest -vv \ --durations=0 \ -m integration \ --ignore=tests/perf \ + --ignore=tests/test_example_k8s_dags.py \ -k 'not (sqlite or example_cosmos_sources or example_cosmos_python_models or example_virtualenv or jaffle_shop_kubernetes)' From 0ad6dcf568c33c48b2ae2898f9dae2d049eb2e4e Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Thu, 15 Aug 2024 22:33:54 +0530 Subject: [PATCH 63/64] Update .github/workflows/test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a249f6fb..558eb755f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ on: push: # Run on pushes to the default branch branches: [main, kube_mode_ci] pull_request_target: # Also run on pull requests originated from forks - branches: [main, kube_mode_ci] + branches: [main] concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} From 05cedca9ec12e8c5799477819ea7fbc825aa46c2 Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Thu, 15 Aug 2024 22:34:00 +0530 Subject: [PATCH 64/64] Update .github/workflows/test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 558eb755f..a5736c295 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: push: # Run on pushes to the default branch - branches: [main, kube_mode_ci] + branches: [main] pull_request_target: # Also run on pull requests originated from forks branches: [main]