From c8629316896cc4ed7b59970d32bd255c3ec2b2dd Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Tue, 7 May 2024 14:48:55 +0100 Subject: [PATCH] Improve logs to troubleshoot issue in 1.4.0a2 with astro-cli MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [2024-05-07, 14:20:09 UTC] {taskinstance.py:441} ▼ Post task execution logs [2024-05-07, 14:20:09 UTC] {taskinstance.py:2905} ERROR - Task failed with exception Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/airflow/models/taskinstance.py", line 465, in _execute_task result = _execute_callable(context=context, **execute_callable_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/airflow/models/taskinstance.py", line 432, in _execute_callable return execute_callable(context=context, **execute_callable_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/airflow/models/baseoperator.py", line 400, in wrapper return func(self, *args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/cosmos/operators/base.py", line 266, in execute self.build_and_run_cmd(context=context, cmd_flags=self.add_cmd_flags()) File "/usr/local/lib/python3.11/site-packages/cosmos/operators/local.py", line 470, in build_and_run_cmd result = self.run_command(cmd=dbt_cmd, env=env, context=context) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/cosmos/operators/local.py", line 343, in run_command outlets = self.get_datasets("outputs") ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/cosmos/operators/local.py", line 410, in get_datasets return [Dataset(uri) for uri in uris] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/cosmos/operators/local.py", line 410, in return [Dataset(uri) for uri in uris] ^^^^^^^^^^^^ File "", line 3, in __init__ _setattr('uri', __attr_converter_uri(uri)) ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/airflow/datasets/__init__.py", line 81, in _sanitize_uri parsed = normalizer(parsed) ^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/airflow/providers/postgres/datasets/postgres.py", line 34, in sanitize_uri raise ValueError("URI format postgres:// must contain database, schema, and table names") This improvement allowed us to confirm how were the Dataset URIs Cosmos was attempting to generate: https://github.com/astronomer/astronomer-cosmos/issues/945 (cherry picked from commit c7a4599ee68b51b4326a2541c80b49c1e2f07a90) --- cosmos/operators/local.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cosmos/operators/local.py b/cosmos/operators/local.py index 0f7fa7f5f..efc022bf6 100644 --- a/cosmos/operators/local.py +++ b/cosmos/operators/local.py @@ -406,6 +406,7 @@ def get_datasets(self, source: Literal["inputs", "outputs"]) -> list[Dataset]: for output in getattr(completed, source): dataset_uri = output.namespace + "/" + output.name uris.append(dataset_uri) + logger.debug("URIs to be converted to Dataset: %s", uris) return [Dataset(uri) for uri in uris] def register_dataset(self, new_inlets: list[Dataset], new_outlets: list[Dataset]) -> None: