Support parsing dbt_project.yml without target-path

As of dbt v1.5, usage of target-path in the dbt_project.yml file has been deprecated, now preferring a CLI flag or env var. It will be removed in a future version. See dbt-labs/dbt-core#6882

Docs: https://docs.getdbt.com/reference/project-configs/target-path

This change allows users to run DbtLocalArtifactProcessor in dbt projects that don't declare target-path

Fix: OpenLineage#2093

integration/common/openlineage/common/provider/dbt/local.py

@@ -12,6 +12,10 @@
 from openlineage.common.utils import get_from_nullable_chain
 
 
+DBT_TARGET_PATH_ENVVAR = "DBT_TARGET_PATH"
+DEFAULT_TARGET_PATH = "target"
+
+
 class SkipUndefined(Undefined):
     def __getattr__(self, name):
         return SkipUndefined(name=f"{self._undefined_name}.{name}")
@@ -53,15 +57,16 @@ def __init__(
         dbt_project = self.load_yaml_with_jinja(
             os.path.join(project_dir, "dbt_project.yml")
         )
+        target_path = self.build_target_folder(dbt_project)
 
         self.manifest_path = os.path.join(
-            absolute_dir, dbt_project["target-path"], "manifest.json"
+            absolute_dir, target_path, "manifest.json"
         )
         self.run_result_path = os.path.join(
-            absolute_dir, dbt_project["target-path"], "run_results.json"
+            absolute_dir, target_path, "run_results.json"
         )
         self.catalog_path = os.path.join(
-            absolute_dir, dbt_project["target-path"], "catalog.json"
+            absolute_dir, target_path, "catalog.json"
         )
 
         self.target = target
@@ -71,6 +76,19 @@ def __init__(
         if not self.profile_name:
             raise KeyError(f"profile not found in {dbt_project}")
 
+    def build_target_folder(self, dbt_project: dict) -> str:
+        """
+        Build dbt target path. Uses the following:
+        1. DBT_TARGET_PATH environment variable
+        2. target-path in dbt_project.yml
+
+        Precedence order: env var > dbt_project.yml
+
+        Reference:
+        https://docs.getdbt.com/reference/project-configs/target-path
+        """
+        return os.getenv(DBT_TARGET_PATH_ENVVAR) or dbt_project.get("target-path", DEFAULT_TARGET_PATH)
+
     @classmethod
     def load_metadata(
         cls, path: str, desired_schema_versions: List[int], logger: logging.Logger

integration/common/tests/dbt/small/dbt_project.yml

@@ -19,7 +19,6 @@ data-paths: ["data"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
 
-target-path: "target"  # directory which will store compiled SQL files
 clean-targets:         # directories to be removed by `dbt clean`
     - "target"
     - "dbt_modules"

integration/common/tests/dbt/test_dbt_local.py

@@ -238,3 +238,31 @@ def test_logging_handler_does_not_warn():
     DbtLocalArtifactProcessor.load_metadata(path, [2], logger)
 
     logger.warning.assert_not_called()
+
+
+@mock.patch.dict(os.environ, {"DBT_TARGET_PATH": "target-from-envvar"}, clear=True)
+def test_build_target_folder_with_envvar():
+    processor = DbtLocalArtifactProcessor(
+        producer='https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt',
+        project_dir='tests/dbt/env_vars',
+        target='prod',
+        job_namespace="ol-namespace"
+    )
+    assert processor.build_target_folder({}) == "target-from-envvar"
+
+@pytest.mark.parametrize(
+    "test_name,dbt_project,expected",
+    [
+        ("with_dbt_project", {"target-path": "from-dbt-project"}, "from-dbt-project"),
+        ("with_default", {}, "target")
+    ]
+
+)
+def test_build_target_folder(test_name, dbt_project, expected):
+    processor = DbtLocalArtifactProcessor(
+        producer='https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt',
+        project_dir='tests/dbt/env_vars',
+        target='prod',
+        job_namespace="ol-namespace"
+    )
+    assert processor.build_target_folder(dbt_project) == expected