Merge branch '1.7.latest' into ADAP-9681/fix_renamed_relations_overri…

…de_pre_1.8

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.7.8
+current_version = 1.7.10
 parse = (?P<major>[\d]+) # major version number
 	\.(?P<minor>[\d]+) # minor version number
 	\.(?P<patch>[\d]+) # patch version number

.changes/1.7.10.md

@@ -0,0 +1,11 @@
+## dbt-core 1.7.10 - March 14, 2024
+
+### Fixes
+
+- Do not add duplicate input_measures ([#9360](https://github.com/dbt-labs/dbt-core/issues/9360))
+- Fix partial parsing `KeyError` on deleted schema files ([#8860](https://github.com/dbt-labs/dbt-core/issues/8860))
+- Support saved queries in `dbt list` ([#9532](https://github.com/dbt-labs/dbt-core/issues/9532))
+
+### Dependencies
+
+- Restrict protobuf to 4.* versions ([#9566](https://github.com/dbt-labs/dbt-core/pull/9566))

.changes/1.7.9.md

@@ -0,0 +1,18 @@
+## dbt-core 1.7.9 - February 28, 2024
+
+### Fixes
+
+- Fix node_info contextvar handling so incorrect node_info doesn't persist ([#8866](https://github.com/dbt-labs/dbt-core/issues/8866))
+- Add target-path to retry ([#8948](https://github.com/dbt-labs/dbt-core/issues/8948))
+
+### Under the Hood
+
+- Make dbt-core compatible with Python 3.12 ([#9007](https://github.com/dbt-labs/dbt-core/issues/9007))
+- Restrict protobuf to major version 4. ([#9566](https://github.com/dbt-labs/dbt-core/issues/9566))
+
+### Security
+
+- Update Jinja2 to >= 3.1.3 to address CVE-2024-22195 ([#CVE-2024-22195](https://github.com/dbt-labs/dbt-core/pull/CVE-2024-22195))
+
+### Contributors
+- [@l1xnan](https://github.com/l1xnan) ([#9007](https://github.com/dbt-labs/dbt-core/issues/9007))

.changes/unreleased/Fixes-20240216-145632.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Tighten exception handling to avoid worker thread hangs.
+time: 2024-02-16T14:56:32.858967-05:00
+custom:
+  Author: peterallenwebb
+  Issue: "9583"

CHANGELOG.md

@@ -5,15 +5,46 @@
 - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
 - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry)
 
+## dbt-core 1.7.10 - March 14, 2024
+
+### Fixes
+
+- Do not add duplicate input_measures ([#9360](https://github.com/dbt-labs/dbt-core/issues/9360))
+- Fix partial parsing `KeyError` on deleted schema files ([#8860](https://github.com/dbt-labs/dbt-core/issues/8860))
+- Support saved queries in `dbt list` ([#9532](https://github.com/dbt-labs/dbt-core/issues/9532))
+
+### Dependencies
+
+- Restrict protobuf to 4.* versions ([#9566](https://github.com/dbt-labs/dbt-core/pull/9566))
+
+
+
+## dbt-core 1.7.9 - February 28, 2024
+
+### Fixes
+
+- Fix node_info contextvar handling so incorrect node_info doesn't persist ([#8866](https://github.com/dbt-labs/dbt-core/issues/8866))
+- Add target-path to retry ([#8948](https://github.com/dbt-labs/dbt-core/issues/8948))
+
+### Under the Hood
+
+- Make dbt-core compatible with Python 3.12 ([#9007](https://github.com/dbt-labs/dbt-core/issues/9007))
+- Restrict protobuf to major version 4. ([#9566](https://github.com/dbt-labs/dbt-core/issues/9566))
+
+### Security
+
+- Update Jinja2 to >= 3.1.3 to address CVE-2024-22195 ([#CVE-2024-22195](https://github.com/dbt-labs/dbt-core/pull/CVE-2024-22195))
+
+### Contributors
+- [@l1xnan](https://github.com/l1xnan) ([#9007](https://github.com/dbt-labs/dbt-core/issues/9007))
+
 ## dbt-core 1.7.8 - February 14, 2024
 
 ### Fixes
 
 - When patching versioned models, set constraints after config ([#9364](https://github.com/dbt-labs/dbt-core/issues/9364))
 - Store node_info in node associated logging events ([#9557](https://github.com/dbt-labs/dbt-core/issues/9557))
 
-
-
 ## dbt-core 1.7.7 - February 01, 2024
 
 ### Fixes

core/dbt/cli/params.py

@@ -390,6 +390,7 @@
         [
             "metric",
             "semantic_model",
+            "saved_query",
             "source",
             "analysis",
             "model",

core/dbt/contracts/graph/nodes.py

@@ -1551,6 +1551,12 @@ def same_contents(self, old: Optional["Metric"]) -> bool:
             and True
         )
 
+    def add_input_measure(self, input_measure: MetricInputMeasure) -> None:
+        for existing_input_measure in self.type_params.input_measures:
+            if input_measure == existing_input_measure:
+                return
+        self.type_params.input_measures.append(input_measure)
+
 
 # ====================================
 # Group node

core/dbt/parser/manifest.py

@@ -1572,7 +1572,7 @@ def _process_metric_node(
         assert (
             metric.type_params.measure is not None
         ), f"{metric} should have a measure defined, but it does not."
-        metric.type_params.input_measures.append(metric.type_params.measure)
+        metric.add_input_measure(metric.type_params.measure)
         _process_metric_depends_on(
             manifest=manifest, current_project=current_project, metric=metric
         )
@@ -1581,8 +1581,8 @@ def _process_metric_node(
         assert (
             conversion_type_params
         ), f"{metric.name} is a conversion metric and must have conversion_type_params defined."
-        metric.type_params.input_measures.append(conversion_type_params.base_measure)
-        metric.type_params.input_measures.append(conversion_type_params.conversion_measure)
+        metric.add_input_measure(conversion_type_params.base_measure)
+        metric.add_input_measure(conversion_type_params.conversion_measure)
         _process_metric_depends_on(
             manifest=manifest, current_project=current_project, metric=metric
         )
@@ -1618,7 +1618,8 @@ def _process_metric_node(
             _process_metric_node(
                 manifest=manifest, current_project=current_project, metric=target_metric
             )
-            metric.type_params.input_measures.extend(target_metric.type_params.input_measures)
+            for input_measure in target_metric.type_params.input_measures:
+                metric.add_input_measure(input_measure)
             metric.depends_on.add_node(target_metric.unique_id)
     else:
         assert_values_exhausted(metric.type)

core/dbt/parser/partial.py

@@ -208,6 +208,7 @@ def add_to_pp_files(self, source_file):
         if (
             file_id not in self.project_parser_files[project_name][parser_name]
             and file_id not in self.file_diff["deleted"]
+            and file_id not in self.file_diff["deleted_schema_files"]
         ):
             self.project_parser_files[project_name][parser_name].append(file_id)
 
@@ -446,7 +447,11 @@ def schedule_nodes_for_parsing(self, unique_ids):
 
     def _schedule_for_parsing(self, dict_key: str, element, name, delete: Callable) -> None:
         file_id = element.file_id
-        if file_id in self.saved_files and file_id not in self.file_diff["deleted"]:
+        if (
+            file_id in self.saved_files
+            and file_id not in self.file_diff["deleted"]
+            and file_id not in self.file_diff["deleted_schema_files"]
+        ):
             schema_file = self.saved_files[file_id]
             elements = []
             assert isinstance(schema_file, SchemaSourceFile)

core/dbt/task/README.md

@@ -1 +1,43 @@
 # Task README
+
+### Task Hierarchy
+```
+BaseTask
+ ┣ CleanTask
+ ┣ ConfiguredTask
+ ┃ ┣ GraphRunnableTask
+ ┃ ┃  ┣ CloneTask
+ ┃ ┃  ┣ CompileTask
+ ┃ ┃  ┃ ┣ GenerateTask
+ ┃ ┃  ┃ ┣ RunTask
+ ┃ ┃  ┃ ┃ ┣ BuildTask
+ ┃ ┃  ┃ ┃ ┣ FreshnessTask
+ ┃ ┃  ┃ ┃ ┣ SeedTask
+ ┃ ┃  ┃ ┃ ┣ SnapshotTask
+ ┃ ┃  ┃ ┃ ┗ TestTask 
+ ┃ ┃  ┃ ┗ ShowTask
+ ┃ ┃  ┗ ListTask
+ ┃ ┣ RetryTask 
+ ┃ ┣ RunOperationTask
+ ┃ ┗ ServeTask
+ ┣ DebugTask
+ ┣ DepsTask
+ ┗ InitTask
+```
+
+### Runner Hierarchy
+```
+BaseRunner
+ ┣ CloneRunner
+ ┣ CompileRunner
+ ┃ ┣ GenericSqlRunner
+ ┃ ┃  ┣ SqlCompileRunner
+ ┃ ┃  ┗ SqlExecuteRunner
+ ┃ ┣ ModelRunner
+ ┃ ┃ ┣ SeedRunner
+ ┃ ┃ ┗ SnapshotRunner
+ ┃ ┣ ShowRunner
+ ┃ ┗ TestRunner
+ ┣ FreshnessRunner
+ ┗ SavedQueryRunner
+```

core/dbt/task/list.py

@@ -1,6 +1,12 @@
 import json
 
-from dbt.contracts.graph.nodes import Exposure, SourceDefinition, Metric, SemanticModel
+from dbt.contracts.graph.nodes import (
+    Exposure,
+    SourceDefinition,
+    Metric,
+    SavedQuery,
+    SemanticModel,
+)
 from dbt.flags import get_flags
 from dbt.graph import ResourceTypeSelector
 from dbt.task.runnable import GraphRunnableTask
@@ -28,6 +34,7 @@ class ListTask(GraphRunnableTask):
             NodeType.Source,
             NodeType.Exposure,
             NodeType.Metric,
+            NodeType.SavedQuery,
             NodeType.SemanticModel,
         )
     )
@@ -77,10 +84,12 @@ def _iterate_selected_nodes(self):
                 yield self.manifest.metrics[node]
             elif node in self.manifest.semantic_models:
                 yield self.manifest.semantic_models[node]
+            elif node in self.manifest.saved_queries:
+                yield self.manifest.saved_queries[node]
             else:
                 raise DbtRuntimeError(
                     f'Got an unexpected result from node selection: "{node}"'
-                    f"Expected a source or a node!"
+                    f"Listing this node type is not yet supported!"
                 )
 
     def generate_selectors(self):
@@ -100,6 +109,10 @@ def generate_selectors(self):
                 # metrics are searched for by pkg.metric_name
                 metric_selector = ".".join([node.package_name, node.name])
                 yield f"metric:{metric_selector}"
+            elif node.resource_type == NodeType.SavedQuery:
+                assert isinstance(node, SavedQuery)
+                saved_query_selector = ".".join([node.package_name, node.name])
+                yield f"saved_query:{saved_query_selector}"
             elif node.resource_type == NodeType.SemanticModel:
                 assert isinstance(node, SemanticModel)
                 semantic_model_selector = ".".join([node.package_name, node.name])

core/dbt/task/runnable.py

@@ -35,6 +35,7 @@
     ConcurrencyLine,
     EndRunResult,
     NothingToDo,
+    GenericExceptionOnRun,
 )
 from dbt.exceptions import (
     DbtInternalError,
@@ -204,17 +205,44 @@ def call_runner(self, runner: BaseRunner) -> RunResult:
                     )
                 )
             status: Dict[str, str] = {}
+            result = None
+            thread_exception = None
             try:
                 result = runner.run_with_hooks(self.manifest)
+            except Exception as e:
+                thread_exception = e
             finally:
                 finishctx = TimestampNamed("finished_at")
                 with finishctx, DbtModelState(status):
-                    fire_event(
-                        NodeFinished(
-                            node_info=runner.node.node_info,
-                            run_result=result.to_msg_dict(),
+                    if result is not None:
+                        fire_event(
+                            NodeFinished(
+                                node_info=runner.node.node_info,
+                                run_result=result.to_msg_dict(),
+                            )
                         )
-                    )
+                    else:
+                        msg = f"Exception on worker thread. {thread_exception}"
+
+                        fire_event(
+                            GenericExceptionOnRun(
+                                unique_id=runner.node.unique_id,
+                                exc=str(thread_exception),
+                                node_info=runner.node.node_info,
+                            )
+                        )
+
+                        result = RunResult(
+                            status=RunStatus.Error,  # type: ignore
+                            timing=[],
+                            thread_id="",
+                            execution_time=0.0,
+                            adapter_response={},
+                            message=msg,
+                            failures=None,
+                            node=runner.node,
+                        )
+
             # `_event_status` dict is only used for logging.  Make sure
             # it gets deleted when we're done with it
             runner.node.clear_event_status()

core/dbt/version.py

@@ -232,5 +232,5 @@ def _get_adapter_plugin_names() -> Iterator[str]:
             yield plugin_name
 
 
-__version__ = "1.7.8"
+__version__ = "1.7.10"
 installed = get_installed_version()

core/setup.py

@@ -25,7 +25,7 @@
 
 
 package_name = "dbt-core"
-package_version = "1.7.8"
+package_version = "1.7.10"
 description = """With dbt, data analysts and engineers can build analytics \
 the way engineers build applications."""
 
@@ -79,7 +79,7 @@
         # Expect compatibility with all new versions of these packages, so lower bounds only.
         "jsonschema>=3.0",
         "packaging>20.9",
-        "protobuf>=4.0.0",
+        "protobuf>=4.0.0,<5",
         "pytz>=2015.7",
         "pyyaml>=6.0",
         "typing-extensions>=3.7.4",

docker/Dockerfile

@@ -16,12 +16,12 @@ FROM --platform=$build_for python:3.10.7-slim-bullseye as base
 # N.B. The refs updated automagically every release via bumpversion
 # N.B. dbt-postgres is currently found in the core codebase so a value of dbt-core@<some_version> is correct
 
-ARG [email protected].8
-ARG [email protected].8
-ARG [email protected].8
-ARG [email protected].8
-ARG [email protected].8
-ARG [email protected].8
+ARG [email protected].10
+ARG [email protected].10
+ARG [email protected].10
+ARG [email protected].10
+ARG [email protected].10
+ARG [email protected].10
 # special case args
 ARG dbt_spark_version=all
 ARG dbt_third_party

plugins/postgres/dbt/adapters/postgres/__version__.py

@@ -1 +1 @@
-version = "1.7.8"
+version = "1.7.10"