From d01d33c28a4a8d8e2628dd641074a867b5530547 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Wed, 28 Feb 2024 21:08:53 +0000 Subject: [PATCH 1/7] Bumping version to 1.7.9 and generate changelog --- .bumpversion.cfg | 2 +- .changes/1.7.9.md | 18 +++++++++++++++ .../unreleased/Fixes-20240222-100958.yaml | 6 ----- .../unreleased/Fixes-20240223-162107.yaml | 6 ----- .../unreleased/Security-20240222-152445.yaml | 6 ----- .../Under the Hood-20240221-104518.yaml | 6 ----- .../Under the Hood-20240221-145058.yaml | 6 ----- CHANGELOG.md | 22 +++++++++++++++++-- core/dbt/version.py | 2 +- core/setup.py | 2 +- docker/Dockerfile | 12 +++++----- .../dbt/adapters/postgres/__version__.py | 2 +- plugins/postgres/setup.py | 2 +- .../adapter/dbt/tests/adapter/__version__.py | 2 +- tests/adapter/setup.py | 2 +- 15 files changed, 51 insertions(+), 45 deletions(-) create mode 100644 .changes/1.7.9.md delete mode 100644 .changes/unreleased/Fixes-20240222-100958.yaml delete mode 100644 .changes/unreleased/Fixes-20240223-162107.yaml delete mode 100644 .changes/unreleased/Security-20240222-152445.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240221-104518.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240221-145058.yaml diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ece452dd88f..8fb51b0e717 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.7.8 +current_version = 1.7.9 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.7.9.md b/.changes/1.7.9.md new file mode 100644 index 00000000000..a1a960f6ff8 --- /dev/null +++ b/.changes/1.7.9.md @@ -0,0 +1,18 @@ +## dbt-core 1.7.9 - February 28, 2024 + +### Fixes + +- Fix node_info contextvar handling so incorrect node_info doesn't persist ([#8866](https://github.com/dbt-labs/dbt-core/issues/8866)) +- Add target-path to retry ([#8948](https://github.com/dbt-labs/dbt-core/issues/8948)) + +### Under the Hood + +- Make dbt-core compatible with Python 3.12 ([#9007](https://github.com/dbt-labs/dbt-core/issues/9007)) +- Restrict protobuf to major version 4. ([#9566](https://github.com/dbt-labs/dbt-core/issues/9566)) + +### Security + +- Update Jinja2 to >= 3.1.3 to address CVE-2024-22195 ([#CVE-2024-22195](https://github.com/dbt-labs/dbt-core/pull/CVE-2024-22195)) + +### Contributors +- [@l1xnan](https://github.com/l1xnan) ([#9007](https://github.com/dbt-labs/dbt-core/issues/9007)) diff --git a/.changes/unreleased/Fixes-20240222-100958.yaml b/.changes/unreleased/Fixes-20240222-100958.yaml deleted file mode 100644 index 1fb2ff46c6f..00000000000 --- a/.changes/unreleased/Fixes-20240222-100958.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix node_info contextvar handling so incorrect node_info doesn't persist -time: 2024-02-22T10:09:58.122809-05:00 -custom: - Author: gshank - Issue: "8866" diff --git a/.changes/unreleased/Fixes-20240223-162107.yaml b/.changes/unreleased/Fixes-20240223-162107.yaml deleted file mode 100644 index 446cf6d077a..00000000000 --- a/.changes/unreleased/Fixes-20240223-162107.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Add target-path to retry -time: 2024-02-23T16:21:07.83639Z -custom: - Author: aranke - Issue: "8948" diff --git a/.changes/unreleased/Security-20240222-152445.yaml b/.changes/unreleased/Security-20240222-152445.yaml deleted file mode 100644 index 5c92c452ab0..00000000000 --- a/.changes/unreleased/Security-20240222-152445.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Security -body: Update Jinja2 to >= 3.1.3 to address CVE-2024-22195 -time: 2024-02-22T15:24:45.158305-08:00 -custom: - Author: QMalcolm - PR: CVE-2024-22195 diff --git a/.changes/unreleased/Under the Hood-20240221-104518.yaml b/.changes/unreleased/Under the Hood-20240221-104518.yaml deleted file mode 100644 index 56c077fcd1c..00000000000 --- a/.changes/unreleased/Under the Hood-20240221-104518.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Restrict protobuf to major version 4. -time: 2024-02-21T10:45:18.315195-05:00 -custom: - Author: peterallenwebb - Issue: "9566" diff --git a/.changes/unreleased/Under the Hood-20240221-145058.yaml b/.changes/unreleased/Under the Hood-20240221-145058.yaml deleted file mode 100644 index a847bb68c53..00000000000 --- a/.changes/unreleased/Under the Hood-20240221-145058.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Make dbt-core compatible with Python 3.12 -time: 2024-02-21T14:50:58.983559Z -custom: - Author: l1xnan aranke - Issue: "9007" diff --git a/CHANGELOG.md b/CHANGELOG.md index 368d21550ae..baea75edc0d 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,26 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-core 1.7.9 - February 28, 2024 + +### Fixes + +- Fix node_info contextvar handling so incorrect node_info doesn't persist ([#8866](https://github.com/dbt-labs/dbt-core/issues/8866)) +- Add target-path to retry ([#8948](https://github.com/dbt-labs/dbt-core/issues/8948)) + +### Under the Hood + +- Make dbt-core compatible with Python 3.12 ([#9007](https://github.com/dbt-labs/dbt-core/issues/9007)) +- Restrict protobuf to major version 4. ([#9566](https://github.com/dbt-labs/dbt-core/issues/9566)) + +### Security + +- Update Jinja2 to >= 3.1.3 to address CVE-2024-22195 ([#CVE-2024-22195](https://github.com/dbt-labs/dbt-core/pull/CVE-2024-22195)) + +### Contributors +- [@l1xnan](https://github.com/l1xnan) ([#9007](https://github.com/dbt-labs/dbt-core/issues/9007)) + + ## dbt-core 1.7.8 - February 14, 2024 ### Fixes @@ -12,8 +32,6 @@ - When patching versioned models, set constraints after config ([#9364](https://github.com/dbt-labs/dbt-core/issues/9364)) - Store node_info in node associated logging events ([#9557](https://github.com/dbt-labs/dbt-core/issues/9557)) - - ## dbt-core 1.7.7 - February 01, 2024 ### Fixes diff --git a/core/dbt/version.py b/core/dbt/version.py index 78c4d5723e7..1799ed6b8c3 100644 --- a/core/dbt/version.py +++ b/core/dbt/version.py @@ -232,5 +232,5 @@ def _get_adapter_plugin_names() -> Iterator[str]: yield plugin_name -__version__ = "1.7.8" +__version__ = "1.7.9" installed = get_installed_version() diff --git a/core/setup.py b/core/setup.py index 136f8d10483..ec1607786ac 100644 --- a/core/setup.py +++ b/core/setup.py @@ -25,7 +25,7 @@ package_name = "dbt-core" -package_version = "1.7.8" +package_version = "1.7.9" description = """With dbt, data analysts and engineers can build analytics \ the way engineers build applications.""" diff --git a/docker/Dockerfile b/docker/Dockerfile index e804684fd7e..ae9cb508628 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,12 +16,12 @@ FROM --platform=$build_for python:3.10.7-slim-bullseye as base # N.B. The refs updated automagically every release via bumpversion # N.B. dbt-postgres is currently found in the core codebase so a value of dbt-core@ is correct -ARG dbt_core_ref=dbt-core@v1.7.8 -ARG dbt_postgres_ref=dbt-core@v1.7.8 -ARG dbt_redshift_ref=dbt-redshift@v1.7.8 -ARG dbt_bigquery_ref=dbt-bigquery@v1.7.8 -ARG dbt_snowflake_ref=dbt-snowflake@v1.7.8 -ARG dbt_spark_ref=dbt-spark@v1.7.8 +ARG dbt_core_ref=dbt-core@v1.7.9 +ARG dbt_postgres_ref=dbt-core@v1.7.9 +ARG dbt_redshift_ref=dbt-redshift@v1.7.9 +ARG dbt_bigquery_ref=dbt-bigquery@v1.7.9 +ARG dbt_snowflake_ref=dbt-snowflake@v1.7.9 +ARG dbt_spark_ref=dbt-spark@v1.7.9 # special case args ARG dbt_spark_version=all ARG dbt_third_party diff --git a/plugins/postgres/dbt/adapters/postgres/__version__.py b/plugins/postgres/dbt/adapters/postgres/__version__.py index 9bebf548a8e..52a993a81b5 100644 --- a/plugins/postgres/dbt/adapters/postgres/__version__.py +++ b/plugins/postgres/dbt/adapters/postgres/__version__.py @@ -1 +1 @@ -version = "1.7.8" +version = "1.7.9" diff --git a/plugins/postgres/setup.py b/plugins/postgres/setup.py index 9957b759b20..b20f639ef77 100644 --- a/plugins/postgres/setup.py +++ b/plugins/postgres/setup.py @@ -41,7 +41,7 @@ def _dbt_psycopg2_name(): package_name = "dbt-postgres" -package_version = "1.7.8" +package_version = "1.7.9" description = """The postgres adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/adapter/dbt/tests/adapter/__version__.py b/tests/adapter/dbt/tests/adapter/__version__.py index 9bebf548a8e..52a993a81b5 100644 --- a/tests/adapter/dbt/tests/adapter/__version__.py +++ b/tests/adapter/dbt/tests/adapter/__version__.py @@ -1 +1 @@ -version = "1.7.8" +version = "1.7.9" diff --git a/tests/adapter/setup.py b/tests/adapter/setup.py index ee11678e938..99ef6f2e504 100644 --- a/tests/adapter/setup.py +++ b/tests/adapter/setup.py @@ -20,7 +20,7 @@ package_name = "dbt-tests-adapter" -package_version = "1.7.8" +package_version = "1.7.9" description = """The dbt adapter tests for adapter plugins""" this_directory = os.path.abspath(os.path.dirname(__file__)) From 6919c501d3cd289388e2a1e297c583b19af9b007 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Feb 2024 14:21:34 -0800 Subject: [PATCH 2/7] [Backport 1.7.latest] Restrict protobuf to 4.* versions (#9707) * Restrict protobuf to 4.* versions (#9630) Protobuf v5 has breaking changes. Here we are limiting the protobuf dependency to one major version, 4, so that we don't have to patch over handling 2 different major versions of protobuf. (cherry picked from commit e4fe839e4574187b574473596a471092267a9f2e) --------- Co-authored-by: Quigley Malcolm Co-authored-by: Quigley Malcolm --- .changes/unreleased/Dependencies-20240222-102947.yaml | 6 ++++++ core/setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20240222-102947.yaml diff --git a/.changes/unreleased/Dependencies-20240222-102947.yaml b/.changes/unreleased/Dependencies-20240222-102947.yaml new file mode 100644 index 00000000000..9c21d51e21e --- /dev/null +++ b/.changes/unreleased/Dependencies-20240222-102947.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: Restrict protobuf to 4.* versions +time: 2024-02-22T10:29:47.595435-08:00 +custom: + Author: QMalcolm + PR: "9566" diff --git a/core/setup.py b/core/setup.py index ec1607786ac..f3f51ba5ad2 100644 --- a/core/setup.py +++ b/core/setup.py @@ -79,7 +79,7 @@ # Expect compatibility with all new versions of these packages, so lower bounds only. "jsonschema>=3.0", "packaging>20.9", - "protobuf>=4.0.0", + "protobuf>=4.0.0,<5", "pytz>=2015.7", "pyyaml>=6.0", "typing-extensions>=3.7.4", From 53d28eed1e2d31759526f4935d6b58aaca55802a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 13:40:18 -0500 Subject: [PATCH 3/7] [Backport 1.7.latest] Stop trying to parse deleted schema files (#9732) * Stop trying to parse deleted schema files (#9722) * Add test around deleting a YAML file containing semantic models and metrics It was raised in https://github.com/dbt-labs/dbt-core/issues/8860 that an error is being raised during partial parsing when files containing metrics/semantic models are deleted. In further testing it looks like this error specifically happens when a file containing both semantic models and metrics is deleted. If the deleted file contains just semantic models or metrics there seems to be no issue. The next commit should contain the fix. * Skip deleted schema files when scheduling files during partial parsing Waaaay back (in 7563b99) deleted schema files started being separated out from deleted non-schema files. However ever since, when it came to scheduling files for reparsing, we've only done so for deleted non-schema files. We even missed this when we refactored the scheduling code in b37e5b5. This change updates `_schedule_for_parsing` which is used by `schedule_nodes_for_parsing` to begin skipping deleted schema files in addition to deleted non schema files. * Update `add_to_pp_files` to ignore `deleted_schema_files` As noted in the previous commit, we started separating out deleted schema files from deleted non-schema files a looong time ago. However, this whole time we've been adding `deleted_schema_files` to the list of files to be parsed. This change corrects for that. * Add changie doc for partial parsing KeyError fix (cherry picked from commit deedeeb9cecad7069152740079eda8315d7fa420) * Empty commit to trigger github actions --------- Co-authored-by: Quigley Malcolm Co-authored-by: Quigley Malcolm --- .../unreleased/Fixes-20240301-135536.yaml | 6 +++ core/dbt/parser/partial.py | 7 ++- tests/functional/partial_parsing/fixtures.py | 54 +++++++++++++++++++ .../partial_parsing/test_pp_metrics.py | 31 ++++++++++- 4 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240301-135536.yaml diff --git a/.changes/unreleased/Fixes-20240301-135536.yaml b/.changes/unreleased/Fixes-20240301-135536.yaml new file mode 100644 index 00000000000..2a96bd7eeec --- /dev/null +++ b/.changes/unreleased/Fixes-20240301-135536.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix partial parsing `KeyError` on deleted schema files +time: 2024-03-01T13:55:36.533176-08:00 +custom: + Author: QMalcolm + Issue: "8860" diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py index 7fcfe48f87d..4463b544244 100644 --- a/core/dbt/parser/partial.py +++ b/core/dbt/parser/partial.py @@ -208,6 +208,7 @@ def add_to_pp_files(self, source_file): if ( file_id not in self.project_parser_files[project_name][parser_name] and file_id not in self.file_diff["deleted"] + and file_id not in self.file_diff["deleted_schema_files"] ): self.project_parser_files[project_name][parser_name].append(file_id) @@ -446,7 +447,11 @@ def schedule_nodes_for_parsing(self, unique_ids): def _schedule_for_parsing(self, dict_key: str, element, name, delete: Callable) -> None: file_id = element.file_id - if file_id in self.saved_files and file_id not in self.file_diff["deleted"]: + if ( + file_id in self.saved_files + and file_id not in self.file_diff["deleted"] + and file_id not in self.file_diff["deleted_schema_files"] + ): schema_file = self.saved_files[file_id] elements = [] assert isinstance(schema_file, SchemaSourceFile) diff --git a/tests/functional/partial_parsing/fixtures.py b/tests/functional/partial_parsing/fixtures.py index 52578d90308..78068151dc3 100644 --- a/tests/functional/partial_parsing/fixtures.py +++ b/tests/functional/partial_parsing/fixtures.py @@ -452,6 +452,60 @@ agg_time_dimension: created_at """ +people_sl_yml = """ +version: 2 + +semantic_models: + - name: semantic_people + model: ref('people') + dimensions: + - name: favorite_color + type: categorical + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + agg: SUM + expr: tenure + - name: people + agg: count + expr: id + entities: + - name: id + type: primary + defaults: + agg_time_dimension: created_at + +metrics: + + - name: number_of_people + description: Total count of people + label: "Number of people" + type: simple + type_params: + measure: people + meta: + my_meta: 'testing' + + - name: collective_tenure + description: Total number of years of team experience + label: "Collective tenure" + type: simple + type_params: + measure: + name: years_tenure + filter: "{{ Dimension('id__loves_dbt') }} is true" + + - name: average_tenure + label: Average Tenure + type: ratio + type_params: + numerator: collective_tenure + denominator: number_of_people +""" + env_var_metrics_yml = """ metrics: diff --git a/tests/functional/partial_parsing/test_pp_metrics.py b/tests/functional/partial_parsing/test_pp_metrics.py index da994e09808..19da625604b 100644 --- a/tests/functional/partial_parsing/test_pp_metrics.py +++ b/tests/functional/partial_parsing/test_pp_metrics.py @@ -1,6 +1,8 @@ import pytest -from dbt.tests.util import run_dbt, write_file, get_manifest +from dbt.cli.main import dbtRunner +from dbt.contracts.graph.manifest import Manifest +from dbt.tests.util import run_dbt, rm_file, write_file, get_manifest from tests.functional.partial_parsing.fixtures import ( people_sql, metricflow_time_spine_sql, @@ -9,6 +11,7 @@ people_metrics2_yml, metric_model_a_sql, people_metrics3_yml, + people_sl_yml, ) from dbt.exceptions import CompilationError @@ -84,3 +87,29 @@ def test_metrics(self, project): # We use "parse" here and not "run" because we're checking that the CompilationError # occurs at parse time, not compilation results = run_dbt(["parse"]) + + +class TestDeleteFileWithMetricsAndSemanticModels: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "people_sl.yml": people_sl_yml, + } + + def test_metrics(self, project): + # Initial parsing + runner = dbtRunner() + result = runner.invoke(["parse"]) + assert result.success + manifest = result.result + assert isinstance(manifest, Manifest) + assert len(manifest.metrics) == 3 + + # Remove metric file + rm_file(project.project_root, "models", "people_sl.yml") + + # Rerun parse, shouldn't fail + result = runner.invoke(["parse"]) + assert result.exception is None, result.exception From 88bb7a3537ac2c183d65ac6f9631d1450ec3fd2c Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Mon, 11 Mar 2024 15:14:54 -0700 Subject: [PATCH 4/7] [Backport 1.7.latest][MANUAL] Fix listing saved queries (#9748) * Add tests to check that saved queries show in `dbt list` * Update `list` task to support saved queries This is built off of @jtcohen6 work in d6e7cda on jerco/fix-9532. I didn't directly cherry pick because there was more work to do as well as merge conflicts. That is to say @jtcohen6 should be credited with some of the work. * Update error message when iterating over nodes during list command errors This was originally suggested by @jtcohen6 in d6e7cda of jerco/fix-9532. This commit just makes sure the change gets included because I didn't cherry-pick that commit into this work. * Add changie log for saved query list support --- .../unreleased/Fixes-20240307-142459.yaml | 6 +++++ core/dbt/cli/params.py | 1 + core/dbt/task/list.py | 17 +++++++++++-- tests/functional/list/fixtures.py | 24 +++++++++++++++++++ tests/functional/list/test_list.py | 5 ++++ 5 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240307-142459.yaml diff --git a/.changes/unreleased/Fixes-20240307-142459.yaml b/.changes/unreleased/Fixes-20240307-142459.yaml new file mode 100644 index 00000000000..14c08da2816 --- /dev/null +++ b/.changes/unreleased/Fixes-20240307-142459.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Support saved queries in `dbt list` +time: 2024-03-07T14:24:59.530072-05:00 +custom: + Author: QMalcolm jtcohen6 + Issue: "9532" diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 1898815a724..b29cd0fefd1 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -390,6 +390,7 @@ [ "metric", "semantic_model", + "saved_query", "source", "analysis", "model", diff --git a/core/dbt/task/list.py b/core/dbt/task/list.py index 3b9448aeb9d..4cd5f56275a 100644 --- a/core/dbt/task/list.py +++ b/core/dbt/task/list.py @@ -1,6 +1,12 @@ import json -from dbt.contracts.graph.nodes import Exposure, SourceDefinition, Metric, SemanticModel +from dbt.contracts.graph.nodes import ( + Exposure, + SourceDefinition, + Metric, + SavedQuery, + SemanticModel, +) from dbt.flags import get_flags from dbt.graph import ResourceTypeSelector from dbt.task.runnable import GraphRunnableTask @@ -28,6 +34,7 @@ class ListTask(GraphRunnableTask): NodeType.Source, NodeType.Exposure, NodeType.Metric, + NodeType.SavedQuery, NodeType.SemanticModel, ) ) @@ -77,10 +84,12 @@ def _iterate_selected_nodes(self): yield self.manifest.metrics[node] elif node in self.manifest.semantic_models: yield self.manifest.semantic_models[node] + elif node in self.manifest.saved_queries: + yield self.manifest.saved_queries[node] else: raise DbtRuntimeError( f'Got an unexpected result from node selection: "{node}"' - f"Expected a source or a node!" + f"Listing this node type is not yet supported!" ) def generate_selectors(self): @@ -100,6 +109,10 @@ def generate_selectors(self): # metrics are searched for by pkg.metric_name metric_selector = ".".join([node.package_name, node.name]) yield f"metric:{metric_selector}" + elif node.resource_type == NodeType.SavedQuery: + assert isinstance(node, SavedQuery) + saved_query_selector = ".".join([node.package_name, node.name]) + yield f"saved_query:{saved_query_selector}" elif node.resource_type == NodeType.SemanticModel: assert isinstance(node, SemanticModel) semantic_model_selector = ".".join([node.package_name, node.name]) diff --git a/tests/functional/list/fixtures.py b/tests/functional/list/fixtures.py index ea42e2a004f..7d7fb81c7e2 100644 --- a/tests/functional/list/fixtures.py +++ b/tests/functional/list/fixtures.py @@ -145,6 +145,24 @@ """ +saved_queries__sq_yml = """ +saved_queries: + - name: my_saved_query + label: My Saved Query + query_params: + metrics: + - total_outer + group_by: + - "Dimension('my_entity__created_at')" + exports: + - name: my_export + config: + alias: my_export_alias + export_as: table + schema: my_export_schema_name +""" + + @pytest.fixture(scope="class") def snapshots(): return {"snapshot.sql": snapshots__snapshot_sql} @@ -164,6 +182,7 @@ def models(): "docs.md": models__docs_md, "outer.sql": models__outer_sql, "metricflow_time_spine.sql": models__metric_flow, + "sq.yml": saved_queries__sq_yml, "sm.yml": semantic_models__sm_yml, "m.yml": metrics__m_yml, "sub": {"inner.sql": models__sub__inner_sql}, @@ -195,6 +214,11 @@ def metrics(): return {"m.yml": metrics__m_yml} +@pytest.fixture(scope="class") +def saved_queries(): + return {"sq.yml": saved_queries__sq_yml} + + @pytest.fixture(scope="class") def project_files( project_root, diff --git a/tests/functional/list/test_list.py b/tests/functional/list/test_list.py index 49c09e186ba..95c52c8d1dc 100644 --- a/tests/functional/list/test_list.py +++ b/tests/functional/list/test_list.py @@ -14,6 +14,7 @@ analyses, semantic_models, metrics, + saved_queries, project_files, ) @@ -596,6 +597,7 @@ def expect_all_output(self): "test.t", "semantic_model:test.my_sm", "metric:test.total_outer", + "saved_query:test.my_saved_query", } # analyses have their type inserted into their fqn like tests expected_all = expected_default | {"test.analysis.a"} @@ -626,6 +628,9 @@ def expect_select(self): results = self.run_dbt_ls(["--resource-type", "metric"]) assert set(results) == {"metric:test.total_outer"} + results = self.run_dbt_ls(["--resource-type", "saved_query"]) + assert set(results) == {"saved_query:test.my_saved_query"} + results = self.run_dbt_ls(["--resource-type", "model", "--select", "outer+"]) assert set(results) == {"test.outer", "test.sub.inner"} From 5d0dd2a7aab118a2be5eb33d8e374ec107091a49 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 13 Mar 2024 20:36:39 -0400 Subject: [PATCH 5/7] Do not add duplicate input_measures when parsing metrics (#9677) (#9760) --- .../unreleased/Fixes-20240226-173227.yaml | 6 +++ core/dbt/contracts/graph/nodes.py | 6 +++ core/dbt/parser/manifest.py | 9 ++-- tests/functional/metrics/fixtures.py | 53 +++++++++++++++++++ tests/functional/metrics/test_metrics.py | 22 +++++++- 5 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240226-173227.yaml diff --git a/.changes/unreleased/Fixes-20240226-173227.yaml b/.changes/unreleased/Fixes-20240226-173227.yaml new file mode 100644 index 00000000000..fa1bf0ab8cf --- /dev/null +++ b/.changes/unreleased/Fixes-20240226-173227.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Do not add duplicate input_measures +time: 2024-02-26T17:32:27.837427-05:00 +custom: + Author: gshank + Issue: "9360" diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 6c243a523b3..1b17cb2ce32 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1551,6 +1551,12 @@ def same_contents(self, old: Optional["Metric"]) -> bool: and True ) + def add_input_measure(self, input_measure: MetricInputMeasure) -> None: + for existing_input_measure in self.type_params.input_measures: + if input_measure == existing_input_measure: + return + self.type_params.input_measures.append(input_measure) + # ==================================== # Group node diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 9d27de3125c..6ac197ee6f9 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -1572,7 +1572,7 @@ def _process_metric_node( assert ( metric.type_params.measure is not None ), f"{metric} should have a measure defined, but it does not." - metric.type_params.input_measures.append(metric.type_params.measure) + metric.add_input_measure(metric.type_params.measure) _process_metric_depends_on( manifest=manifest, current_project=current_project, metric=metric ) @@ -1581,8 +1581,8 @@ def _process_metric_node( assert ( conversion_type_params ), f"{metric.name} is a conversion metric and must have conversion_type_params defined." - metric.type_params.input_measures.append(conversion_type_params.base_measure) - metric.type_params.input_measures.append(conversion_type_params.conversion_measure) + metric.add_input_measure(conversion_type_params.base_measure) + metric.add_input_measure(conversion_type_params.conversion_measure) _process_metric_depends_on( manifest=manifest, current_project=current_project, metric=metric ) @@ -1618,7 +1618,8 @@ def _process_metric_node( _process_metric_node( manifest=manifest, current_project=current_project, metric=target_metric ) - metric.type_params.input_measures.extend(target_metric.type_params.input_measures) + for input_measure in target_metric.type_params.input_measures: + metric.add_input_measure(input_measure) metric.depends_on.add_node(target_metric.unique_id) else: assert_values_exhausted(metric.type) diff --git a/tests/functional/metrics/fixtures.py b/tests/functional/metrics/fixtures.py index 5a8373fbe5d..ab7cc0cc0e7 100644 --- a/tests/functional/metrics/fixtures.py +++ b/tests/functional/metrics/fixtures.py @@ -664,3 +664,56 @@ conversion_measure: num_orders entity: purchase """ + +filtered_metrics_yml = """ +version: 2 + +metrics: + + - name: collective_tenure_measure_filter_str + label: "Collective tenure1" + description: Total number of years of team experience + type: simple + type_params: + measure: + name: "years_tenure" + filter: "{{ Dimension('id__loves_dbt') }} is true" + + - name: collective_tenure_metric_filter_str + label: Collective tenure3 + description: Total number of years of team experience + type: simple + type_params: + measure: + name: "years_tenure" + filter: "{{ Dimension('id__loves_dbt') }} is true" + + + - name: average_tenure_filter_str + label: Average tenure of people who love dbt1 + description: Average tenure of people who love dbt + type: derived + type_params: + expr: "average_tenure" + metrics: + - name: average_tenure + filter: "{{ Dimension('id__loves_dbt') }} is true" +""" + +duplicate_measure_metric_yml = """ +metrics: + # Simple metrics + - name: people_with_tenure + description: "Count of people with tenure" + type: simple + label: People with tenure + type_params: + measure: people + - name: ratio_tenure_to_people + description: People to years of tenure + label: New customers to all customers + type: ratio + type_params: + numerator: people_with_tenure + denominator: number_of_people +""" diff --git a/tests/functional/metrics/test_metrics.py b/tests/functional/metrics/test_metrics.py index 70ebcfa35b8..22e8599f884 100644 --- a/tests/functional/metrics/test_metrics.py +++ b/tests/functional/metrics/test_metrics.py @@ -28,6 +28,8 @@ semantic_model_people_yml, semantic_model_purchasing_yml, purchasing_model_sql, + basic_metrics_yml, + duplicate_measure_metric_yml, ) @@ -76,7 +78,7 @@ def test_simple_metric( "metric.test.average_tenure_minus_people" ].type_params.input_measures ) - == 3 + == 2 ) @@ -399,3 +401,21 @@ def test_conversion_metric( ].type_params.conversion_type_params.entity == "purchase" ) + + +class TestDuplicateInputMeasures: + @pytest.fixture(scope="class") + def models(self): + return { + "basic_metrics.yml": basic_metrics_yml, + "filtered_metrics.yml": duplicate_measure_metric_yml, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_model_people.yml": semantic_model_people_yml, + "people.sql": models_people_sql, + } + + def test_duplicate_input_measures(self, project): + runner = dbtRunner() + result = runner.invoke(["parse"]) + assert result.success + assert isinstance(result.result, Manifest) From 3b524cad91ba10c51ffe853c4d04e31b85f285eb Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Thu, 14 Mar 2024 00:51:54 +0000 Subject: [PATCH 6/7] Bumping version to 1.7.10 and generate changelog --- .bumpversion.cfg | 2 +- .changes/1.7.10.md | 11 +++++++++++ .../unreleased/Dependencies-20240222-102947.yaml | 6 ------ .changes/unreleased/Fixes-20240226-173227.yaml | 6 ------ .changes/unreleased/Fixes-20240301-135536.yaml | 6 ------ .changes/unreleased/Fixes-20240307-142459.yaml | 6 ------ CHANGELOG.md | 15 ++++++++++++++- core/dbt/version.py | 2 +- core/setup.py | 2 +- docker/Dockerfile | 12 ++++++------ .../postgres/dbt/adapters/postgres/__version__.py | 2 +- plugins/postgres/setup.py | 2 +- tests/adapter/dbt/tests/adapter/__version__.py | 2 +- tests/adapter/setup.py | 2 +- 14 files changed, 38 insertions(+), 38 deletions(-) create mode 100644 .changes/1.7.10.md delete mode 100644 .changes/unreleased/Dependencies-20240222-102947.yaml delete mode 100644 .changes/unreleased/Fixes-20240226-173227.yaml delete mode 100644 .changes/unreleased/Fixes-20240301-135536.yaml delete mode 100644 .changes/unreleased/Fixes-20240307-142459.yaml diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8fb51b0e717..82f2fae4b2f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.7.9 +current_version = 1.7.10 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.7.10.md b/.changes/1.7.10.md new file mode 100644 index 00000000000..e9a47911665 --- /dev/null +++ b/.changes/1.7.10.md @@ -0,0 +1,11 @@ +## dbt-core 1.7.10 - March 14, 2024 + +### Fixes + +- Do not add duplicate input_measures ([#9360](https://github.com/dbt-labs/dbt-core/issues/9360)) +- Fix partial parsing `KeyError` on deleted schema files ([#8860](https://github.com/dbt-labs/dbt-core/issues/8860)) +- Support saved queries in `dbt list` ([#9532](https://github.com/dbt-labs/dbt-core/issues/9532)) + +### Dependencies + +- Restrict protobuf to 4.* versions ([#9566](https://github.com/dbt-labs/dbt-core/pull/9566)) diff --git a/.changes/unreleased/Dependencies-20240222-102947.yaml b/.changes/unreleased/Dependencies-20240222-102947.yaml deleted file mode 100644 index 9c21d51e21e..00000000000 --- a/.changes/unreleased/Dependencies-20240222-102947.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: Restrict protobuf to 4.* versions -time: 2024-02-22T10:29:47.595435-08:00 -custom: - Author: QMalcolm - PR: "9566" diff --git a/.changes/unreleased/Fixes-20240226-173227.yaml b/.changes/unreleased/Fixes-20240226-173227.yaml deleted file mode 100644 index fa1bf0ab8cf..00000000000 --- a/.changes/unreleased/Fixes-20240226-173227.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Do not add duplicate input_measures -time: 2024-02-26T17:32:27.837427-05:00 -custom: - Author: gshank - Issue: "9360" diff --git a/.changes/unreleased/Fixes-20240301-135536.yaml b/.changes/unreleased/Fixes-20240301-135536.yaml deleted file mode 100644 index 2a96bd7eeec..00000000000 --- a/.changes/unreleased/Fixes-20240301-135536.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix partial parsing `KeyError` on deleted schema files -time: 2024-03-01T13:55:36.533176-08:00 -custom: - Author: QMalcolm - Issue: "8860" diff --git a/.changes/unreleased/Fixes-20240307-142459.yaml b/.changes/unreleased/Fixes-20240307-142459.yaml deleted file mode 100644 index 14c08da2816..00000000000 --- a/.changes/unreleased/Fixes-20240307-142459.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Support saved queries in `dbt list` -time: 2024-03-07T14:24:59.530072-05:00 -custom: - Author: QMalcolm jtcohen6 - Issue: "9532" diff --git a/CHANGELOG.md b/CHANGELOG.md index baea75edc0d..7b0e9fcffb7 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-core 1.7.10 - March 14, 2024 + +### Fixes + +- Do not add duplicate input_measures ([#9360](https://github.com/dbt-labs/dbt-core/issues/9360)) +- Fix partial parsing `KeyError` on deleted schema files ([#8860](https://github.com/dbt-labs/dbt-core/issues/8860)) +- Support saved queries in `dbt list` ([#9532](https://github.com/dbt-labs/dbt-core/issues/9532)) + +### Dependencies + +- Restrict protobuf to 4.* versions ([#9566](https://github.com/dbt-labs/dbt-core/pull/9566)) + + + ## dbt-core 1.7.9 - February 28, 2024 ### Fixes @@ -24,7 +38,6 @@ ### Contributors - [@l1xnan](https://github.com/l1xnan) ([#9007](https://github.com/dbt-labs/dbt-core/issues/9007)) - ## dbt-core 1.7.8 - February 14, 2024 ### Fixes diff --git a/core/dbt/version.py b/core/dbt/version.py index 1799ed6b8c3..70342595b6c 100644 --- a/core/dbt/version.py +++ b/core/dbt/version.py @@ -232,5 +232,5 @@ def _get_adapter_plugin_names() -> Iterator[str]: yield plugin_name -__version__ = "1.7.9" +__version__ = "1.7.10" installed = get_installed_version() diff --git a/core/setup.py b/core/setup.py index f3f51ba5ad2..621bfc74bf7 100644 --- a/core/setup.py +++ b/core/setup.py @@ -25,7 +25,7 @@ package_name = "dbt-core" -package_version = "1.7.9" +package_version = "1.7.10" description = """With dbt, data analysts and engineers can build analytics \ the way engineers build applications.""" diff --git a/docker/Dockerfile b/docker/Dockerfile index ae9cb508628..a7c69121754 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,12 +16,12 @@ FROM --platform=$build_for python:3.10.7-slim-bullseye as base # N.B. The refs updated automagically every release via bumpversion # N.B. dbt-postgres is currently found in the core codebase so a value of dbt-core@ is correct -ARG dbt_core_ref=dbt-core@v1.7.9 -ARG dbt_postgres_ref=dbt-core@v1.7.9 -ARG dbt_redshift_ref=dbt-redshift@v1.7.9 -ARG dbt_bigquery_ref=dbt-bigquery@v1.7.9 -ARG dbt_snowflake_ref=dbt-snowflake@v1.7.9 -ARG dbt_spark_ref=dbt-spark@v1.7.9 +ARG dbt_core_ref=dbt-core@v1.7.10 +ARG dbt_postgres_ref=dbt-core@v1.7.10 +ARG dbt_redshift_ref=dbt-redshift@v1.7.10 +ARG dbt_bigquery_ref=dbt-bigquery@v1.7.10 +ARG dbt_snowflake_ref=dbt-snowflake@v1.7.10 +ARG dbt_spark_ref=dbt-spark@v1.7.10 # special case args ARG dbt_spark_version=all ARG dbt_third_party diff --git a/plugins/postgres/dbt/adapters/postgres/__version__.py b/plugins/postgres/dbt/adapters/postgres/__version__.py index 52a993a81b5..d32d8189efa 100644 --- a/plugins/postgres/dbt/adapters/postgres/__version__.py +++ b/plugins/postgres/dbt/adapters/postgres/__version__.py @@ -1 +1 @@ -version = "1.7.9" +version = "1.7.10" diff --git a/plugins/postgres/setup.py b/plugins/postgres/setup.py index b20f639ef77..506d33268a4 100644 --- a/plugins/postgres/setup.py +++ b/plugins/postgres/setup.py @@ -41,7 +41,7 @@ def _dbt_psycopg2_name(): package_name = "dbt-postgres" -package_version = "1.7.9" +package_version = "1.7.10" description = """The postgres adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/adapter/dbt/tests/adapter/__version__.py b/tests/adapter/dbt/tests/adapter/__version__.py index 52a993a81b5..d32d8189efa 100644 --- a/tests/adapter/dbt/tests/adapter/__version__.py +++ b/tests/adapter/dbt/tests/adapter/__version__.py @@ -1 +1 @@ -version = "1.7.9" +version = "1.7.10" diff --git a/tests/adapter/setup.py b/tests/adapter/setup.py index 99ef6f2e504..4705297ce52 100644 --- a/tests/adapter/setup.py +++ b/tests/adapter/setup.py @@ -20,7 +20,7 @@ package_name = "dbt-tests-adapter" -package_version = "1.7.9" +package_version = "1.7.10" description = """The dbt adapter tests for adapter plugins""" this_directory = os.path.abspath(os.path.dirname(__file__)) From a0372fc0780fc5d11a5d52b748d64970c9f169b5 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 20 Mar 2024 16:06:14 -0400 Subject: [PATCH 7/7] Handle exceptions during node execution more elegantly. (#9585) (#9778) * Handle exceptions during node execution more elegantly. * Add changelog entry. * Fix import * Add task documentation. * Change event type for noting thread exceptions. Co-authored-by: Peter Webb --- .../unreleased/Fixes-20240216-145632.yaml | 6 +++ core/dbt/task/README.md | 42 +++++++++++++++++++ core/dbt/task/runnable.py | 38 ++++++++++++++--- 3 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240216-145632.yaml diff --git a/.changes/unreleased/Fixes-20240216-145632.yaml b/.changes/unreleased/Fixes-20240216-145632.yaml new file mode 100644 index 00000000000..a02027f66a5 --- /dev/null +++ b/.changes/unreleased/Fixes-20240216-145632.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Tighten exception handling to avoid worker thread hangs. +time: 2024-02-16T14:56:32.858967-05:00 +custom: + Author: peterallenwebb + Issue: "9583" diff --git a/core/dbt/task/README.md b/core/dbt/task/README.md index 9de939e4cc4..2b32f5dbfa8 100644 --- a/core/dbt/task/README.md +++ b/core/dbt/task/README.md @@ -1 +1,43 @@ # Task README + +### Task Hierarchy +``` +BaseTask + ┣ CleanTask + ┣ ConfiguredTask + ┃ ┣ GraphRunnableTask + ┃ ┃ ┣ CloneTask + ┃ ┃ ┣ CompileTask + ┃ ┃ ┃ ┣ GenerateTask + ┃ ┃ ┃ ┣ RunTask + ┃ ┃ ┃ ┃ ┣ BuildTask + ┃ ┃ ┃ ┃ ┣ FreshnessTask + ┃ ┃ ┃ ┃ ┣ SeedTask + ┃ ┃ ┃ ┃ ┣ SnapshotTask + ┃ ┃ ┃ ┃ ┗ TestTask + ┃ ┃ ┃ ┗ ShowTask + ┃ ┃ ┗ ListTask + ┃ ┣ RetryTask + ┃ ┣ RunOperationTask + ┃ ┗ ServeTask + ┣ DebugTask + ┣ DepsTask + ┗ InitTask +``` + +### Runner Hierarchy +``` +BaseRunner + ┣ CloneRunner + ┣ CompileRunner + ┃ ┣ GenericSqlRunner + ┃ ┃ ┣ SqlCompileRunner + ┃ ┃ ┗ SqlExecuteRunner + ┃ ┣ ModelRunner + ┃ ┃ ┣ SeedRunner + ┃ ┃ ┗ SnapshotRunner + ┃ ┣ ShowRunner + ┃ ┗ TestRunner + ┣ FreshnessRunner + ┗ SavedQueryRunner +``` diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index 242e155a4d7..e9ea21f661d 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -35,6 +35,7 @@ ConcurrencyLine, EndRunResult, NothingToDo, + GenericExceptionOnRun, ) from dbt.exceptions import ( DbtInternalError, @@ -204,17 +205,44 @@ def call_runner(self, runner: BaseRunner) -> RunResult: ) ) status: Dict[str, str] = {} + result = None + thread_exception = None try: result = runner.run_with_hooks(self.manifest) + except Exception as e: + thread_exception = e finally: finishctx = TimestampNamed("finished_at") with finishctx, DbtModelState(status): - fire_event( - NodeFinished( - node_info=runner.node.node_info, - run_result=result.to_msg_dict(), + if result is not None: + fire_event( + NodeFinished( + node_info=runner.node.node_info, + run_result=result.to_msg_dict(), + ) ) - ) + else: + msg = f"Exception on worker thread. {thread_exception}" + + fire_event( + GenericExceptionOnRun( + unique_id=runner.node.unique_id, + exc=str(thread_exception), + node_info=runner.node.node_info, + ) + ) + + result = RunResult( + status=RunStatus.Error, # type: ignore + timing=[], + thread_id="", + execution_time=0.0, + adapter_response={}, + message=msg, + failures=None, + node=runner.node, + ) + # `_event_status` dict is only used for logging. Make sure # it gets deleted when we're done with it runner.node.clear_event_status()