Skip to content

Commit

Permalink
ADAP-1017: Fix configuration change monitoring for scenarios with no …
Browse files Browse the repository at this point in the history
…changes (#1009)

* changie
* add a test demonstrating the issue
* check that the change collection has changes instead of checking that it exists
* remove the partition config value from the config change since it is not hashable
  • Loading branch information
mikealfare authored Nov 8, 2023
1 parent 548d532 commit 64e042a
Show file tree
Hide file tree
Showing 5 changed files with 46 additions and 3 deletions.
7 changes: 7 additions & 0 deletions .changes/unreleased/Fixes-20231107-174352.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
kind: Fixes
body: Fixed issue where materialized views were failing on re-run with minimal config
parameters
time: 2023-11-07T17:43:52.972135-05:00
custom:
Author: "mikealfare"
Issue: "1007"
5 changes: 3 additions & 2 deletions dbt/adapters/bigquery/relation.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,10 @@ def materialized_view_config_changeset(
)

if new_materialized_view.partition != existing_materialized_view.partition:
# the existing PartitionConfig is not hashable, but since we need to do
# a full refresh either way, we don't need to provide a context
config_change_collection.partition = BigQueryPartitionConfigChange(
action=RelationConfigChangeAction.alter,
context=new_materialized_view.partition,
)

if new_materialized_view.cluster != existing_materialized_view.cluster:
Expand All @@ -95,7 +96,7 @@ def materialized_view_config_changeset(
context=new_materialized_view.cluster,
)

if config_change_collection:
if config_change_collection.has_changes:
return config_change_collection
return None

Expand Down
2 changes: 1 addition & 1 deletion dbt/adapters/bigquery/relation_configs/_partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ def parse_bq_table(cls, table: BigQueryTable) -> Dict[str, Any]:

@dataclass(frozen=True, eq=True, unsafe_hash=True)
class BigQueryPartitionConfigChange(RelationConfigChange):
context: Optional[PartitionConfig]
context: Optional[Any] = None

@property
def requires_full_refresh(self) -> bool:
Expand Down
11 changes: 11 additions & 0 deletions tests/functional/adapter/materialized_view_tests/_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,14 @@
record_valid_date
from {{ ref('my_seed') }}
"""


MY_MINIMAL_MATERIALIZED_VIEW = """
{{
config(
materialized = 'materialized_view',
)
}}
select * from {{ ref('my_seed') }}
"""
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from dbt.tests.adapter.materialized_view.basic import MaterializedViewBasic

from tests.functional.adapter.materialized_view_tests._mixin import BigQueryMaterializedViewMixin
from tests.functional.adapter.materialized_view_tests import _files


class TestBigqueryMaterializedViewsBasic(BigQueryMaterializedViewMixin, MaterializedViewBasic):
Expand All @@ -27,3 +28,26 @@ def test_materialized_view_only_updates_after_refresh(
self, project, my_materialized_view, my_seed
):
pass


class TestMaterializedViewRerun:
"""
This addresses: https://github.com/dbt-labs/dbt-bigquery/issues/1007
This effectively tests that defaults get properly set so that the run is idempotent.
If the defaults are not properly set, changes could appear when there are no changes
and cause unexpected scenarios.
"""

@pytest.fixture(scope="class", autouse=True)
def models(self):
return {"my_minimal_materialized_view.sql": _files.MY_MINIMAL_MATERIALIZED_VIEW}

@pytest.fixture(scope="class", autouse=True)
def seeds(self):
return {"my_seed.csv": _files.MY_SEED}

def test_minimal_materialized_view_is_idempotent(self, project):
run_dbt(["seed"])
run_dbt(["run"])
run_dbt(["run"])

0 comments on commit 64e042a

Please sign in to comment.