From 4a20a71e3f93133cdc161b333ca7abef70c4d3e1 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 4 Dec 2023 10:38:46 -0500 Subject: [PATCH 01/17] Initial addition of FixtureSourceFile --- core/dbt/contracts/files.py | 7 ++++++- core/dbt/parser/read_files.py | 14 +++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/core/dbt/contracts/files.py b/core/dbt/contracts/files.py index e85a8d91b5e..8d10b595350 100644 --- a/core/dbt/contracts/files.py +++ b/core/dbt/contracts/files.py @@ -22,6 +22,7 @@ class ParseFileType(StrEnum): Documentation = "docs" Schema = "schema" Hook = "hook" # not a real filetype, from dbt_project.yml + Fixture = "fixture" parse_file_type_to_parser = { @@ -35,6 +36,7 @@ class ParseFileType(StrEnum): ParseFileType.Documentation: "DocumentationParser", ParseFileType.Schema: "SchemaParser", ParseFileType.Hook: "HookParser", + ParseFileType.Fixture: None, } @@ -152,7 +154,6 @@ class BaseSourceFile(dbtClassMixin, SerializableType): parse_file_type: Optional[ParseFileType] = None # we don't want to serialize this contents: Optional[str] = None - # the unique IDs contained in this file @property def file_id(self): @@ -328,4 +329,8 @@ def delete_from_env_vars(self, yaml_key, name): del self.env_vars[yaml_key] +class FixtureSourceFile(BaseSourceFile): + unit_tests: List[str] = field(default_factory=list) + + AnySourceFile = Union[SchemaSourceFile, SourceFile] diff --git a/core/dbt/parser/read_files.py b/core/dbt/parser/read_files.py index d07c7fb1d48..c032d0183e0 100644 --- a/core/dbt/parser/read_files.py +++ b/core/dbt/parser/read_files.py @@ -10,6 +10,7 @@ FileHash, AnySourceFile, SchemaSourceFile, + FixtureSourceFile, ) from dbt.config import Project from dbt.dataclass_schema import dbtClassMixin @@ -46,7 +47,13 @@ def load_source_file( saved_files, ) -> Optional[AnySourceFile]: - sf_cls = SchemaSourceFile if parse_file_type == ParseFileType.Schema else SourceFile + if parse_file_type == ParseFileType.Schema: + sf_cls = SchemaSourceFile + elif parse_file_type == ParseFileType.Fixture: + sf_cls = FixtureSourceFile # type:ignore[assignment] + else: + sf_cls = SourceFile # type:ignore[assignment] + source_file = sf_cls( path=path, checksum=FileHash.empty(), @@ -422,5 +429,10 @@ def get_file_types_for_project(project): "extensions": [".yml", ".yaml"], "parser": "SchemaParser", }, + ParseFileType.Fixture: { + "paths": project.fixture_paths, + "extensions": [".csv"], + "parser": "FixtureParser", + }, } return file_types From d45a9ad5d1f8004c8caac3ae5d30151f17b0cafd Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 4 Dec 2023 11:27:43 -0500 Subject: [PATCH 02/17] Add UnitTestFixture node type --- core/dbt/contracts/files.py | 4 ++++ core/dbt/contracts/graph/manifest.py | 3 ++- core/dbt/contracts/graph/nodes.py | 7 +++++++ core/dbt/graph/selector_methods.py | 3 +++ core/dbt/node_types.py | 1 + 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/core/dbt/contracts/files.py b/core/dbt/contracts/files.py index 8d10b595350..312bf6fca75 100644 --- a/core/dbt/contracts/files.py +++ b/core/dbt/contracts/files.py @@ -332,5 +332,9 @@ def delete_from_env_vars(self, yaml_key, name): class FixtureSourceFile(BaseSourceFile): unit_tests: List[str] = field(default_factory=list) + def add_unit_test(self, value): + if value not in self.unit_tests: + self.unit_tests.append(value) + AnySourceFile = Union[SchemaSourceFile, SourceFile] diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index fed87f0244b..324eb0380ec 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -43,6 +43,7 @@ SourceDefinition, UnpatchedSourceDefinition, UnitTestDefinition, + UnitTestDef, ) from dbt.contracts.graph.unparsed import SourcePatch, NodeVersion, UnparsedVersion from dbt.contracts.graph.manifest_upgrade import upgrade_manifest_json @@ -1620,7 +1621,7 @@ class WritableManifest(ArtifactMixin): description="Metadata about the manifest", ) ) - unit_tests: Mapping[UniqueID, UnitTestDefinition] = field( + unit_tests: Mapping[UniqueID, UnitTestDef] = field( metadata=dict( description="The unit tests defined in the project", ) diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index d19a59353a2..040c7f9593a 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1134,6 +1134,11 @@ def same_contents(self, other: Optional["UnitTestDefinition"]) -> bool: return self.checksum == other.checksum +class UnitTestFixture(BaseNode): + resource_type: Literal[NodeType.Fixture] + rows: Optional[List[Dict[str, Any]]] = None + + # ==================================== # Snapshot node # ==================================== @@ -1966,3 +1971,5 @@ class ParsedMacroPatch(ParsedPatch): ] TestNode = Union[SingularTestNode, GenericTestNode] + +UnitTestDef = Union[UnitTestDefinition, UnitTestFixture] diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index ba931c65dd9..644e63dd47b 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -155,6 +155,8 @@ def unit_tests( self, included_nodes: Set[UniqueId] ) -> Iterator[Tuple[UniqueId, UnitTestDefinition]]: for unique_id, unit_test in self.manifest.unit_tests.items(): + if unit_test.resource_type != NodeType.Unit: + next unique_id = UniqueId(unique_id) if unique_id not in included_nodes: continue @@ -741,6 +743,7 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu elif node in manifest.semantic_models: previous_node = manifest.semantic_models[node] elif node in manifest.unit_tests: + assert isinstance(node, UnitTestDefinition) previous_node = manifest.unit_tests[node] keyword_args = {} diff --git a/core/dbt/node_types.py b/core/dbt/node_types.py index bb35993b8d7..542357913f2 100644 --- a/core/dbt/node_types.py +++ b/core/dbt/node_types.py @@ -36,6 +36,7 @@ class NodeType(StrEnum): SavedQuery = "saved_query" SemanticModel = "semantic_model" Unit = "unit_test" + Fixture = "fixture" @classmethod def executable(cls) -> List["NodeType"]: From c266ca756228e3eae581c0e75b92a3fabb33e97b Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 4 Dec 2023 13:04:28 -0500 Subject: [PATCH 03/17] Add fixture parser and put fixture node-like objects in manifest.unit_tests --- core/dbt/compilation.py | 2 ++ core/dbt/contracts/graph/manifest.py | 18 ++++++++++---- core/dbt/graph/selector.py | 3 ++- core/dbt/graph/selector_methods.py | 32 ++++++++++++------------- core/dbt/parser/fixtures.py | 35 ++++++++++++++++++++++++++++ core/dbt/parser/manifest.py | 2 ++ core/dbt/parser/unit_tests.py | 2 +- tests/unit/test_node_types.py | 1 + 8 files changed, 72 insertions(+), 23 deletions(-) create mode 100644 core/dbt/parser/fixtures.py diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index affc47bbd7f..e0b703b3ba7 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -199,6 +199,8 @@ def link_graph(self, manifest: Manifest): for metric in manifest.metrics.values(): self.link_node(metric, manifest) for unit_test in manifest.unit_tests.values(): + if not isinstance(unit_test, UnitTestDefinition): + continue self.link_node(unit_test, manifest) for saved_query in manifest.saved_queries.values(): self.link_node(saved_query, manifest) diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 324eb0380ec..435513b4bca 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -42,12 +42,17 @@ SemanticModel, SourceDefinition, UnpatchedSourceDefinition, - UnitTestDefinition, UnitTestDef, ) from dbt.contracts.graph.unparsed import SourcePatch, NodeVersion, UnparsedVersion from dbt.contracts.graph.manifest_upgrade import upgrade_manifest_json -from dbt.contracts.files import SourceFile, SchemaSourceFile, FileHash, AnySourceFile +from dbt.contracts.files import ( + SourceFile, + SchemaSourceFile, + FileHash, + AnySourceFile, + FixtureSourceFile, +) from dbt.contracts.util import ( BaseArtifactMetadata, SourceKey, @@ -801,7 +806,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin): disabled: MutableMapping[str, List[GraphMemberNode]] = field(default_factory=dict) env_vars: MutableMapping[str, str] = field(default_factory=dict) semantic_models: MutableMapping[str, SemanticModel] = field(default_factory=dict) - unit_tests: MutableMapping[str, UnitTestDefinition] = field(default_factory=dict) + unit_tests: MutableMapping[str, UnitTestDef] = field(default_factory=dict) saved_queries: MutableMapping[str, SavedQuery] = field(default_factory=dict) _doc_lookup: Optional[DocLookup] = field( @@ -1055,7 +1060,8 @@ def expect(self, unique_id: str) -> GraphMemberNode: elif unique_id in self.semantic_models: return self.semantic_models[unique_id] elif unique_id in self.unit_tests: - return self.unit_tests[unique_id] + # This should only be UnitTestDefinition, not UnitTestFixture + return self.unit_tests[unique_id] # type:ignore[return-value] elif unique_id in self.saved_queries: return self.saved_queries[unique_id] else: @@ -1500,7 +1506,9 @@ def add_semantic_model(self, source_file: SchemaSourceFile, semantic_model: Sema self.semantic_models[semantic_model.unique_id] = semantic_model source_file.semantic_models.append(semantic_model.unique_id) - def add_unit_test(self, source_file: SchemaSourceFile, unit_test: UnitTestDefinition): + def add_unit_test( + self, source_file: Union[SchemaSourceFile, FixtureSourceFile], unit_test: UnitTestDef + ): if unit_test.unique_id in self.unit_tests: raise DuplicateResourceNameError(unit_test, self.unit_tests[unit_test.unique_id]) self.unit_tests[unit_test.unique_id] = unit_test diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index bcaaa5514f6..f2fb3d9d92a 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -205,7 +205,8 @@ def _is_match(self, unique_id: UniqueId) -> bool: elif unique_id in self.manifest.semantic_models: node = self.manifest.semantic_models[unique_id] elif unique_id in self.manifest.unit_tests: - node = self.manifest.unit_tests[unique_id] + # This should only be a UnitTestDefinition, not a UnitTestFixture + node = self.manifest.unit_tests[unique_id] # type:ignore[assignment] elif unique_id in self.manifest.saved_queries: node = self.manifest.saved_queries[unique_id] else: diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index 644e63dd47b..40fcc78ed9d 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -160,6 +160,7 @@ def unit_tests( unique_id = UniqueId(unique_id) if unique_id not in included_nodes: continue + assert isinstance(unit_test, UnitTestDefinition) yield unique_id, unit_test def parsed_and_unit_nodes(self, included_nodes: Set[UniqueId]): @@ -729,22 +730,21 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu manifest: WritableManifest = self.previous_state.manifest - for node, real_node in self.all_nodes(included_nodes): + for unique_id, node in self.all_nodes(included_nodes): previous_node: Optional[SelectorTarget] = None - if node in manifest.nodes: - previous_node = manifest.nodes[node] - elif node in manifest.sources: - previous_node = manifest.sources[node] - elif node in manifest.exposures: - previous_node = manifest.exposures[node] - elif node in manifest.metrics: - previous_node = manifest.metrics[node] - elif node in manifest.semantic_models: - previous_node = manifest.semantic_models[node] - elif node in manifest.unit_tests: - assert isinstance(node, UnitTestDefinition) - previous_node = manifest.unit_tests[node] + if unique_id in manifest.nodes: + previous_node = manifest.nodes[unique_id] + elif unique_id in manifest.sources: + previous_node = manifest.sources[unique_id] + elif unique_id in manifest.exposures: + previous_node = manifest.exposures[unique_id] + elif unique_id in manifest.metrics: + previous_node = manifest.metrics[unique_id] + elif unique_id in manifest.semantic_models: + previous_node = manifest.semantic_models[unique_id] + elif unique_id in manifest.unit_tests: + previous_node = manifest.unit_tests[unique_id] keyword_args = {} if checker.__name__ in [ @@ -754,8 +754,8 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu ]: keyword_args["adapter_type"] = adapter_type # type: ignore - if checker(previous_node, real_node, **keyword_args): # type: ignore - yield node + if checker(previous_node, node, **keyword_args): # type: ignore + yield unique_id class ResultSelectorMethod(SelectorMethod): diff --git a/core/dbt/parser/fixtures.py b/core/dbt/parser/fixtures.py new file mode 100644 index 00000000000..d58df69b96e --- /dev/null +++ b/core/dbt/parser/fixtures.py @@ -0,0 +1,35 @@ +from typing import Optional + +from dbt.contracts.files import FixtureSourceFile +from dbt.contracts.graph.nodes import UnitTestFixture +from dbt.node_types import NodeType +from dbt.parser.base import Parser +from dbt.parser.search import FileBlock + + +class FixtureParser(Parser[UnitTestFixture]): + @property + def resource_type(self) -> NodeType: + return NodeType.Fixture + + @classmethod + def get_compiled_path(cls, block: FileBlock): + # Is this necessary? + return block.path.relative_path + + def generate_unique_id(self, resource_name: str, _: Optional[str] = None) -> str: + return f"fixture.{self.project.project_name}.{resource_name}" + + def parse_file(self, file_block: FileBlock): + assert isinstance(file_block.file, FixtureSourceFile) + unique_id = self.generate_unique_id(file_block.name) + + fixture = UnitTestFixture( + name=file_block.name, + path=file_block.file.path.relative_path, + original_file_path=file_block.path.original_file_path, + package_name=self.project.project_name, + unique_id=unique_id, + resource_type=NodeType.Fixture, + ) + self.manifest.add_unit_test(file_block.file, fixture) diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index f0da24e630f..e65dea6a65a 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -114,6 +114,7 @@ from dbt.parser.generic_test import GenericTestParser from dbt.parser.singular_test import SingularTestParser from dbt.parser.docs import DocumentationParser +from dbt.parser.fixtures import FixtureParser from dbt.parser.hooks import HookParser from dbt.parser.macros import MacroParser from dbt.parser.models import ModelParser @@ -471,6 +472,7 @@ def load(self) -> Manifest: SeedParser, DocumentationParser, HookParser, + FixtureParser, ] for project in self.all_projects.values(): if project.project_name not in project_parser_files: diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 0f636b3f874..48770b16eae 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -47,7 +47,7 @@ def __init__(self, manifest, root_project, selected) -> None: def load(self) -> Manifest: for unique_id in self.selected: if unique_id in self.manifest.unit_tests: - unit_test_case = self.manifest.unit_tests[unique_id] + unit_test_case: UnitTestDefinition = self.manifest.unit_tests[unique_id] # type: ignore[assignment] self.parse_unit_test_case(unit_test_case) return self.unit_test_manifest diff --git a/tests/unit/test_node_types.py b/tests/unit/test_node_types.py index f07022ff2ef..e9b5d1bae82 100644 --- a/tests/unit/test_node_types.py +++ b/tests/unit/test_node_types.py @@ -19,6 +19,7 @@ NodeType.SemanticModel: "semantic_models", NodeType.Unit: "unit_tests", NodeType.SavedQuery: "saved_queries", + NodeType.Fixture: "fixtures", } From 4f1e2c7c45c1836f8e8d45eeb5c5727919ec2678 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 5 Dec 2023 12:14:15 -0500 Subject: [PATCH 04/17] Put fixtures in manifest.fixtures dictionary. --- core/dbt/compilation.py | 2 -- core/dbt/contracts/files.py | 3 ++- core/dbt/contracts/graph/manifest.py | 21 ++++++++++++++------ core/dbt/contracts/graph/nodes.py | 2 -- core/dbt/graph/selector.py | 3 +-- core/dbt/graph/selector_methods.py | 3 --- core/dbt/parser/fixtures.py | 2 +- core/dbt/parser/unit_tests.py | 29 +++++++++++++++++++++++++++- 8 files changed, 47 insertions(+), 18 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index e0b703b3ba7..affc47bbd7f 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -199,8 +199,6 @@ def link_graph(self, manifest: Manifest): for metric in manifest.metrics.values(): self.link_node(metric, manifest) for unit_test in manifest.unit_tests.values(): - if not isinstance(unit_test, UnitTestDefinition): - continue self.link_node(unit_test, manifest) for saved_query in manifest.saved_queries.values(): self.link_node(saved_query, manifest) diff --git a/core/dbt/contracts/files.py b/core/dbt/contracts/files.py index 312bf6fca75..e9ca543b1ba 100644 --- a/core/dbt/contracts/files.py +++ b/core/dbt/contracts/files.py @@ -36,7 +36,7 @@ class ParseFileType(StrEnum): ParseFileType.Documentation: "DocumentationParser", ParseFileType.Schema: "SchemaParser", ParseFileType.Hook: "HookParser", - ParseFileType.Fixture: None, + ParseFileType.Fixture: "FixtureParser", } @@ -330,6 +330,7 @@ def delete_from_env_vars(self, yaml_key, name): class FixtureSourceFile(BaseSourceFile): + fixture: Optional[str] = None unit_tests: List[str] = field(default_factory=list) def add_unit_test(self, value): diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 435513b4bca..c30b23a9278 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -42,7 +42,8 @@ SemanticModel, SourceDefinition, UnpatchedSourceDefinition, - UnitTestDef, + UnitTestDefinition, + UnitTestFixture, ) from dbt.contracts.graph.unparsed import SourcePatch, NodeVersion, UnparsedVersion from dbt.contracts.graph.manifest_upgrade import upgrade_manifest_json @@ -806,8 +807,9 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin): disabled: MutableMapping[str, List[GraphMemberNode]] = field(default_factory=dict) env_vars: MutableMapping[str, str] = field(default_factory=dict) semantic_models: MutableMapping[str, SemanticModel] = field(default_factory=dict) - unit_tests: MutableMapping[str, UnitTestDef] = field(default_factory=dict) + unit_tests: MutableMapping[str, UnitTestDefinition] = field(default_factory=dict) saved_queries: MutableMapping[str, SavedQuery] = field(default_factory=dict) + fixtures: MutableMapping[str, UnitTestFixture] = field(default_factory=dict) _doc_lookup: Optional[DocLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} @@ -1060,8 +1062,7 @@ def expect(self, unique_id: str) -> GraphMemberNode: elif unique_id in self.semantic_models: return self.semantic_models[unique_id] elif unique_id in self.unit_tests: - # This should only be UnitTestDefinition, not UnitTestFixture - return self.unit_tests[unique_id] # type:ignore[return-value] + return self.unit_tests[unique_id] elif unique_id in self.saved_queries: return self.saved_queries[unique_id] else: @@ -1507,13 +1508,21 @@ def add_semantic_model(self, source_file: SchemaSourceFile, semantic_model: Sema source_file.semantic_models.append(semantic_model.unique_id) def add_unit_test( - self, source_file: Union[SchemaSourceFile, FixtureSourceFile], unit_test: UnitTestDef + self, + source_file: Union[SchemaSourceFile, FixtureSourceFile], + unit_test: UnitTestDefinition, ): if unit_test.unique_id in self.unit_tests: raise DuplicateResourceNameError(unit_test, self.unit_tests[unit_test.unique_id]) self.unit_tests[unit_test.unique_id] = unit_test source_file.unit_tests.append(unit_test.unique_id) + def add_fixture(self, source_file: FixtureSourceFile, fixture: UnitTestFixture): + if fixture.unique_id in self.fixtures: + raise DuplicateResourceNameError(fixture, self.fixtures[fixture.unique_id]) + self.fixtures[fixture.unique_id] = fixture + source_file.fixture = fixture.unique_id + def add_saved_query(self, source_file: SchemaSourceFile, saved_query: SavedQuery) -> None: _check_duplicates(saved_query, self.saved_queries) self.saved_queries[saved_query.unique_id] = saved_query @@ -1629,7 +1638,7 @@ class WritableManifest(ArtifactMixin): description="Metadata about the manifest", ) ) - unit_tests: Mapping[UniqueID, UnitTestDef] = field( + unit_tests: Mapping[UniqueID, UnitTestDefinition] = field( metadata=dict( description="The unit tests defined in the project", ) diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 040c7f9593a..ee3a84b2753 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1971,5 +1971,3 @@ class ParsedMacroPatch(ParsedPatch): ] TestNode = Union[SingularTestNode, GenericTestNode] - -UnitTestDef = Union[UnitTestDefinition, UnitTestFixture] diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index f2fb3d9d92a..bcaaa5514f6 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -205,8 +205,7 @@ def _is_match(self, unique_id: UniqueId) -> bool: elif unique_id in self.manifest.semantic_models: node = self.manifest.semantic_models[unique_id] elif unique_id in self.manifest.unit_tests: - # This should only be a UnitTestDefinition, not a UnitTestFixture - node = self.manifest.unit_tests[unique_id] # type:ignore[assignment] + node = self.manifest.unit_tests[unique_id] elif unique_id in self.manifest.saved_queries: node = self.manifest.saved_queries[unique_id] else: diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index 40fcc78ed9d..6c2d521ad6e 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -155,12 +155,9 @@ def unit_tests( self, included_nodes: Set[UniqueId] ) -> Iterator[Tuple[UniqueId, UnitTestDefinition]]: for unique_id, unit_test in self.manifest.unit_tests.items(): - if unit_test.resource_type != NodeType.Unit: - next unique_id = UniqueId(unique_id) if unique_id not in included_nodes: continue - assert isinstance(unit_test, UnitTestDefinition) yield unique_id, unit_test def parsed_and_unit_nodes(self, included_nodes: Set[UniqueId]): diff --git a/core/dbt/parser/fixtures.py b/core/dbt/parser/fixtures.py index d58df69b96e..191c93b2789 100644 --- a/core/dbt/parser/fixtures.py +++ b/core/dbt/parser/fixtures.py @@ -32,4 +32,4 @@ def parse_file(self, file_block: FileBlock): unique_id=unique_id, resource_type=NodeType.Fixture, ) - self.manifest.add_unit_test(file_block.file, fixture) + self.manifest.add_fixture(file_block.file, fixture) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 48770b16eae..cd60c335bb9 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -47,7 +47,7 @@ def __init__(self, manifest, root_project, selected) -> None: def load(self) -> Manifest: for unique_id in self.selected: if unique_id in self.manifest.unit_tests: - unit_test_case: UnitTestDefinition = self.manifest.unit_tests[unique_id] # type: ignore[assignment] + unit_test_case: UnitTestDefinition = self.manifest.unit_tests[unique_id] self.parse_unit_test_case(unit_test_case) return self.unit_test_manifest @@ -264,6 +264,9 @@ def parse(self) -> ParseResult: config=unit_test_config, schema=tested_model_node.schema, ) + # For partial parsing, we add the unique_id of the unit test definition to the + # fixture file records + self._add_unit_test_to_fixture_files(unit_test_definition) # for calculating state:modified unit_test_definition.build_unit_test_checksum( self.schema_parser.project.project_root, self.schema_parser.project.fixture_paths @@ -272,6 +275,30 @@ def parse(self) -> ParseResult: return ParseResult() + def _add_unit_test_to_fixture_files(self, unit_test_definition): + for given in unit_test_definition.given: + if given.fixture: + # find fixture file object and store unit_test_definition unique_id + fixture_source_file = self.get_fixture_source_file( + given.fixture, self.project.project_name + ) + fixture_source_file.unit_tests.append(unit_test_definition.unique_id) + if unit_test_definition.expect.fixture: + # find fixture file object and store unit_test_definition unique_id + fixture_source_file = self.get_fixture_source_file( + unit_test_definition.expect.fixture, self.project.project_name + ) + fixture_source_file.unit_tests.append(unit_test_definition.unique_id) + + def get_fixture_source_file(self, fixture_name: str, project_name: str): + fixture_unique_id = f"fixture.{project_name}.{fixture_name}" + if fixture_unique_id in self.manifest.fixtures: + fixture = self.manifest.fixture[fixture_unique_id] + fixture_source_file = self.manifest.files[fixture.file_id] + return fixture_source_file + else: + raise ParsingError(f"Fixture file not found: {fixture_unique_id}") + def _get_unit_test(self, data: Dict[str, Any]) -> UnparsedUnitTest: try: UnparsedUnitTest.validate(data) From 4f4a185ebc5f0b2cf5900a5240d3e875e6bf965e Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 5 Dec 2023 12:58:00 -0500 Subject: [PATCH 05/17] Added tests of FixtureSourceFile. Needs mypy cleanup of files dict --- core/dbt/contracts/files.py | 3 +++ core/dbt/contracts/graph/manifest.py | 4 +++- core/dbt/parser/partial.py | 2 ++ core/dbt/parser/unit_tests.py | 2 +- core/dbt/task/parse.py | 0 tests/functional/unit_testing/test_csv_fixtures.py | 7 +++++++ 6 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 core/dbt/task/parse.py diff --git a/core/dbt/contracts/files.py b/core/dbt/contracts/files.py index e9ca543b1ba..a0aa5c987b2 100644 --- a/core/dbt/contracts/files.py +++ b/core/dbt/contracts/files.py @@ -169,6 +169,8 @@ def _serialize(self): def _deserialize(cls, dct: Dict[str, int]): if dct["parse_file_type"] == "schema": sf = SchemaSourceFile.from_dict(dct) + elif dct["parse_file_type"] == "fixture": + sf = FixtureSourceFile.from_dict(dct) else: sf = SourceFile.from_dict(dct) return sf @@ -329,6 +331,7 @@ def delete_from_env_vars(self, yaml_key, name): del self.env_vars[yaml_key] +@dataclass class FixtureSourceFile(BaseSourceFile): fixture: Optional[str] = None unit_tests: List[str] = field(default_factory=list) diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index c30b23a9278..439f86a99b5 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -799,7 +799,9 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin): metrics: MutableMapping[str, Metric] = field(default_factory=dict) groups: MutableMapping[str, Group] = field(default_factory=dict) selectors: MutableMapping[str, Any] = field(default_factory=dict) - files: MutableMapping[str, AnySourceFile] = field(default_factory=dict) + files: MutableMapping[str, Union[AnySourceFile, FixtureSourceFile]] = field( + default_factory=dict + ) metadata: ManifestMetadata = field(default_factory=ManifestMetadata) flat_graph: Dict[str, Any] = field(default_factory=dict) state_check: ManifestStateCheck = field(default_factory=ManifestStateCheck) diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py index 54ed2a732d0..532431ccd08 100644 --- a/core/dbt/parser/partial.py +++ b/core/dbt/parser/partial.py @@ -1021,6 +1021,8 @@ def build_env_vars_to_files(self): # Create a list of file_ids for source_files that need to be reparsed, and # a dictionary of file_ids to yaml_keys to names. for source_file in self.saved_files.values(): + if source_file.parse_file_type == ParseFileType.Fixture: + continue file_id = source_file.file_id if not source_file.env_vars: continue diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index cd60c335bb9..88575405d2f 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -293,7 +293,7 @@ def _add_unit_test_to_fixture_files(self, unit_test_definition): def get_fixture_source_file(self, fixture_name: str, project_name: str): fixture_unique_id = f"fixture.{project_name}.{fixture_name}" if fixture_unique_id in self.manifest.fixtures: - fixture = self.manifest.fixture[fixture_unique_id] + fixture = self.manifest.fixtures[fixture_unique_id] fixture_source_file = self.manifest.files[fixture.file_id] return fixture_source_file else: diff --git a/core/dbt/task/parse.py b/core/dbt/task/parse.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/functional/unit_testing/test_csv_fixtures.py b/tests/functional/unit_testing/test_csv_fixtures.py index f639f48331f..83ebf5f4d96 100644 --- a/tests/functional/unit_testing/test_csv_fixtures.py +++ b/tests/functional/unit_testing/test_csv_fixtures.py @@ -90,6 +90,13 @@ def test_unit_test(self, project): results = run_dbt(["run"]) assert len(results) == 3 + manifest = run_dbt(["parse"]) # Note: this manifest is deserialized from msgpack + fixture_source_file = manifest.files["test://tests/fixtures/test_my_model_a_fixture.csv"] + assert fixture_source_file.fixture == "fixture.test.test_my_model_a_fixture" + assert fixture_source_file.unit_tests == [ + "unit_test.test.my_model.test_my_model_string_concat" + ] + # Select by model name results = run_dbt(["test", "--select", "my_model"], expect_pass=False) assert len(results) == 5 From eded6a41afa94f4cfa3d2858cac3136b065a0c67 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 5 Dec 2023 13:28:26 -0500 Subject: [PATCH 06/17] Make mypy happy --- core/dbt/contracts/files.py | 2 +- core/dbt/contracts/graph/manifest.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/dbt/contracts/files.py b/core/dbt/contracts/files.py index a0aa5c987b2..271f9d0f9df 100644 --- a/core/dbt/contracts/files.py +++ b/core/dbt/contracts/files.py @@ -341,4 +341,4 @@ def add_unit_test(self, value): self.unit_tests.append(value) -AnySourceFile = Union[SchemaSourceFile, SourceFile] +AnySourceFile = Union[SchemaSourceFile, SourceFile, FixtureSourceFile] diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 439f86a99b5..85a7c4c8f78 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -799,9 +799,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin): metrics: MutableMapping[str, Metric] = field(default_factory=dict) groups: MutableMapping[str, Group] = field(default_factory=dict) selectors: MutableMapping[str, Any] = field(default_factory=dict) - files: MutableMapping[str, Union[AnySourceFile, FixtureSourceFile]] = field( - default_factory=dict - ) + files: MutableMapping[str, Union[AnySourceFile]] = field(default_factory=dict) metadata: ManifestMetadata = field(default_factory=ManifestMetadata) flat_graph: Dict[str, Any] = field(default_factory=dict) state_check: ManifestStateCheck = field(default_factory=ManifestStateCheck) @@ -1454,6 +1452,8 @@ def add_node(self, source_file: AnySourceFile, node: ManifestNode, test_from=Non source_file.exposures.append(node.unique_id) if isinstance(node, Group): source_file.groups.append(node.unique_id) + elif isinstance(source_file, FixtureSourceFile): + pass else: source_file.nodes.append(node.unique_id) @@ -1496,6 +1496,8 @@ def add_disabled(self, source_file: AnySourceFile, node: ResultNode, test_from=N source_file.semantic_models.append(node.unique_id) if isinstance(node, Exposure): source_file.exposures.append(node.unique_id) + elif isinstance(source_file, FixtureSourceFile): + pass else: source_file.nodes.append(node.unique_id) From 03911b891f57730cf46e68db3c04384361519ffc Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 5 Dec 2023 13:47:38 -0500 Subject: [PATCH 07/17] Fix up some tests --- core/dbt/contracts/graph/manifest.py | 8 ++------ core/dbt/parser/unit_tests.py | 7 +++++-- .../unit_testing/test_csv_fixtures.py | 20 +++++-------------- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 85a7c4c8f78..c47cb4aace2 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -799,7 +799,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin): metrics: MutableMapping[str, Metric] = field(default_factory=dict) groups: MutableMapping[str, Group] = field(default_factory=dict) selectors: MutableMapping[str, Any] = field(default_factory=dict) - files: MutableMapping[str, Union[AnySourceFile]] = field(default_factory=dict) + files: MutableMapping[str, AnySourceFile] = field(default_factory=dict) metadata: ManifestMetadata = field(default_factory=ManifestMetadata) flat_graph: Dict[str, Any] = field(default_factory=dict) state_check: ManifestStateCheck = field(default_factory=ManifestStateCheck) @@ -1511,11 +1511,7 @@ def add_semantic_model(self, source_file: SchemaSourceFile, semantic_model: Sema self.semantic_models[semantic_model.unique_id] = semantic_model source_file.semantic_models.append(semantic_model.unique_id) - def add_unit_test( - self, - source_file: Union[SchemaSourceFile, FixtureSourceFile], - unit_test: UnitTestDefinition, - ): + def add_unit_test(self, source_file: SchemaSourceFile, unit_test: UnitTestDefinition): if unit_test.unique_id in self.unit_tests: raise DuplicateResourceNameError(unit_test, self.unit_tests[unit_test.unique_id]) self.unit_tests[unit_test.unique_id] = unit_test diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 88575405d2f..cb642523e35 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -286,7 +286,8 @@ def _add_unit_test_to_fixture_files(self, unit_test_definition): if unit_test_definition.expect.fixture: # find fixture file object and store unit_test_definition unique_id fixture_source_file = self.get_fixture_source_file( - unit_test_definition.expect.fixture, self.project.project_name + unit_test_definition.expect.fixture, + self.project.project_name, ) fixture_source_file.unit_tests.append(unit_test_definition.unique_id) @@ -297,7 +298,9 @@ def get_fixture_source_file(self, fixture_name: str, project_name: str): fixture_source_file = self.manifest.files[fixture.file_id] return fixture_source_file else: - raise ParsingError(f"Fixture file not found: {fixture_unique_id}") + raise ParsingError( + f"File not found for fixture '{fixture_name}' in unit tests in {self.yaml.path.original_file_path}" + ) def _get_unit_test(self, data: Dict[str, Any]) -> UnparsedUnitTest: try: diff --git a/tests/functional/unit_testing/test_csv_fixtures.py b/tests/functional/unit_testing/test_csv_fixtures.py index 83ebf5f4d96..89219d2ddcc 100644 --- a/tests/functional/unit_testing/test_csv_fixtures.py +++ b/tests/functional/unit_testing/test_csv_fixtures.py @@ -1,5 +1,5 @@ import pytest -from dbt.exceptions import ParsingError, YamlParseDictError +from dbt.exceptions import ParsingError, YamlParseDictError, DuplicateResourceNameError from dbt.tests.util import run_dbt, write_file from fixtures import ( my_model_sql, @@ -186,13 +186,8 @@ def models(self): } def test_missing(self, project): - results = run_dbt(["run"]) - assert len(results) == 3 - - # Select by model name - expected_error = "Could not find fixture file fake_fixture for unit test" - results = run_dbt(["test", "--select", "my_model"], expect_pass=False) - assert expected_error in results[0].message + with pytest.raises(ParsingError): + run_dbt(["run"]) class TestUnitTestsDuplicateCSVFile: @@ -219,10 +214,5 @@ def tests(self): } def test_duplicate(self, project): - results = run_dbt(["run"]) - assert len(results) == 3 - - # Select by model name - results = run_dbt(["test", "--select", "my_model"], expect_pass=False) - expected_error = "Found multiple fixture files named test_my_model_basic_fixture" - assert expected_error in results[0].message + with pytest.raises(DuplicateResourceNameError): + run_dbt(["run"]) From ee8d247eb9e1294d22f96574faa6ba6a16ed69f5 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 5 Dec 2023 19:37:25 -0500 Subject: [PATCH 08/17] Update partial parsing for fixtures --- core/dbt/parser/partial.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py index 532431ccd08..9c9180d4131 100644 --- a/core/dbt/parser/partial.py +++ b/core/dbt/parser/partial.py @@ -280,6 +280,10 @@ def delete_from_saved(self, file_id): if saved_source_file.parse_file_type == ParseFileType.Documentation: self.delete_doc_node(saved_source_file) + # fixtures + if saved_source_file.parse_file_type == ParseFileType.Fixture: + self.delete_fixture_node(saved_source_file) + fire_event(PartialParsingFile(operation="deleted", file_id=file_id)) # Updates for non-schema files @@ -293,6 +297,8 @@ def update_in_saved(self, file_id): self.update_macro_in_saved(new_source_file, old_source_file) elif new_source_file.parse_file_type == ParseFileType.Documentation: self.update_doc_in_saved(new_source_file, old_source_file) + elif new_source_file.parse_file_type == ParseFileType.Fixture: + self.update_fixture_in_saved(new_source_file, old_source_file) else: raise Exception(f"Invalid parse_file_type in source_file {file_id}") fire_event(PartialParsingFile(operation="updated", file_id=file_id)) @@ -377,6 +383,13 @@ def update_doc_in_saved(self, new_source_file, old_source_file): self.saved_files[new_source_file.file_id] = deepcopy(new_source_file) self.add_to_pp_files(new_source_file) + def update_fixture_in_saved(self, new_source_file, old_source_file): + if self.already_scheduled_for_parsing(old_source_file): + return + self.delete_fixture_node(old_source_file) + self.saved_files[new_source_file.file_id] = deepcopy(new_source_file) + self.add_to_pp_files(new_source_file) + def remove_mssat_file(self, source_file): # nodes [unique_ids] -- SQL files # There should always be a node for a SQL file @@ -579,6 +592,20 @@ def delete_doc_node(self, source_file): # Remove the file object self.saved_manifest.files.pop(source_file.file_id) + def delete_fixture_node(self, source_file): + # remove fixtures from the "fixtures" dictionary + fixture_unique_id = source_file.fixture + self.saved_manifest.fixtures.pop(fixture_unique_id) + unit_tests = source_file.unit_tests.copy() + for unique_id in unit_tests: + unit_test = self.saved_manifest.unit_tests.pop(unique_id) + # schedule unit_test for parsing + self._schedule_for_parsing( + "unit_tests", unit_test, unit_test.name, self.delete_schema_unit_test + ) + source_file.unit_tests.remove(unique_id) + self.saved_manifest.files.pop(source_file.file_id) + # Schema files ----------------------- # Changed schema files def change_schema_file(self, file_id): From 1557818074945f67797a1780de80fd51f2ef557e Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 5 Dec 2023 20:02:44 -0500 Subject: [PATCH 09/17] Partial parsing tests --- .../unit_testing/test_csv_fixtures.py | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/functional/unit_testing/test_csv_fixtures.py b/tests/functional/unit_testing/test_csv_fixtures.py index 89219d2ddcc..19bc65a8fd5 100644 --- a/tests/functional/unit_testing/test_csv_fixtures.py +++ b/tests/functional/unit_testing/test_csv_fixtures.py @@ -1,6 +1,6 @@ import pytest from dbt.exceptions import ParsingError, YamlParseDictError, DuplicateResourceNameError -from dbt.tests.util import run_dbt, write_file +from dbt.tests.util import run_dbt, write_file, rm_file from fixtures import ( my_model_sql, my_model_a_sql, @@ -101,6 +101,35 @@ def test_unit_test(self, project): results = run_dbt(["test", "--select", "my_model"], expect_pass=False) assert len(results) == 5 + # Check partial parsing remove fixture file + rm_file(project.project_root, "tests", "fixtures", "test_my_model_a_fixture.csv") + with pytest.raises( + ParsingError, + match="File not found for fixture 'test_my_model_a_fixture' in unit tests", + ): + run_dbt(["test", "--select", "my_model"], expect_pass=False) + # put back file and check that it works + write_file( + test_my_model_a_fixture_csv, + project.project_root, + "tests", + "fixtures", + "test_my_model_a_fixture.csv", + ) + results = run_dbt(["test", "--select", "my_model"], expect_pass=False) + assert len(results) == 5 + # Now update file + write_file( + test_my_model_a_fixture_csv + "2,2", + project.project_root, + "tests", + "fixtures", + "test_my_model_a_fixture.csv", + ) + manifest = run_dbt(["parse"]) + fixture_source_file = manifest.files["test://tests/fixtures/test_my_model_a_fixture.csv"] + assert fixture_source_file.contents == "id,string_a\n1,a\n2,2" + # Check error with invalid format key write_file( test_my_model_file_csv_yml + datetime_test_invalid_format_key, From d115533dbb55aef43fa45b20deb5623d03782e8d Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 5 Dec 2023 20:04:55 -0500 Subject: [PATCH 10/17] Changie --- .changes/unreleased/Features-20231205-200447.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20231205-200447.yaml diff --git a/.changes/unreleased/Features-20231205-200447.yaml b/.changes/unreleased/Features-20231205-200447.yaml new file mode 100644 index 00000000000..6af669a81e3 --- /dev/null +++ b/.changes/unreleased/Features-20231205-200447.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Make fixture files full-fledged parts of the manifest and enable partial parsing +time: 2023-12-05T20:04:47.117029-05:00 +custom: + Author: gshank + Issue: "9067" From 38e26a5078a6f659021dfea3d0f43b88acae21f1 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 5 Dec 2023 20:48:49 -0500 Subject: [PATCH 11/17] Fix test for Windows, remove --no-partial-parse on state test, rename method --- core/dbt/parser/unit_tests.py | 6 +++--- tests/functional/unit_testing/test_csv_fixtures.py | 3 ++- tests/functional/unit_testing/test_state.py | 4 +--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index cb642523e35..680c154f13a 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -279,19 +279,19 @@ def _add_unit_test_to_fixture_files(self, unit_test_definition): for given in unit_test_definition.given: if given.fixture: # find fixture file object and store unit_test_definition unique_id - fixture_source_file = self.get_fixture_source_file( + fixture_source_file = self._get_fixture_source_file( given.fixture, self.project.project_name ) fixture_source_file.unit_tests.append(unit_test_definition.unique_id) if unit_test_definition.expect.fixture: # find fixture file object and store unit_test_definition unique_id - fixture_source_file = self.get_fixture_source_file( + fixture_source_file = self._get_fixture_source_file( unit_test_definition.expect.fixture, self.project.project_name, ) fixture_source_file.unit_tests.append(unit_test_definition.unique_id) - def get_fixture_source_file(self, fixture_name: str, project_name: str): + def _get_fixture_source_file(self, fixture_name: str, project_name: str): fixture_unique_id = f"fixture.{project_name}.{fixture_name}" if fixture_unique_id in self.manifest.fixtures: fixture = self.manifest.fixtures[fixture_unique_id] diff --git a/tests/functional/unit_testing/test_csv_fixtures.py b/tests/functional/unit_testing/test_csv_fixtures.py index 19bc65a8fd5..a635cf87d3f 100644 --- a/tests/functional/unit_testing/test_csv_fixtures.py +++ b/tests/functional/unit_testing/test_csv_fixtures.py @@ -127,7 +127,8 @@ def test_unit_test(self, project): "test_my_model_a_fixture.csv", ) manifest = run_dbt(["parse"]) - fixture_source_file = manifest.files["test://tests/fixtures/test_my_model_a_fixture.csv"] + fixture = manifest.fixtures["fixture.test.test_my_model_a_fixture"] + fixture_source_file = manifest.files[fixture.file_id] assert fixture_source_file.contents == "id,string_a\n1,a\n2,2" # Check error with invalid format key diff --git a/tests/functional/unit_testing/test_state.py b/tests/functional/unit_testing/test_state.py index 1a1501d05c5..1d56fa8c221 100644 --- a/tests/functional/unit_testing/test_state.py +++ b/tests/functional/unit_testing/test_state.py @@ -70,10 +70,8 @@ def test_state_modified(self, project): "fixtures", "test_my_model_fixture.csv", ) - # TODO: remove --no-partial-parse as part of https://github.com/dbt-labs/dbt-core/issues/9067 results = run_dbt( - ["--no-partial-parse", "test", "--select", "state:modified", "--state", "state"], - expect_pass=True, + ["test", "--select", "state:modified", "--state", "state"], expect_pass=True ) assert len(results) == 1 assert results[0].node.name.endswith("test_depends_on_fixture") From ed0571d4f65943820e6e883e596022c424137163 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 6 Dec 2023 09:55:34 -0500 Subject: [PATCH 12/17] Fix the other Windows test --- tests/functional/unit_testing/test_csv_fixtures.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/unit_testing/test_csv_fixtures.py b/tests/functional/unit_testing/test_csv_fixtures.py index a635cf87d3f..1211cd49d78 100644 --- a/tests/functional/unit_testing/test_csv_fixtures.py +++ b/tests/functional/unit_testing/test_csv_fixtures.py @@ -91,7 +91,8 @@ def test_unit_test(self, project): assert len(results) == 3 manifest = run_dbt(["parse"]) # Note: this manifest is deserialized from msgpack - fixture_source_file = manifest.files["test://tests/fixtures/test_my_model_a_fixture.csv"] + fixture = manifest.fixtures["fixture.test.test_my_model_a_fixture"] + fixture_source_file = manifest.files[fixture.file_id] assert fixture_source_file.fixture == "fixture.test.test_my_model_a_fixture" assert fixture_source_file.unit_tests == [ "unit_test.test.my_model.test_my_model_string_concat" From a2b88681b4d622e9d66e58e6a5b26243c52c3daf Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 6 Dec 2023 10:53:14 -0500 Subject: [PATCH 13/17] Fix Windows test again --- tests/functional/unit_testing/test_csv_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/unit_testing/test_csv_fixtures.py b/tests/functional/unit_testing/test_csv_fixtures.py index 1211cd49d78..250cc0abe8f 100644 --- a/tests/functional/unit_testing/test_csv_fixtures.py +++ b/tests/functional/unit_testing/test_csv_fixtures.py @@ -130,7 +130,7 @@ def test_unit_test(self, project): manifest = run_dbt(["parse"]) fixture = manifest.fixtures["fixture.test.test_my_model_a_fixture"] fixture_source_file = manifest.files[fixture.file_id] - assert fixture_source_file.contents == "id,string_a\n1,a\n2,2" + assert "2,2" in fixture_source_file.contents # Check error with invalid format key write_file( From 9b8d663b1679d71bc378b78115d3bbd6b46fd180 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 6 Dec 2023 11:36:53 -0500 Subject: [PATCH 14/17] Rename to UnitTestFileFixture, create rows when parsing --- core/dbt/contracts/graph/manifest.py | 6 +++--- core/dbt/contracts/graph/nodes.py | 3 ++- core/dbt/contracts/graph/unparsed.py | 1 - core/dbt/parser/fixtures.py | 19 +++++++++++++++---- .../unit_testing/test_csv_fixtures.py | 1 + 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index c47cb4aace2..0a607b4bab5 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -43,7 +43,7 @@ SourceDefinition, UnpatchedSourceDefinition, UnitTestDefinition, - UnitTestFixture, + UnitTestFileFixture, ) from dbt.contracts.graph.unparsed import SourcePatch, NodeVersion, UnparsedVersion from dbt.contracts.graph.manifest_upgrade import upgrade_manifest_json @@ -809,7 +809,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin): semantic_models: MutableMapping[str, SemanticModel] = field(default_factory=dict) unit_tests: MutableMapping[str, UnitTestDefinition] = field(default_factory=dict) saved_queries: MutableMapping[str, SavedQuery] = field(default_factory=dict) - fixtures: MutableMapping[str, UnitTestFixture] = field(default_factory=dict) + fixtures: MutableMapping[str, UnitTestFileFixture] = field(default_factory=dict) _doc_lookup: Optional[DocLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} @@ -1517,7 +1517,7 @@ def add_unit_test(self, source_file: SchemaSourceFile, unit_test: UnitTestDefini self.unit_tests[unit_test.unique_id] = unit_test source_file.unit_tests.append(unit_test.unique_id) - def add_fixture(self, source_file: FixtureSourceFile, fixture: UnitTestFixture): + def add_fixture(self, source_file: FixtureSourceFile, fixture: UnitTestFileFixture): if fixture.unique_id in self.fixtures: raise DuplicateResourceNameError(fixture, self.fixtures[fixture.unique_id]) self.fixtures[fixture.unique_id] = fixture diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index ee3a84b2753..a941cd78ae5 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1134,7 +1134,8 @@ def same_contents(self, other: Optional["UnitTestDefinition"]) -> bool: return self.checksum == other.checksum -class UnitTestFixture(BaseNode): +@dataclass +class UnitTestFileFixture(BaseNode): resource_type: Literal[NodeType.Fixture] rows: Optional[List[Dict[str, Any]]] = None diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 576dbbee87c..30705c888f3 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -799,7 +799,6 @@ def get_rows(self, project_root: str, paths: List[str]) -> List[Dict[str, Any]]: assert isinstance(self.rows, str) dummy_file = StringIO(self.rows) reader = csv.DictReader(dummy_file) - rows = [] for row in reader: rows.append(row) return rows diff --git a/core/dbt/parser/fixtures.py b/core/dbt/parser/fixtures.py index 191c93b2789..f12cc6f272a 100644 --- a/core/dbt/parser/fixtures.py +++ b/core/dbt/parser/fixtures.py @@ -1,13 +1,15 @@ -from typing import Optional +from typing import Optional, Dict, List, Any +from io import StringIO +import csv from dbt.contracts.files import FixtureSourceFile -from dbt.contracts.graph.nodes import UnitTestFixture +from dbt.contracts.graph.nodes import UnitTestFileFixture from dbt.node_types import NodeType from dbt.parser.base import Parser from dbt.parser.search import FileBlock -class FixtureParser(Parser[UnitTestFixture]): +class FixtureParser(Parser[UnitTestFileFixture]): @property def resource_type(self) -> NodeType: return NodeType.Fixture @@ -24,12 +26,21 @@ def parse_file(self, file_block: FileBlock): assert isinstance(file_block.file, FixtureSourceFile) unique_id = self.generate_unique_id(file_block.name) - fixture = UnitTestFixture( + fixture = UnitTestFileFixture( name=file_block.name, path=file_block.file.path.relative_path, original_file_path=file_block.path.original_file_path, package_name=self.project.project_name, unique_id=unique_id, resource_type=NodeType.Fixture, + rows=self.get_rows(file_block.file.contents), ) self.manifest.add_fixture(file_block.file, fixture) + + def get_rows(self, contents) -> List[Dict[str, Any]]: + rows = [] + dummy_file = StringIO(contents) + reader = csv.DictReader(dummy_file) + for row in reader: + rows.append(row) + return rows diff --git a/tests/functional/unit_testing/test_csv_fixtures.py b/tests/functional/unit_testing/test_csv_fixtures.py index 250cc0abe8f..cb80a76fac5 100644 --- a/tests/functional/unit_testing/test_csv_fixtures.py +++ b/tests/functional/unit_testing/test_csv_fixtures.py @@ -131,6 +131,7 @@ def test_unit_test(self, project): fixture = manifest.fixtures["fixture.test.test_my_model_a_fixture"] fixture_source_file = manifest.files[fixture.file_id] assert "2,2" in fixture_source_file.contents + assert fixture.rows == [{'id': '1', 'string_a': 'a'}, {'id': '2', 'string_a': '2'}] # Check error with invalid format key write_file( From c55a53f6cc2cb6c0571362904023fe3434881fe6 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 6 Dec 2023 14:56:48 -0500 Subject: [PATCH 15/17] Normalize input and output fixtures to have dictionary rows at parse time --- core/dbt/contracts/graph/nodes.py | 4 +- core/dbt/contracts/graph/unparsed.py | 36 ------ core/dbt/parser/unit_tests.py | 117 +++++++++++------- .../unit_testing/test_csv_fixtures.py | 2 +- tests/unit/test_unit_test_parser.py | 2 +- 5 files changed, 73 insertions(+), 88 deletions(-) diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index a941cd78ae5..226f8b00fd9 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1116,14 +1116,14 @@ def tags(self) -> List[str]: tags = self.config.tags return [tags] if isinstance(tags, str) else tags - def build_unit_test_checksum(self, project_root: str, fixture_paths: List[str]): + def build_unit_test_checksum(self): # everything except 'description' data = f"{self.model}-{self.given}-{self.expect}-{self.overrides}" # include underlying fixture data for input in self.given: if input.fixture: - data += f"-{input.get_rows(project_root, fixture_paths)}" + data += f"-{input.rows}" self.checksum = hashlib.new("sha256", data.encode("utf-8")).hexdigest() diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 30705c888f3..d3e5c3a85d7 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -1,10 +1,7 @@ import datetime import re -import csv -from io import StringIO from dbt import deprecations -from dbt.clients.system import find_matching from dbt.node_types import NodeType from dbt.contracts.graph.semantic_models import ( Defaults, @@ -782,39 +779,6 @@ def rows(self) -> Optional[Union[str, List[Dict[str, Any]]]]: def fixture(self) -> Optional[str]: return None - def get_rows(self, project_root: str, paths: List[str]) -> List[Dict[str, Any]]: - if self.format == UnitTestFormat.Dict: - assert isinstance(self.rows, List) - return self.rows - elif self.format == UnitTestFormat.CSV: - rows = [] - if self.fixture is not None: - assert isinstance(self.fixture, str) - file_path = self.get_fixture_path(self.fixture, project_root, paths) - with open(file_path, newline="") as csvfile: - reader = csv.DictReader(csvfile) - for row in reader: - rows.append(row) - else: # using inline csv - assert isinstance(self.rows, str) - dummy_file = StringIO(self.rows) - reader = csv.DictReader(dummy_file) - for row in reader: - rows.append(row) - return rows - - def get_fixture_path(self, fixture: str, project_root: str, paths: List[str]) -> str: - fixture_path = f"{fixture}.csv" - matches = find_matching(project_root, paths, fixture_path) - if len(matches) == 0: - raise ParsingError(f"Could not find fixture file {fixture} for unit test") - elif len(matches) > 1: - raise ParsingError( - f"Found multiple fixture files named {fixture} at {[d['relative_path'] for d in matches]}. Please use a unique name for each fixture file." - ) - - return matches[0]["absolute_path"] - def validate_fixture(self, fixture_type, test_name) -> None: if self.format == UnitTestFormat.Dict and not isinstance(self.rows, list): raise ParsingError( diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 680c154f13a..9da9a986880 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -2,6 +2,8 @@ from pathlib import Path from typing import List, Set, Dict, Any import os +from io import StringIO +import csv from dbt_extractor import py_extract_from_source, ExtractionError # type: ignore @@ -20,7 +22,7 @@ UnitTestConfig, UnitTestSourceDefinition, ) -from dbt.contracts.graph.unparsed import UnparsedUnitTest +from dbt.contracts.graph.unparsed import UnparsedUnitTest, UnitTestFormat from dbt.exceptions import ParsingError, InvalidUnitTestGivenInput from dbt.graph import UniqueId from dbt.node_types import NodeType @@ -72,9 +74,7 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): unique_id=test_case.unique_id, config=UnitTestNodeConfig( materialized="unit", - expected_rows=test_case.expect.get_rows( - self.root_project.project_root, self.root_project.fixture_paths - ), + expected_rows=test_case.expect.rows, # type:ignore ), raw_code=tested_node.raw_code, database=tested_node.database, @@ -116,7 +116,6 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): # extract the original_input_node from the ref in the "input" key of the given list original_input_node = self._get_original_input_node(given.input, tested_node) - project_root = self.root_project.project_root common_fields = { "resource_type": NodeType.Model, "package_name": test_case.package_name, @@ -127,9 +126,7 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): "schema": original_input_node.schema, "fqn": original_input_node.fqn, "checksum": FileHash.empty(), - "raw_code": self._build_fixture_raw_code( - given.get_rows(project_root, self.root_project.fixture_paths), None - ), + "raw_code": self._build_fixture_raw_code(given.rows, None), } if original_input_node.resource_type in ( @@ -240,13 +237,6 @@ def parse(self) -> ParseResult: ) unit_test_config = self._build_unit_test_config(unit_test_fqn, unit_test.config) - # Check that format and type of rows matches for each given input - for input in unit_test.given: - if input.rows is None and input.fixture is None: - input.rows = self._load_rows_from_seed(input.input) - input.validate_fixture("input", unit_test.name) - unit_test.expect.validate_fixture("expected", unit_test.name) - unit_test_definition = UnitTestDefinition( name=unit_test.name, model=unit_test.model, @@ -264,44 +254,20 @@ def parse(self) -> ParseResult: config=unit_test_config, schema=tested_model_node.schema, ) - # For partial parsing, we add the unique_id of the unit test definition to the - # fixture file records - self._add_unit_test_to_fixture_files(unit_test_definition) + + # Check that format and type of rows matches for each given input + # convert rows to a list of dictionaries + # and add the unique_id of the unit_test_definition to the fixture + # source_file for partial parsing. + self._validate_and_normalize_given(unit_test_definition) + self._validate_and_normalize_expect(unit_test_definition) + # for calculating state:modified - unit_test_definition.build_unit_test_checksum( - self.schema_parser.project.project_root, self.schema_parser.project.fixture_paths - ) + unit_test_definition.build_unit_test_checksum() self.manifest.add_unit_test(self.yaml.file, unit_test_definition) return ParseResult() - def _add_unit_test_to_fixture_files(self, unit_test_definition): - for given in unit_test_definition.given: - if given.fixture: - # find fixture file object and store unit_test_definition unique_id - fixture_source_file = self._get_fixture_source_file( - given.fixture, self.project.project_name - ) - fixture_source_file.unit_tests.append(unit_test_definition.unique_id) - if unit_test_definition.expect.fixture: - # find fixture file object and store unit_test_definition unique_id - fixture_source_file = self._get_fixture_source_file( - unit_test_definition.expect.fixture, - self.project.project_name, - ) - fixture_source_file.unit_tests.append(unit_test_definition.unique_id) - - def _get_fixture_source_file(self, fixture_name: str, project_name: str): - fixture_unique_id = f"fixture.{project_name}.{fixture_name}" - if fixture_unique_id in self.manifest.fixtures: - fixture = self.manifest.fixtures[fixture_unique_id] - fixture_source_file = self.manifest.files[fixture.file_id] - return fixture_source_file - else: - raise ParsingError( - f"File not found for fixture '{fixture_name}' in unit tests in {self.yaml.path.original_file_path}" - ) - def _get_unit_test(self, data: Dict[str, Any]) -> UnparsedUnitTest: try: UnparsedUnitTest.validate(data) @@ -351,6 +317,61 @@ def _build_fqn(self, package_name, original_file_path, model_name, test_name): fqn.append(test_name) return fqn + def _get_fixture(self, fixture_name: str, project_name: str): + fixture_unique_id = f"fixture.{project_name}.{fixture_name}" + if fixture_unique_id in self.manifest.fixtures: + fixture = self.manifest.fixtures[fixture_unique_id] + return fixture + else: + raise ParsingError( + f"File not found for fixture '{fixture_name}' in unit tests in {self.yaml.path.original_file_path}" + ) + + def _validate_and_normalize_given(self, unit_test_definition): + for ut_input in unit_test_definition.given: + self._validate_and_normalize_rows(ut_input, unit_test_definition, "input") + + def _validate_and_normalize_expect(self, unit_test_definition): + self._validate_and_normalize_rows( + unit_test_definition.expect, unit_test_definition, "expected" + ) + + def _validate_and_normalize_rows(self, ut_fixture, unit_test_definition, fixture_type) -> None: + if ut_fixture.format == UnitTestFormat.Dict: + if ut_fixture.rows is None and ut_fixture.fixture is None: # This is a seed + ut_fixture.rows = self._load_rows_from_seed(ut_fixture.input) + if not isinstance(ut_fixture.rows, list): + raise ParsingError( + f"Unit test {unit_test_definition.name} has {fixture_type} rows " + f"which do not match format {ut_fixture.format}" + ) + elif ut_fixture.format == UnitTestFormat.CSV: + if not (isinstance(ut_fixture.rows, str) or isinstance(ut_fixture.fixture, str)): + raise ParsingError( + f"Unit test {unit_test_definition.name} has {fixture_type} rows or fixtures " + f"which do not match format {ut_fixture.format}. Expected string." + ) + + if ut_fixture.fixture: + # find fixture file object and store unit_test_definition unique_id + fixture = self._get_fixture(ut_fixture.fixture, self.project.project_name) + fixture_source_file = self.manifest.files[fixture.file_id] + fixture_source_file.unit_tests.append(unit_test_definition.unique_id) + ut_fixture.rows = fixture.rows + else: + ut_fixture.rows = self._convert_csv_to_list_of_dicts(ut_fixture.rows) + else: + # This is a dictionary already + pass + + def _convert_csv_to_list_of_dicts(self, csv_string: str) -> List[Dict[str, Any]]: + dummy_file = StringIO(csv_string) + reader = csv.DictReader(dummy_file) + rows = [] + for row in reader: + rows.append(row) + return rows + def _load_rows_from_seed(self, ref_str: str) -> List[Dict[str, Any]]: """Read rows from seed file on disk if not specified in YAML config. If seed file doesn't exist, return empty list.""" ref = py_extract_from_source("{{ " + ref_str + " }}")["refs"][0] diff --git a/tests/functional/unit_testing/test_csv_fixtures.py b/tests/functional/unit_testing/test_csv_fixtures.py index cb80a76fac5..6aae95abed6 100644 --- a/tests/functional/unit_testing/test_csv_fixtures.py +++ b/tests/functional/unit_testing/test_csv_fixtures.py @@ -131,7 +131,7 @@ def test_unit_test(self, project): fixture = manifest.fixtures["fixture.test.test_my_model_a_fixture"] fixture_source_file = manifest.files[fixture.file_id] assert "2,2" in fixture_source_file.contents - assert fixture.rows == [{'id': '1', 'string_a': 'a'}, {'id': '2', 'string_a': '2'}] + assert fixture.rows == [{"id": "1", "string_a": "a"}, {"id": "2", "string_a": "2"}] # Check error with invalid format key write_file( diff --git a/tests/unit/test_unit_test_parser.py b/tests/unit/test_unit_test_parser.py index 998eba410f4..b0ad4a54613 100644 --- a/tests/unit/test_unit_test_parser.py +++ b/tests/unit/test_unit_test_parser.py @@ -134,7 +134,7 @@ def test_basic(self): config=UnitTestConfig(), schema="test_schema", ) - expected.build_unit_test_checksum("anything", "anything") + expected.build_unit_test_checksum() assertEqualNodes(unit_test, expected) def test_unit_test_config(self): From 00a7187c5b6e692c7262f29cca3d4ecfcaef4838 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 6 Dec 2023 18:55:26 -0500 Subject: [PATCH 16/17] Minor cleanup --- core/dbt/contracts/graph/unparsed.py | 30 ++-------------------------- core/dbt/parser/unit_tests.py | 10 +++------- 2 files changed, 5 insertions(+), 35 deletions(-) diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index d3e5c3a85d7..97cf6f2ad1d 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -766,34 +766,8 @@ class UnitTestFormat(StrEnum): Dict = "dict" -class UnitTestFixture: - @property - def format(self) -> UnitTestFormat: - return UnitTestFormat.Dict - - @property - def rows(self) -> Optional[Union[str, List[Dict[str, Any]]]]: - return None - - @property - def fixture(self) -> Optional[str]: - return None - - def validate_fixture(self, fixture_type, test_name) -> None: - if self.format == UnitTestFormat.Dict and not isinstance(self.rows, list): - raise ParsingError( - f"Unit test {test_name} has {fixture_type} rows which do not match format {self.format}" - ) - if self.format == UnitTestFormat.CSV and not ( - isinstance(self.rows, str) or isinstance(self.fixture, str) - ): - raise ParsingError( - f"Unit test {test_name} has {fixture_type} rows or fixtures which do not match format {self.format}. Expected string." - ) - - @dataclass -class UnitTestInputFixture(dbtClassMixin, UnitTestFixture): +class UnitTestInputFixture(dbtClassMixin): input: str rows: Optional[Union[str, List[Dict[str, Any]]]] = None format: UnitTestFormat = UnitTestFormat.Dict @@ -801,7 +775,7 @@ class UnitTestInputFixture(dbtClassMixin, UnitTestFixture): @dataclass -class UnitTestOutputFixture(dbtClassMixin, UnitTestFixture): +class UnitTestOutputFixture(dbtClassMixin): rows: Optional[Union[str, List[Dict[str, Any]]]] = None format: UnitTestFormat = UnitTestFormat.Dict fixture: Optional[str] = None diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 9da9a986880..4eeb5f5ca4f 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -255,10 +255,9 @@ def parse(self) -> ParseResult: schema=tested_model_node.schema, ) - # Check that format and type of rows matches for each given input - # convert rows to a list of dictionaries - # and add the unique_id of the unit_test_definition to the fixture - # source_file for partial parsing. + # Check that format and type of rows matches for each given input, + # convert rows to a list of dictionaries, and add the unique_id of + # the unit_test_definition to the fixture source_file for partial parsing. self._validate_and_normalize_given(unit_test_definition) self._validate_and_normalize_expect(unit_test_definition) @@ -360,9 +359,6 @@ def _validate_and_normalize_rows(self, ut_fixture, unit_test_definition, fixture ut_fixture.rows = fixture.rows else: ut_fixture.rows = self._convert_csv_to_list_of_dicts(ut_fixture.rows) - else: - # This is a dictionary already - pass def _convert_csv_to_list_of_dicts(self, csv_string: str) -> List[Dict[str, Any]]: dummy_file = StringIO(csv_string) From 21178809e527de9b0585078f2f4bc91f1fb78341 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 7 Dec 2023 10:31:53 -0500 Subject: [PATCH 17/17] Use NodeType.Fixture in unique_id --- core/dbt/parser/unit_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 4eeb5f5ca4f..c2c744b8dd5 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -317,7 +317,7 @@ def _build_fqn(self, package_name, original_file_path, model_name, test_name): return fqn def _get_fixture(self, fixture_name: str, project_name: str): - fixture_unique_id = f"fixture.{project_name}.{fixture_name}" + fixture_unique_id = f"{NodeType.Fixture}.{project_name}.{fixture_name}" if fixture_unique_id in self.manifest.fixtures: fixture = self.manifest.fixtures[fixture_unique_id] return fixture