diff --git a/.changes/unreleased/Features-20230102-091335.yaml b/.changes/unreleased/Features-20230102-091335.yaml new file mode 100644 index 00000000000..78154c12e53 --- /dev/null +++ b/.changes/unreleased/Features-20230102-091335.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Adds buildable selection mode +time: 2023-01-02T09:13:35.663627-05:00 +custom: + Author: agpapa + Issue: "6365" diff --git a/core/dbt/graph/cli.py b/core/dbt/graph/cli.py index 6059de6b042..2ae0d814327 100644 --- a/core/dbt/graph/cli.py +++ b/core/dbt/graph/cli.py @@ -44,12 +44,14 @@ def parse_union( components=intersection_components, expect_exists=expect_exists, raw=raw_spec, + indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION), ) ) return SelectionUnion( components=union_components, expect_exists=False, raw=components, + indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION), ) @@ -78,9 +80,12 @@ def parse_difference( include, DEFAULT_INCLUDES, indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION) ) excluded = parse_union_from_default( - exclude, DEFAULT_EXCLUDES, indirect_selection=IndirectSelection.Eager + exclude, DEFAULT_EXCLUDES, indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION) + ) + return SelectionDifference( + components=[included, excluded], + indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION), ) - return SelectionDifference(components=[included, excluded]) RawDefinition = Union[str, Dict[str, Any]] diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index ed91596712b..8f9561c6519 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -134,7 +134,9 @@ def select_nodes_recursively(self, spec: SelectionSpec) -> Tuple[Set[UniqueId], initial_direct = spec.combined(direct_sets) indirect_nodes = spec.combined(indirect_sets) - direct_nodes = self.incorporate_indirect_nodes(initial_direct, indirect_nodes) + direct_nodes = self.incorporate_indirect_nodes( + initial_direct, indirect_nodes, spec.indirect_selection + ) if spec.expect_exists and len(direct_nodes) == 0: warn_or_error(NoNodesForSelectionCriteria(spec_raw=str(spec.raw))) @@ -197,7 +199,7 @@ def expand_selection( ) -> Tuple[Set[UniqueId], Set[UniqueId]]: # Test selection by default expands to include an implicitly/indirectly selected tests. # `dbt test -m model_a` also includes tests that directly depend on `model_a`. - # Expansion has two modes, EAGER and CAUTIOUS. + # Expansion has three modes, EAGER, CAUTIOUS and BUILDABLE. # # EAGER mode: If ANY parent is selected, select the test. # @@ -205,11 +207,22 @@ def expand_selection( # - If ALL parents are selected, select the test. # - If ANY parent is missing, return it separately. We'll keep it around # for later and see if its other parents show up. + # + # BUILDABLE mode: + # - If ALL parents are selected, or the parents of the test are themselves parents of the selected, select the test. + # - If ANY parent is missing, return it separately. We'll keep it around + # for later and see if its other parents show up. + # # Users can opt out of inclusive EAGER mode by passing --indirect-selection cautious # CLI argument or by specifying `indirect_selection: true` in a yaml selector direct_nodes = set(selected) indirect_nodes = set() + selected_and_parents = set() + if indirect_selection == IndirectSelection.Buildable: + selected_and_parents = selected.union(self.graph.select_parents(selected)).union( + self.manifest.sources + ) for unique_id in self.graph.select_successors(selected): if unique_id in self.manifest.nodes: @@ -220,14 +233,20 @@ def expand_selection( node.depends_on_nodes ) <= set(selected): direct_nodes.add(unique_id) - # if not: + elif indirect_selection == IndirectSelection.Buildable and set( + node.depends_on_nodes + ) <= set(selected_and_parents): + direct_nodes.add(unique_id) else: indirect_nodes.add(unique_id) return direct_nodes, indirect_nodes def incorporate_indirect_nodes( - self, direct_nodes: Set[UniqueId], indirect_nodes: Set[UniqueId] = set() + self, + direct_nodes: Set[UniqueId], + indirect_nodes: Set[UniqueId] = set(), + indirect_selection: IndirectSelection = IndirectSelection.Eager, ) -> Set[UniqueId]: # Check tests previously selected indirectly to see if ALL their # parents are now present. @@ -238,11 +257,19 @@ def incorporate_indirect_nodes( selected = set(direct_nodes) - for unique_id in indirect_nodes: - if unique_id in self.manifest.nodes: - node = self.manifest.nodes[unique_id] - if set(node.depends_on_nodes) <= set(selected): - selected.add(unique_id) + if indirect_selection == IndirectSelection.Cautious: + for unique_id in indirect_nodes: + if unique_id in self.manifest.nodes: + node = self.manifest.nodes[unique_id] + if set(node.depends_on_nodes) <= set(selected): + selected.add(unique_id) + elif indirect_selection == IndirectSelection.Buildable: + selected_and_parents = selected.union(self.graph.select_parents(selected)) + for unique_id in indirect_nodes: + if unique_id in self.manifest.nodes: + node = self.manifest.nodes[unique_id] + if set(node.depends_on_nodes) <= set(selected_and_parents): + selected.add(unique_id) return selected diff --git a/core/dbt/graph/selector_spec.py b/core/dbt/graph/selector_spec.py index 991ae7fcb89..5b8e4560d5e 100644 --- a/core/dbt/graph/selector_spec.py +++ b/core/dbt/graph/selector_spec.py @@ -24,6 +24,7 @@ class IndirectSelection(StrEnum): Eager = "eager" Cautious = "cautious" + Buildable = "buildable" def _probably_path(value: str): @@ -173,12 +174,14 @@ class BaseSelectionGroup(dbtClassMixin, Iterable[SelectionSpec], metaclass=ABCMe def __init__( self, components: Iterable[SelectionSpec], + indirect_selection: IndirectSelection = IndirectSelection.Eager, expect_exists: bool = False, raw: Any = None, ): self.components: List[SelectionSpec] = list(components) self.expect_exists = expect_exists self.raw = raw + self.indirect_selection = indirect_selection def __iter__(self) -> Iterator[SelectionSpec]: for component in self.components: diff --git a/core/dbt/main.py b/core/dbt/main.py index 55920e8a5cc..7b97235740b 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -385,7 +385,7 @@ def _build_build_subparser(subparsers, base_subparser): ) sub.add_argument( "--indirect-selection", - choices=["eager", "cautious"], + choices=["eager", "cautious", "buildable"], default="eager", dest="indirect_selection", help=""" @@ -763,7 +763,7 @@ def _build_test_subparser(subparsers, base_subparser): ) sub.add_argument( "--indirect-selection", - choices=["eager", "cautious"], + choices=["eager", "cautious", "buildable"], default="eager", dest="indirect_selection", help=""" @@ -869,7 +869,7 @@ def _build_list_subparser(subparsers, base_subparser): ) sub.add_argument( "--indirect-selection", - choices=["eager", "cautious"], + choices=["eager", "cautious", "buildable"], default="eager", dest="indirect_selection", help=""" diff --git a/test/unit/test_flags.py b/test/unit/test_flags.py index 4be866338a2..8bb248af443 100644 --- a/test/unit/test_flags.py +++ b/test/unit/test_flags.py @@ -206,6 +206,9 @@ def test__flags(self): self.user_config.indirect_selection = 'cautious' flags.set_from_args(self.args, self.user_config) self.assertEqual(flags.INDIRECT_SELECTION, IndirectSelection.Cautious) + self.user_config.indirect_selection = 'buildable' + flags.set_from_args(self.args, self.user_config) + self.assertEqual(flags.INDIRECT_SELECTION, IndirectSelection.Buildable) self.user_config.indirect_selection = None flags.set_from_args(self.args, self.user_config) self.assertEqual(flags.INDIRECT_SELECTION, IndirectSelection.Eager) diff --git a/tests/functional/test_selection/fixtures.py b/tests/functional/test_selection/fixtures.py index ae798edd3fd..48c3f40c62d 100644 --- a/tests/functional/test_selection/fixtures.py +++ b/tests/functional/test_selection/fixtures.py @@ -64,7 +64,7 @@ tags = ['a_or_b'] ) }} -select 1 as fun +select * FROM {{ref('model_b')}} """ diff --git a/tests/functional/test_selection/test_selection_expansion.py b/tests/functional/test_selection/test_selection_expansion.py index b563398e89f..e006fd50258 100644 --- a/tests/functional/test_selection/test_selection_expansion.py +++ b/tests/functional/test_selection/test_selection_expansion.py @@ -184,6 +184,18 @@ def test_model_a_exclude_specific_test_cautious( self.list_tests_and_assert(select, exclude, expected, indirect_selection) self.run_tests_and_assert(select, exclude, expected, indirect_selection) + def test_model_a_exclude_specific_test_buildable( + self, + project, + ): + select = "model_a" + exclude = "unique_model_a_fun" + expected = ["just_a", "cf_a_b", "cf_a_src", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_"] + indirect_selection = "buildable" + + self.list_tests_and_assert(select, exclude, expected, indirect_selection) + self.run_tests_and_assert(select, exclude, expected, indirect_selection) + def test_only_generic( self, project, @@ -374,6 +386,40 @@ def test_model_a_indirect_selection_eager( self.list_tests_and_assert(select, exclude, expected, indirect_selection) self.run_tests_and_assert(select, exclude, expected, indirect_selection) + def test_model_a_indirect_selection_cautious( + self, + project, + ): + select = "model_a" + exclude = None + expected = [ + "just_a", + "unique_model_a_fun", + ] + indirect_selection = "cautious" + + self.list_tests_and_assert(select, exclude, expected, indirect_selection) + self.run_tests_and_assert(select, exclude, expected, indirect_selection) + + def test_model_a_indirect_selection_buildable( + self, + project, + ): + select = "model_a" + exclude = None + expected = [ + "cf_a_b", + "cf_a_src", + "just_a", + "relationships_model_a_fun__fun__ref_model_b_", + "relationships_model_a_fun__fun__source_my_src_my_tbl_", + "unique_model_a_fun", + ] + indirect_selection = "buildable" + + self.list_tests_and_assert(select, exclude, expected, indirect_selection) + self.run_tests_and_assert(select, exclude, expected, indirect_selection) + def test_model_a_indirect_selection_exclude_unique_tests( self, project, @@ -402,16 +448,21 @@ def selectors(self): definition: method: fqn value: model_a - - name: model_a_no_indirect_selection + - name: model_a_cautious_indirect_selection definition: method: fqn value: model_a indirect_selection: "cautious" - - name: model_a_yes_indirect_selection + - name: model_a_eager_indirect_selection definition: method: fqn value: model_a indirect_selection: "eager" + - name: model_a_buildable_indirect_selection + definition: + method: fqn + value: model_a + indirect_selection: "buildable" """ def test_selector_model_a_unset_indirect_selection( @@ -440,7 +491,7 @@ def test_selector_model_a_unset_indirect_selection( selector_name="model_a_unset_indirect_selection", ) - def test_selector_model_a_no_indirect_selection( + def test_selector_model_a_cautious_indirect_selection( self, project, ): @@ -450,16 +501,42 @@ def test_selector_model_a_no_indirect_selection( include=None, exclude=None, expected_tests=expected, - selector_name="model_a_no_indirect_selection", + selector_name="model_a_cautious_indirect_selection", + ) + self.run_tests_and_assert( + include=None, + exclude=None, + expected_tests=expected, + selector_name="model_a_cautious_indirect_selection", + ) + + def test_selector_model_a_eager_indirect_selection( + self, + project, + ): + expected = [ + "cf_a_b", + "cf_a_src", + "just_a", + "relationships_model_a_fun__fun__ref_model_b_", + "relationships_model_a_fun__fun__source_my_src_my_tbl_", + "unique_model_a_fun", + ] + + self.list_tests_and_assert( + include=None, + exclude=None, + expected_tests=expected, + selector_name="model_a_eager_indirect_selection", ) self.run_tests_and_assert( include=None, exclude=None, expected_tests=expected, - selector_name="model_a_no_indirect_selection", + selector_name="model_a_eager_indirect_selection", ) - def test_selector_model_a_yes_indirect_selection( + def test_selector_model_a_buildable_indirect_selection( self, project, ): @@ -476,11 +553,11 @@ def test_selector_model_a_yes_indirect_selection( include=None, exclude=None, expected_tests=expected, - selector_name="model_a_yes_indirect_selection", + selector_name="model_a_buildable_indirect_selection", ) self.run_tests_and_assert( include=None, exclude=None, expected_tests=expected, - selector_name="model_a_yes_indirect_selection", + selector_name="model_a_buildable_indirect_selection", )