Allow --select and --exclude multiple times (#7169)

* One argument per line

* Tests for multiple `--select` or `--exclude`

* Allow `--select` and `--exclude` multiple times

* Changelog entry

* MultiOption options must be specified with type=tuple or type=ChoiceTuple

* Testing for `--output-keys` and `--resource-type`

* Validate that any new param with `MultiOption` should also have `type=tuple` (or `ChoiceTuple`) and `multiple=True`

.changes/unreleased/Breaking Changes-20230314-161505.yaml

@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: Allow `--select` and `--exclude` multiple times
+time: 2023-03-14T16:15:05.81741-06:00
+custom:
+  Author: dbeatty10
+  Issue: "7158"

core/dbt/cli/options.py

@@ -1,4 +1,8 @@
 import click
+import inspect
+import typing as t
+from click import Context
+from dbt.cli.option_types import ChoiceTuple
 
 
 # Implementation from: https://stackoverflow.com/a/48394004
@@ -12,6 +16,19 @@ def __init__(self, *args, **kwargs):
         self._previous_parser_process = None
         self._eat_all_parser = None
 
+        # validate that multiple=True
+        multiple = kwargs.pop("multiple", None)
+        msg = f"MultiOption named `{self.name}` must have multiple=True (rather than {multiple})"
+        assert multiple, msg
+
+        # validate that type=tuple or type=ChoiceTuple
+        option_type = kwargs.pop("type", None)
+        msg = f"MultiOption named `{self.name}` must be tuple or ChoiceTuple (rather than {option_type})"
+        if inspect.isclass(option_type):
+            assert issubclass(option_type, tuple), msg
+        else:
+            assert isinstance(option_type, ChoiceTuple), msg
+
     def add_to_parser(self, parser, ctx):
         def parser_process(value, state):
             # method to hook to the parser.process
@@ -42,3 +59,17 @@ def parser_process(value, state):
                 our_parser.process = parser_process
                 break
         return retval
+
+    def type_cast_value(self, ctx: Context, value: t.Any) -> t.Any:
+        def flatten(data):
+            if isinstance(data, tuple):
+                for x in data:
+                    yield from flatten(x)
+            else:
+                yield data
+
+        # there will be nested tuples to flatten when multiple=True
+        value = super(MultiOption, self).type_cast_value(ctx, value)
+        if value:
+            value = tuple(flatten(value))
+        return value

core/dbt/cli/params.py

@@ -84,7 +84,12 @@
 )
 
 exclude = click.option(
-    "--exclude", envvar=None, type=tuple, cls=MultiOption, help="Specify the nodes to exclude."
+    "--exclude",
+    envvar=None,
+    type=tuple,
+    cls=MultiOption,
+    multiple=True,
+    help="Specify the nodes to exclude.",
 )
 
 fail_fast = click.option(
@@ -189,8 +194,9 @@
         "Space-delimited listing of node properties to include as custom keys for JSON output "
         "(e.g. `--output json --output-keys name resource_type description`)"
     ),
-    type=list,
+    type=tuple,
     cls=MultiOption,
+    multiple=True,
     default=[],
 )
 
@@ -315,6 +321,7 @@
         case_sensitive=False,
     ),
     cls=MultiOption,
+    multiple=True,
     default=(),
 )
 
@@ -324,6 +331,7 @@
     "envvar": None,
     "help": "Specify the nodes to include.",
     "cls": MultiOption,
+    "multiple": True,
     "type": tuple,
 }
 

tests/functional/graph_selection/test_graph_selection.py

@@ -163,6 +163,12 @@ def test_concat(self, project):
         results = run_dbt(["run", "--select", "@emails_alt", "users_rollup"], expect_pass=False)
         check_result_nodes_by_name(results, ["users_rollup", "users", "emails_alt"])
 
+    def test_concat_multiple(self, project):
+        results = run_dbt(
+            ["run", "--select", "@emails_alt", "--select", "users_rollup"], expect_pass=False
+        )
+        check_result_nodes_by_name(results, ["users_rollup", "users", "emails_alt"])
+
     def test_concat_exclude(self, project):
         results = run_dbt(
             [
@@ -177,6 +183,22 @@ def test_concat_exclude(self, project):
         )
         check_result_nodes_by_name(results, ["users_rollup", "users"])
 
+    def test_concat_exclude_multiple(self, project):
+        results = run_dbt(
+            [
+                "run",
+                "--select",
+                "@emails_alt",
+                "users_rollup",
+                "--exclude",
+                "users",
+                "--exclude",
+                "emails_alt",
+            ],
+            expect_pass=False,
+        )
+        check_result_nodes_by_name(results, ["users_rollup"])
+
     def test_concat_exclude_concat(self, project):
         results = run_dbt(
             [

tests/functional/list/test_list.py

@@ -557,6 +557,51 @@ def expect_select(self):
             ["--select", "config.incremental_strategy:insert_overwrite"], expect_pass=True
         )
 
+    def expect_resource_type_multiple(self):
+        """Expect selected resources when --resource-type given multiple times"""
+        results = self.run_dbt_ls(["--resource-type", "test", "--resource-type", "model"])
+        assert set(results) == {
+            "test.ephemeral",
+            "test.incremental",
+            "test.not_null_outer_id",
+            "test.outer",
+            "test.sub.inner",
+            "test.t",
+            "test.unique_outer_id",
+        }
+
+        results = self.run_dbt_ls(
+            ["--resource-type", "test", "--resource-type", "model", "--exclude", "unique_outer_id"]
+        )
+        assert set(results) == {
+            "test.ephemeral",
+            "test.incremental",
+            "test.not_null_outer_id",
+            "test.outer",
+            "test.sub.inner",
+            "test.t",
+        }
+
+        results = self.run_dbt_ls(
+            [
+                "--resource-type",
+                "test",
+                "--resource-type",
+                "model",
+                "--select",
+                "+inner",
+                "outer+",
+                "--exclude",
+                "inner",
+            ]
+        )
+        assert set(results) == {
+            "test.ephemeral",
+            "test.not_null_outer_id",
+            "test.unique_outer_id",
+            "test.outer",
+        }
+
     def expect_selected_keys(self, project):
         """Expect selected fields of the the selected model"""
         expectations = [
@@ -576,6 +621,26 @@ def expect_selected_keys(self, project):
         )
         assert len(results) == len(expectations)
 
+        for got, expected in zip(results, expectations):
+            self.assert_json_equal(got, expected)
+
+        """Expect selected fields when --output-keys given multiple times
+        """
+        expectations = [{"database": project.database, "schema": project.test_schema}]
+        results = self.run_dbt_ls(
+            [
+                "--model",
+                "inner",
+                "--output",
+                "json",
+                "--output-keys",
+                "database",
+                "--output-keys",
+                "schema",
+            ]
+        )
+        assert len(results) == len(expectations)
+
         for got, expected in zip(results, expectations):
             self.assert_json_equal(got, expected)
 
@@ -631,6 +696,7 @@ def test_ls(self, project):
         self.expect_seed_output()
         self.expect_test_output()
         self.expect_select()
+        self.expect_resource_type_multiple()
         self.expect_all_output()
         self.expect_selected_keys(project)