From 5261bfe1a32b4318a0b2cee189ab25a4644090d9 Mon Sep 17 00:00:00 2001 From: Justin Baldwin <91483530+justbldwn@users.noreply.github.com> Date: Wed, 25 Jan 2023 15:55:56 -0500 Subject: [PATCH 01/12] :sparkles: adding installed_packages.json functionality --- core/dbt/task/deps.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/dbt/task/deps.py b/core/dbt/task/deps.py index ac6a6c41af3..684b89ab1a8 100644 --- a/core/dbt/task/deps.py +++ b/core/dbt/task/deps.py @@ -1,3 +1,4 @@ +import json from typing import Optional import dbt.utils @@ -60,6 +61,8 @@ def run(self) -> None: fire_event(DepsNoPackagesFound()) return + packages_installed = {} + with downloads_directory(): final_deps = resolve_packages(packages, self.config) @@ -73,7 +76,12 @@ def run(self) -> None: fire_event(DepsStartPackageInstall(package_name=package_name)) package.install(self.config, renderer) + + # add installed package metadata to write to installed_packages.json at the end + packages_installed[package_name] = {"source": source_type, "version": version} + fire_event(DepsInstallInfo(version_name=package.nice_version_name())) + if isinstance(package, RegistryPinnedPackage): version_latest = package.get_version_latest() if version_latest != version: @@ -91,6 +99,11 @@ def run(self) -> None: fire_event(Formatting("")) fire_event(DepsNotifyUpdatesAvailable(packages=ListOfStrings(packages_to_upgrade))) + with open( + f"{self.config.packages_install_path}/installed_packages.json", "w" + ) as json_output: + json.dump(packages_installed, json_output, indent=4) + @classmethod def from_args(cls, args): # deps needs to move to the project directory, as it does put files From 5507a0ec2b9c00acce86dcee50cb409f542eddd6 Mon Sep 17 00:00:00 2001 From: Justin Baldwin <91483530+justbldwn@users.noreply.github.com> Date: Wed, 25 Jan 2023 16:52:24 -0500 Subject: [PATCH 02/12] :white_check_mark: update test_simple_dependency_deps test --- tests/functional/dependencies/test_simple_dependency.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/functional/dependencies/test_simple_dependency.py b/tests/functional/dependencies/test_simple_dependency.py index 140966e1c07..b5e45a99cc7 100644 --- a/tests/functional/dependencies/test_simple_dependency.py +++ b/tests/functional/dependencies/test_simple_dependency.py @@ -203,7 +203,12 @@ def packages(self): def test_simple_dependency_deps(self, project): run_dbt(["deps"]) - assert len(os.listdir("dbt_packages")) == 2 + + dbt_packages = os.listdir("dbt_packages") + dbt_packages_without_json = [x for x in dbt_packages if x != "installed_packages.json"] + + assert len(dbt_packages_without_json) == 2 + assert len(dbt_packages) == 3 class DependencyBranchBase(object): From f0cf216e02d4c381218e2536e59cdaa75aa70ce0 Mon Sep 17 00:00:00 2001 From: Justin Baldwin <91483530+justbldwn@users.noreply.github.com> Date: Wed, 25 Jan 2023 17:00:03 -0500 Subject: [PATCH 03/12] :memo: adding changelog for deps feature via changie --- .changes/unreleased/Features-20230125-165933.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20230125-165933.yaml diff --git a/.changes/unreleased/Features-20230125-165933.yaml b/.changes/unreleased/Features-20230125-165933.yaml new file mode 100644 index 00000000000..28d8132d5a4 --- /dev/null +++ b/.changes/unreleased/Features-20230125-165933.yaml @@ -0,0 +1,6 @@ +kind: Features +body: add log file of installed packages via dbt deps +time: 2023-01-25T16:59:33.786304-05:00 +custom: + Author: jusbaldw + Issue: "6643" From 8ae24e4e91d2e63aea27e53bb21ee553a6c2dfe4 Mon Sep 17 00:00:00 2001 From: Justin Baldwin <91483530+justbldwn@users.noreply.github.com> Date: Thu, 9 Feb 2023 19:35:00 -0500 Subject: [PATCH 04/12] :sparkles: restructure deps command, include lock/add --- core/dbt/cli/main.py | 31 +++++- core/dbt/cli/params.py | 22 ++++ core/dbt/config/project.py | 7 +- core/dbt/deps/resolver.py | 12 +++ core/dbt/events/proto_types.py | 67 +++++++++++++ core/dbt/events/types.py | 45 +++++++++ core/dbt/main.py | 52 +++++++++- core/dbt/task/deps.py | 178 +++++++++++++++++++++++++++++---- 8 files changed, 388 insertions(+), 26 deletions(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 9942db702ca..dcb8aa38767 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -200,15 +200,40 @@ def debug(ctx, **kwargs): # dbt deps -@cli.command("deps") +@cli.group() +@click.pass_context +def deps(ctx, **kwargs): + """Pull the most recent version of the dependencies listed in packages.yml""" + + +# dbt deps lock +@deps.command("lock") @click.pass_context @p.profile @p.profiles_dir @p.project_dir @p.target @p.vars -def deps(ctx, **kwargs): - """Pull the most recent version of the dependencies listed in packages.yml""" +def deps_lock(ctx, **kwargs): + """Pull the most recent version of the dependencies listed in packages.yml into package-lock.yml file""" + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + + +# dbt deps add +@deps.command("add") +@click.pass_context +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.package +@p.package_version +@p.source +@p.dry_run +def deps_add(ctx, **kwargs): + """Add a new package to the packages.yml file""" flags = Flags() click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 3ad3747e962..3661a329d2e 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -74,6 +74,14 @@ help="If set, defer to the state variable for resolving unselected nodes.", ) +dry_run = click.option( + "--dry-run", + envvar=None, + help="Option to run `dbt deps add` without updating package-lock.yml file.", + default=False, + type=click.BOOL, +) + enable_legacy_logger = click.option( "--enable-legacy-logger/--no-enable-legacy-logger", envvar="DBT_ENABLE_LEGACY_LOGGER", @@ -162,6 +170,12 @@ default=PurePath.joinpath(Path.cwd(), "target/sources.json"), ) +package = click.option("--package", envvar=None, help="Name of package to add to packages.yml.") + +package_version = click.option( + "--version/--package_version", envvar=None, help="Version of the package to install." +) + parse_only = click.option( "--parse-only", envvar=None, @@ -273,6 +287,14 @@ "--skip-profile-setup", "-s", envvar=None, help="Skip interactive profile setup.", is_flag=True ) +source = click.option( + "--source", + envvar=None, + help="Source to download page from, must be one of hub, git, or local. Defaults to hub.", + type=click.Choice(["hub", "git", "local"], case_sensitive=True), + default="hub", +) + # TODO: The env var and name (reflected in flags) are corrections! # The original name was `ARTIFACT_STATE_PATH` and used the env var `DBT_ARTIFACT_STATE_PATH`. # Both of which break existing naming conventions. diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 861785cf0c9..18283c26407 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -93,13 +93,14 @@ def _load_yaml(path): return load_yaml_text(contents) -def package_data_from_root(project_root): - package_filepath = resolve_path_from_base("packages.yml", project_root) +def package_data_from_root(project_root, package_file_name="packages.yml"): + package_filepath = resolve_path_from_base(package_file_name, project_root) if path_exists(package_filepath): packages_dict = _load_yaml(package_filepath) else: packages_dict = None + return packages_dict @@ -490,6 +491,7 @@ def from_project_root( project_root = os.path.normpath(project_root) project_dict = _raw_project_from(project_root) config_version = project_dict.get("config-version", 1) + if config_version != 2: raise DbtProjectError( f"Invalid config version: {config_version}, expected 2", @@ -498,6 +500,7 @@ def from_project_root( packages_dict = package_data_from_root(project_root) selectors_dict = selector_data_from_root(project_root) + return cls.from_dicts( project_root=project_root, project_dict=project_dict, diff --git a/core/dbt/deps/resolver.py b/core/dbt/deps/resolver.py index db57ef0f641..1e2cb654b5f 100644 --- a/core/dbt/deps/resolver.py +++ b/core/dbt/deps/resolver.py @@ -132,3 +132,15 @@ def resolve_packages( resolved = final.resolved() _check_for_duplicate_project_names(resolved, config, renderer) return resolved + + +def resolve_lock_packages(packages: List[PackageContract]) -> List[PinnedPackage]: + lock_packages = PackageListing.from_contracts(packages) + final = PackageListing() + + for package in lock_packages: + final.incorporate(package) + + resolved = final.resolved() + + return resolved diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index ee11e01d172..e6b56d74395 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -1807,6 +1807,73 @@ class NoNodesForSelectionCriteriaMsg(betterproto.Message): data: "NoNodesForSelectionCriteria" = betterproto.message_field(2) +@dataclass +class DepsLockCreated(betterproto.Message): + """M031""" + + lock_filepath: str = betterproto.string_field(1) + + +@dataclass +class DepsLockCreatedMsg(betterproto.Message): + info: "EventInfo" = betterproto.message_field(1) + data: "DepsLockCreated" = betterproto.message_field(2) + + +@dataclass +class DepsLockUpdating(betterproto.Message): + """M032""" + + lock_filepath: str = betterproto.string_field(1) + + +@dataclass +class DepsLockUpdatingMsg(betterproto.Message): + info: "EventInfo" = betterproto.message_field(1) + data: "DepsLockUpdating" = betterproto.message_field(2) + + +@dataclass +class DepsAddPackage(betterproto.Message): + """M033""" + + package_name: str = betterproto.string_field(1) + version: str = betterproto.string_field(2) + packages_filepath: str = betterproto.string_field(3) + + +@dataclass +class DepsAddPackageMsg(betterproto.Message): + info: "EventInfo" = betterproto.message_field(1) + data: "DepsAddPackage" = betterproto.message_field(2) + + +@dataclass +class DepsFoundDuplicatePackage(betterproto.Message): + """M034""" + + removed_package: dict = betterproto.string_field(1) + + +@dataclass +class DepsFoundDuplicatePackageMsg(betterproto.Message): + info: "EventInfo" = betterproto.message_field(1) + data: "DepsFoundDuplicatePackage" = betterproto.message_field(2) + + +@dataclass +class DepsVersionMissing(betterproto.Message): + """M035""" + + source: str = betterproto.string_field(1) + + +@dataclass +class DepsVersionMissingMsg(betterproto.Message): + info: "EventInfo" = betterproto.message_field(1) + data: "DepsVersionMissing" = betterproto.message_field(2) + + @dataclass class RunningOperationCaughtError(betterproto.Message): """Q001""" diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index ab2d090a93a..48362c5f27d 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -1457,6 +1457,51 @@ def message(self) -> str: return f"The selection criterion '{self.spec_raw}' does not match any nodes" +@dataclass +class DepsLockCreated(InfoLevel, pt.DepsLockCreated): + def code(self): + return "M031" + + def message(self) -> str: + return f"Created lock file in file path: {self.lock_filepath}" + + +@dataclass +class DepsLockUpdating(InfoLevel, pt.DepsLockUpdating): + def code(self): + return "M032" + + def message(self) -> str: + return f"Updating lock file in file path: {self.lock_filepath}" + + +@dataclass +class DepsAddPackage(InfoLevel, pt.DepsAddPackage): + def code(self): + return "M033" + + def message(self) -> str: + return f"Added new package {self.package_name}@{self.version} to {self.packages_filepath}" + + +@dataclass +class DepsFoundDuplicatePackage(InfoLevel, pt.DepsFoundDuplicatePackage): + def code(self): + return "M034" + + def message(self) -> str: + return f"Found duplicate package in packages.yml, removing: {self.removed_package}" + + +@dataclass +class DepsVersionMissing(InfoLevel, pt.DepsVersionMissing): + def code(self): + return "M035" + + def message(self) -> str: + return f"Version is required to add a package when source is {self.source}" + + # ======================================================= # Q - Node execution # ======================================================= diff --git a/core/dbt/main.py b/core/dbt/main.py index 429d823be52..e72b927a9e8 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -465,6 +465,53 @@ def _build_deps_subparser(subparsers, base_subparser): return sub +def _build_deps_lock_subparser(subparsers, base_subparser): + # it might look like docs_sub is the correct parents entry, but that + # will cause weird errors about 'conflicting option strings'. + lock_sub = subparsers.add_parser( + "lock", + parents=[base_subparser], + help="""Build lock file based on latest installed packages in packages.yml""", + ) + lock_sub.set_defaults(cls=deps_task.LockTask, which="lock", rpc_method="deps.lock") + + _add_defer_arguments(lock_sub) + return lock_sub + + +def _build_deps_add_subparser(subparsers, base_subparser): + add_sub = subparsers.add_parser( + "add", parents=[base_subparser], help="Add a package to packages.yml" + ) + + add_sub.add_argument( + "--package", help="Name of package to add to packages.yml.", required=True + ) + + # Can't make version required because local source package install doesn't require version. + # A check & event type exists in `dbt deps add` + # to ensure a version is passed in when source isn't local. + add_sub.add_argument("--version", help="Version of the package to install.") + + add_sub.add_argument( + "--source", + choices=["hub", "git", "local"], + default="hub", + help="Source to download page from, must be one of hub, git, or local. Defaults to hub.", + ) + + add_sub.add_argument( + "--dry-run", + type=bool, + default=False, + help="Option to run `dbt deps add` without updating package-lock.yml file.", + ) + + add_sub.set_defaults(cls=deps_task.AddTask, which="deps", rpc_method="deps.add") + + return add_sub + + def _build_snapshot_subparser(subparsers, base_subparser): sub = subparsers.add_parser( "snapshot", @@ -1165,11 +1212,14 @@ def parse_args(args, cls=DBTArgumentParser): docs_subs = docs_sub.add_subparsers(title="Available sub-commands") source_sub = _build_source_subparser(subs, base_subparser) source_subs = source_sub.add_subparsers(title="Available sub-commands") + deps_sub = _build_deps_subparser(subs, base_subparser) + deps_subs = deps_sub.add_subparsers(title="Available sub-commands") + _build_deps_lock_subparser(deps_subs, base_subparser) + _build_deps_add_subparser(deps_subs, base_subparser) _build_init_subparser(subs, base_subparser) _build_clean_subparser(subs, base_subparser) _build_debug_subparser(subs, base_subparser) - _build_deps_subparser(subs, base_subparser) _build_list_subparser(subs, base_subparser) build_sub = _build_build_subparser(subs, base_subparser) diff --git a/core/dbt/task/deps.py b/core/dbt/task/deps.py index 684b89ab1a8..2b2de36702f 100644 --- a/core/dbt/task/deps.py +++ b/core/dbt/task/deps.py @@ -1,5 +1,5 @@ -import json from typing import Optional +import yaml import dbt.utils import dbt.deprecations @@ -7,20 +7,26 @@ from dbt.config import UnsetProfileConfig from dbt.config.renderer import DbtProjectYamlRenderer +from dbt.config.project import package_config_from_data, package_data_from_root from dbt.deps.base import downloads_directory -from dbt.deps.resolver import resolve_packages +from dbt.deps.resolver import resolve_lock_packages, resolve_packages from dbt.deps.registry import RegistryPinnedPackage from dbt.events.proto_types import ListOfStrings from dbt.events.functions import fire_event from dbt.events.types import ( + DepsAddPackage, + DepsFoundDuplicatePackage, + DepsInstallInfo, + DepsListSubdirectory, + DepsLockCreated, + DepsLockUpdating, DepsNoPackagesFound, + DepsNotifyUpdatesAvailable, DepsStartPackageInstall, DepsUpdateAvailable, DepsUpToDate, - DepsInstallInfo, - DepsListSubdirectory, - DepsNotifyUpdatesAvailable, + DepsVersionMissing, Formatting, ) from dbt.clients import system @@ -28,12 +34,46 @@ from dbt.task.base import BaseTask, move_to_nearest_project_dir +def _create_packages_yml_entry(package, version, source): + """Create a formatted entry to add to `packages.yml` or `package-lock.yml` file + + Args: + package (str): Name of package to download + version (str): Version of package to download + source (str): Source of where to download package from + + Returns: + dict: Formatted dict to write to `packages.yml` or `package-lock.yml` file + """ + package_key = source + version_key = "version" + + if source == "hub": + package_key = "package" + + if source == "git": + version_key = "revision" + + packages_yml_entry = {package_key: package} + + if version: + if "," in version: + version = version.split(",") + + packages_yml_entry[version_key] = version + + return packages_yml_entry + + class DepsTask(BaseTask): ConfigType = UnsetProfileConfig def __init__(self, args, config: UnsetProfileConfig): super().__init__(args=args, config=config) + if not system.path_exists(f"{self.config.project_root}/package-lock.yml"): + LockTask(args, config).run() + def track_package_install( self, package_name: str, source_type: str, version: Optional[str] ) -> None: @@ -55,21 +95,25 @@ def track_package_install( ) def run(self) -> None: + if system.path_exists(self.config.packages_install_path): + system.rmtree(self.config.packages_install_path) + system.make_directory(self.config.packages_install_path) - packages = self.config.packages.packages - if not packages: + + packages_lock_dict = package_data_from_root(self.config.project_root, "package-lock.yml") + packages_lock_config = package_config_from_data(packages_lock_dict).packages + + if not packages_lock_config: fire_event(DepsNoPackagesFound()) return - packages_installed = {} - with downloads_directory(): - final_deps = resolve_packages(packages, self.config) - + lock_defined_deps = resolve_lock_packages(packages_lock_config) renderer = DbtProjectYamlRenderer(self.config, self.config.cli_vars) packages_to_upgrade = [] - for package in final_deps: + + for package in lock_defined_deps: package_name = package.name source_type = package.source_type() version = package.get_version() @@ -77,36 +121,130 @@ def run(self) -> None: fire_event(DepsStartPackageInstall(package_name=package_name)) package.install(self.config, renderer) - # add installed package metadata to write to installed_packages.json at the end - packages_installed[package_name] = {"source": source_type, "version": version} - fire_event(DepsInstallInfo(version_name=package.nice_version_name())) if isinstance(package, RegistryPinnedPackage): version_latest = package.get_version_latest() + if version_latest != version: packages_to_upgrade.append(package_name) fire_event(DepsUpdateAvailable(version_latest=version_latest)) else: fire_event(DepsUpToDate()) + if package.get_subdirectory(): fire_event(DepsListSubdirectory(subdirectory=package.get_subdirectory())) self.track_package_install( package_name=package_name, source_type=source_type, version=version ) + if packages_to_upgrade: fire_event(Formatting("")) fire_event(DepsNotifyUpdatesAvailable(packages=ListOfStrings(packages_to_upgrade))) - with open( - f"{self.config.packages_install_path}/installed_packages.json", "w" - ) as json_output: - json.dump(packages_installed, json_output, indent=4) - @classmethod def from_args(cls, args): # deps needs to move to the project directory, as it does put files # into the modules directory move_to_nearest_project_dir(args) return super().from_args(args) + + +class LockTask(BaseTask): + ConfigType = UnsetProfileConfig + + def __init__(self, args, config: UnsetProfileConfig): + super().__init__(args=args, config=config) + + def run(self): + lock_filepath = f"{self.config.project_root}/package-lock.yml" + + packages = self.config.packages.packages + packages_installed = {"packages": []} + + if not packages: + fire_event(DepsNoPackagesFound()) + return + + with downloads_directory(): + resolved_deps = resolve_packages(packages, self.config) + + # this loop is to create the package-lock.yml in the same format as original packages.yml + # package-lock.yml includes both the stated packages in packages.yml along with dependent packages + for package in resolved_deps: + lock_entry = _create_packages_yml_entry( + package.name, package.get_version(), package.source_type() + ) + packages_installed["packages"].append(lock_entry) + + with open(lock_filepath, "w") as lock_obj: + yaml.safe_dump(packages_installed, lock_obj) + + fire_event(DepsLockCreated(lock_filepath=lock_filepath)) + + +class AddTask(BaseTask): + ConfigType = UnsetProfileConfig + + def __init__(self, args, config: UnsetProfileConfig): + super().__init__(args=args, config=config) + + def check_for_duplicate_packages(self, packages_yml): + """Loop through contents of `packages.yml` to ensure no duplicate package names + versions. + + This duplicate check will take into consideration exact match of a package name, as well as + a check to see if a package name exists within a name (i.e. a package name inside a git URL). + + Args: + packages_yml (dict): In-memory read of `packages.yml` contents + + Returns: + dict: Updated or untouched packages_yml contents + """ + for i, pkg_entry in enumerate(packages_yml["packages"]): + for val in pkg_entry.values(): + if self.args.package in val: + del packages_yml["packages"][i] + + fire_event(DepsFoundDuplicatePackage(removed_package=pkg_entry)) + + return packages_yml + + def run(self): + packages_yml_filepath = f"{self.config.project_root}/packages.yml" + + if not system.path_exists(packages_yml_filepath): + fire_event(DepsNoPackagesFound()) + return + + if not self.args.version and self.args.source != "local": + fire_event(DepsVersionMissing(source=self.args.source)) + return + + new_package_entry = _create_packages_yml_entry( + self.args.package, self.args.version, self.args.source + ) + + with open(packages_yml_filepath, "r") as user_yml_obj: + packages_yml = yaml.safe_load(user_yml_obj) + packages_yml = self.check_for_duplicate_packages(packages_yml) + packages_yml["packages"].append(new_package_entry) + + if packages_yml: + with open(packages_yml_filepath, "w") as pkg_obj: + yaml.safe_dump(packages_yml, pkg_obj) + + fire_event( + DepsAddPackage( + package_name=self.args.package, + version=self.args.version, + packages_filepath=packages_yml_filepath, + ) + ) + + if not self.args.dry_run: + fire_event( + DepsLockUpdating(lock_filepath=f"{self.config.project_root}/package-lock.yml") + ) + LockTask(self.args, self.config).run() From 94d208f076b78282d769899456719bbcd7d92d68 Mon Sep 17 00:00:00 2001 From: Justin Baldwin <91483530+justbldwn@users.noreply.github.com> Date: Thu, 9 Feb 2023 19:43:46 -0500 Subject: [PATCH 05/12] :white_check_mark: add new deps event types to sample_values --- tests/unit/test_events.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 529a11f5ed9..d785460dd5a 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -256,6 +256,11 @@ def test_event_codes(self): RegistryResponseMissingNestedKeys(response=""), RegistryResponseExtraNestedKeys(response=""), DepsSetDownloadDirectory(path=""), + DepsLockCreated(lock_filepath=""), + DepsLockUpdating(lock_filepath=""), + DepsAddPackage(package_name="", version="", packages_filepath=""), + DepsFoundDuplicatePackage(removed_package={}), + DepsVersionMissing(source=""), # Q - Node execution ====================== RunningOperationCaughtError(exc=""), CompileComplete(), From 948d4003e943b4343e02120ad7b2c3b807106f70 Mon Sep 17 00:00:00 2001 From: Justin Baldwin <91483530+justbldwn@users.noreply.github.com> Date: Thu, 9 Feb 2023 20:57:39 -0500 Subject: [PATCH 06/12] :white_check_mark: fix test_simple_dependency_deps test --- tests/functional/dependencies/test_simple_dependency.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/functional/dependencies/test_simple_dependency.py b/tests/functional/dependencies/test_simple_dependency.py index b5e45a99cc7..140966e1c07 100644 --- a/tests/functional/dependencies/test_simple_dependency.py +++ b/tests/functional/dependencies/test_simple_dependency.py @@ -203,12 +203,7 @@ def packages(self): def test_simple_dependency_deps(self, project): run_dbt(["deps"]) - - dbt_packages = os.listdir("dbt_packages") - dbt_packages_without_json = [x for x in dbt_packages if x != "installed_packages.json"] - - assert len(dbt_packages_without_json) == 2 - assert len(dbt_packages) == 3 + assert len(os.listdir("dbt_packages")) == 2 class DependencyBranchBase(object): From a3df2806e513c8aaaabf153355e14c43f21183fa Mon Sep 17 00:00:00 2001 From: Justin Baldwin <91483530+justbldwn@users.noreply.github.com> Date: Thu, 9 Feb 2023 23:16:37 -0500 Subject: [PATCH 07/12] :bug: attempting to fix cli commands --- core/dbt/cli/main.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 7d43a98c1b3..50517779b31 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -8,7 +8,7 @@ from dbt.contracts.graph.manifest import Manifest from dbt.task.clean import CleanTask from dbt.task.compile import CompileTask -from dbt.task.deps import AddTask, LockTask, DepsTask +from dbt.task.deps import AddTask, DepsTask, LockTask from dbt.task.debug import DebugTask from dbt.task.run import RunTask from dbt.task.serve import ServeTask @@ -289,8 +289,13 @@ def debug(ctx, **kwargs): # dbt deps -@cli.group() +@cli.group(invoke_without_command=True) @click.pass_context +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars @requires.preflight @requires.unset_profile @requires.project @@ -305,11 +310,6 @@ def deps(ctx, **kwargs): # dbt deps lock @deps.command("lock") @click.pass_context -@p.profile -@p.profiles_dir -@p.project_dir -@p.target -@p.vars def deps_lock(ctx, **kwargs): """Pull the most recent version of the dependencies listed in packages.yml into package-lock.yml file""" task = LockTask(ctx.obj["flags"], ctx.obj["project"]) @@ -317,18 +317,10 @@ def deps_lock(ctx, **kwargs): success = task.interpret_results(results) return results, success - # flags = Flags() - # click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") - # dbt deps add @deps.command("add") @click.pass_context -@p.profile -@p.profiles_dir -@p.project_dir -@p.target -@p.vars @p.package @p.package_version @p.source @@ -340,9 +332,6 @@ def deps_add(ctx, **kwargs): success = task.interpret_results(results) return results, success - # flags = Flags() - # click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") - # dbt init @cli.command("init") From 0e01c2e825249e73e6f6d97efa00ad10963f18da Mon Sep 17 00:00:00 2001 From: Justin Baldwin <91483530+justbldwn@users.noreply.github.com> Date: Fri, 10 Feb 2023 10:34:44 -0500 Subject: [PATCH 08/12] :bug: convert dbt deps to dbt deps install also leave dbt deps as just a new click group --- core/dbt/cli/main.py | 28 +++++++++++++++++++++++++--- core/dbt/cli/params.py | 6 ++++-- core/dbt/task/deps.py | 2 +- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 50517779b31..06480c903f4 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -289,7 +289,13 @@ def debug(ctx, **kwargs): # dbt deps -@cli.group(invoke_without_command=True) +@cli.group() +@click.pass_context +def deps(ctx, **kwargs): + """Pull the most recent version of the dependencies listed in packages.yml""" + + +@deps.command("install") @click.pass_context @p.profile @p.profiles_dir @@ -299,8 +305,8 @@ def debug(ctx, **kwargs): @requires.preflight @requires.unset_profile @requires.project -def deps(ctx, **kwargs): - """Pull the most recent version of the dependencies listed in packages.yml""" +def deps_install(ctx, **kwargs): + """Install the most recent version of the dependencies listed in packages.yml""" task = DepsTask(ctx.obj["flags"], ctx.obj["project"]) results = task.run() success = task.interpret_results(results) @@ -310,6 +316,14 @@ def deps(ctx, **kwargs): # dbt deps lock @deps.command("lock") @click.pass_context +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@requires.preflight +@requires.unset_profile +@requires.project def deps_lock(ctx, **kwargs): """Pull the most recent version of the dependencies listed in packages.yml into package-lock.yml file""" task = LockTask(ctx.obj["flags"], ctx.obj["project"]) @@ -321,10 +335,18 @@ def deps_lock(ctx, **kwargs): # dbt deps add @deps.command("add") @click.pass_context +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars @p.package @p.package_version @p.source @p.dry_run +@requires.preflight +@requires.unset_profile +@requires.project def deps_add(ctx, **kwargs): """Add a new package to the packages.yml file""" task = AddTask(ctx.obj["flags"], ctx.obj["project"]) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index c807b137f1d..98d04de56c5 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -168,10 +168,12 @@ default=PurePath.joinpath(Path.cwd(), "target/sources.json"), ) -package = click.option("--package", envvar=None, help="Name of package to add to packages.yml.") +package = click.option( + "--package", envvar=None, help="Name of package to add to packages.yml.", type=click.STRING +) package_version = click.option( - "--version/--package_version", envvar=None, help="Version of the package to install." + "--version", envvar=None, help="Version of the package to install.", type=click.STRING ) parse_only = click.option( diff --git a/core/dbt/task/deps.py b/core/dbt/task/deps.py index 6a598375c25..e880a113c97 100644 --- a/core/dbt/task/deps.py +++ b/core/dbt/task/deps.py @@ -154,7 +154,7 @@ def __init__(self, args: Any, project: Project): def run(self): lock_filepath = f"{self.project.project_root}/package-lock.yml" - packages = self.packages.packages.packages + packages = self.project.packages.packages packages_installed = {"packages": []} if not packages: From 9995193cbc87d4ddf77c7073feacb59d60f5f48b Mon Sep 17 00:00:00 2001 From: Justin Baldwin <91483530+justbldwn@users.noreply.github.com> Date: Fri, 10 Feb 2023 10:44:51 -0500 Subject: [PATCH 09/12] :white_check_mark: update test_command_mutually_exclusive_option change deps command to deps install --- tests/unit/test_dbt_runner.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_dbt_runner.py b/tests/unit/test_dbt_runner.py index 6c17de6dc8c..cc26534b23a 100644 --- a/tests/unit/test_dbt_runner.py +++ b/tests/unit/test_dbt_runner.py @@ -18,7 +18,10 @@ def test_command_invalid_option(self, dbt: dbtRunner) -> None: def test_command_mutually_exclusive_option(self, dbt: dbtRunner) -> None: with pytest.raises(dbtUsageException): - dbt.invoke(["--warn-error", "--warn-error-options", '{"include": "all"}', "deps"]) + # dbt.invoke(["--warn-error", "--warn-error-options", '{"include": "all"}', "deps"]) + dbt.invoke( + ["--warn-error", "--warn-error-options", '{"include": "all"}', "deps install"] + ) def test_invalid_command(self, dbt: dbtRunner) -> None: with pytest.raises(dbtUsageException): From e18282154f59d468a77a1b7cfda46214cd02939f Mon Sep 17 00:00:00 2001 From: Justin Baldwin <91483530+justbldwn@users.noreply.github.com> Date: Fri, 10 Feb 2023 11:13:43 -0500 Subject: [PATCH 10/12] :white_check_mark: update functional tests from deps > deps install --- tests/functional/context_methods/test_cli_vars.py | 2 +- .../context_methods/test_custom_env_vars.py | 2 +- .../context_methods/test_secret_env_vars.py | 6 +++--- .../context_methods/test_var_dependency.py | 2 +- .../dependencies/test_local_dependency.py | 14 +++++++------- .../dependencies/test_simple_dependency.py | 14 +++++++------- .../test_simple_dependency_with_configs.py | 4 ++-- tests/functional/deprecations/test_deprecations.py | 4 ++-- .../functional/duplicates/test_duplicate_model.py | 4 ++-- tests/functional/exit_codes/test_exit_codes.py | 4 ++-- .../test_schema_test_graph_selection.py | 2 +- tests/functional/macros/test_macros.py | 4 ++-- .../test_custom_materialization.py | 10 +++++----- tests/functional/minimal_cli/test_minimal_cli.py | 8 ++++---- .../partial_parsing/test_partial_parsing.py | 2 +- tests/functional/schema/test_custom_schema.py | 2 +- .../schema_tests/test_schema_v2_tests.py | 8 ++++---- .../test_simple_source_override.py | 2 +- .../test_source_overrides_duplicate_model.py | 2 +- tests/unit/test_dbt_runner.py | 2 +- 20 files changed, 49 insertions(+), 49 deletions(-) diff --git a/tests/functional/context_methods/test_cli_vars.py b/tests/functional/context_methods/test_cli_vars.py index 5f5b222f5da..eb2f9d4e4ba 100644 --- a/tests/functional/context_methods/test_cli_vars.py +++ b/tests/functional/context_methods/test_cli_vars.py @@ -138,7 +138,7 @@ def packages_config(self): def test_cli_vars_in_packages(self, project, packages_config): # Run working deps and run commands - run_dbt(["deps"]) + run_dbt(["deps", "install"]) results = run_dbt(["run"]) assert len(results) == 1 diff --git a/tests/functional/context_methods/test_custom_env_vars.py b/tests/functional/context_methods/test_custom_env_vars.py index e74a5dcee09..e9403d2b1ce 100644 --- a/tests/functional/context_methods/test_custom_env_vars.py +++ b/tests/functional/context_methods/test_custom_env_vars.py @@ -28,7 +28,7 @@ def setup(self): def test_extra_filled(self, project): _, log_output = run_dbt_and_capture( - ["--log-format=json", "deps"], + ["--log-format=json", "deps", "install"], ) logs = parse_json_logs(log_output) for log in logs: diff --git a/tests/functional/context_methods/test_secret_env_vars.py b/tests/functional/context_methods/test_secret_env_vars.py index 710c104f551..f1ef957bfae 100644 --- a/tests/functional/context_methods/test_secret_env_vars.py +++ b/tests/functional/context_methods/test_secret_env_vars.py @@ -106,7 +106,7 @@ def profile_target(self): } def test_allow_secrets(self, project, first_dependency): - _, log_output = run_dbt_and_capture(["deps"]) + _, log_output = run_dbt_and_capture(["deps", "install"]) assert not ("first_dependency" in log_output) @@ -131,7 +131,7 @@ def packages(self): def test_fail_clone_with_scrubbing(self, project): with pytest.raises(DbtInternalError) as excinfo: - _, log_output = run_dbt_and_capture(["deps"]) + _, log_output = run_dbt_and_capture(["deps", "install"]) assert "abc123" not in str(excinfo.value) @@ -150,7 +150,7 @@ def packages(self): def test_fail_clone_with_scrubbing(self, project): with pytest.raises(DbtInternalError) as excinfo: - _, log_output = run_dbt_and_capture(["deps"]) + _, log_output = run_dbt_and_capture(["deps", "install"]) # we should not see any manipulated form of the secret value (abc123) here # we should see a manipulated form of the placeholder instead diff --git a/tests/functional/context_methods/test_var_dependency.py b/tests/functional/context_methods/test_var_dependency.py index 6c3f85e94c4..c91a8cef869 100644 --- a/tests/functional/context_methods/test_var_dependency.py +++ b/tests/functional/context_methods/test_var_dependency.py @@ -47,7 +47,7 @@ def project_config_update(self): } def test_var_mutual_overrides_v1_conversion(self, project, first_dependency): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) assert len(run_dbt(["seed"])) == 2 assert len(run_dbt(["run"])) == 2 check_relations_equal(project.adapter, ["root_model_expected", "model"]) diff --git a/tests/functional/dependencies/test_local_dependency.py b/tests/functional/dependencies/test_local_dependency.py index 559a1b25586..976ec14bf38 100644 --- a/tests/functional/dependencies/test_local_dependency.py +++ b/tests/functional/dependencies/test_local_dependency.py @@ -133,7 +133,7 @@ def packages(self): class TestSimpleDependency(BaseDependencyTest): def test_local_dependency(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) run_dbt(["seed"]) results = run_dbt() assert len(results) == 5 @@ -156,7 +156,7 @@ def test_local_dependency(self, project): ) def test_no_dependency_paths(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) run_dbt(["seed"]) # prove dependency does not exist as model in project @@ -206,7 +206,7 @@ def project_config(self): @mock.patch("dbt.config.project.get_installed_version") def test_local_dependency_out_of_date(self, mock_get, project): mock_get.return_value = dbt.semver.VersionSpecifier.from_version_string("0.0.1") - run_dbt(["deps"] + self.dbt_vargs(project.test_schema)) + run_dbt(["deps", "install"] + self.dbt_vargs(project.test_schema)) # check seed with pytest.raises(dbt.exceptions.DbtProjectError) as exc: run_dbt(["seed"] + self.dbt_vargs(project.test_schema)) @@ -219,7 +219,7 @@ def test_local_dependency_out_of_date(self, mock_get, project): @mock.patch("dbt.config.project.get_installed_version") def test_local_dependency_out_of_date_no_check(self, mock_get): mock_get.return_value = dbt.semver.VersionSpecifier.from_version_string("0.0.1") - run_dbt(["deps"]) + run_dbt(["deps", "install"]) run_dbt(["seed", "--no-version-check"]) results = run_dbt(["run", "--no-version-check"]) assert len(results) == 5 @@ -261,7 +261,7 @@ def test_local_dependency_out_of_date_no_check(self, mock_get, project): ) mock_get.return_value = dbt.semver.VersionSpecifier.from_version_string("0.0.1") - run_dbt(["deps", "--vars", vars_arg]) + run_dbt(["deps", "install", "--vars", vars_arg]) run_dbt(["seed", "--vars", vars_arg]) results = run_dbt(["run", "--vars", vars_arg]) len(results) == 5 @@ -310,7 +310,7 @@ def test_hook_dependency(self, prepare_dependencies, project): } ) - run_dbt(["deps", "--vars", cli_vars]) + run_dbt(["deps", "install", "--vars", cli_vars]) results = run_dbt(["run", "--vars", cli_vars]) assert len(results) == 2 check_relations_equal(project.adapter, ["actual", "expected"]) @@ -334,7 +334,7 @@ def prepare_dependencies(self, project): def test_local_dependency_same_name(self, prepare_dependencies, project): with pytest.raises(dbt.exceptions.DependencyError): - run_dbt(["deps"], expect_pass=False) + run_dbt(["deps", "install"], expect_pass=False) def test_local_dependency_same_name_sneaky(self, prepare_dependencies, project): shutil.copytree("duplicate_dependency", "./dbt_packages/duplicate_dependency") diff --git a/tests/functional/dependencies/test_simple_dependency.py b/tests/functional/dependencies/test_simple_dependency.py index 140966e1c07..3fcb3f254c2 100644 --- a/tests/functional/dependencies/test_simple_dependency.py +++ b/tests/functional/dependencies/test_simple_dependency.py @@ -73,7 +73,7 @@ def packages(self): # These two functions included to enable override in ...NoProfile derived test class @pytest.fixture(scope="class") def run_deps(self, project): - return run_dbt(["deps"]) + return run_dbt(["deps", "install"]) @pytest.fixture(scope="function") def run_clean(self, project): @@ -111,7 +111,7 @@ class TestSimpleDependencyNoProfile(SimpleDependencyBase): @pytest.fixture(scope="class") def run_deps(self, project): with tempfile.TemporaryDirectory() as tmpdir: - result = run_dbt(["deps", "--profiles-dir", tmpdir]) + result = run_dbt(["deps", "install", "--profiles-dir", tmpdir]) return result @pytest.fixture(scope="class") @@ -159,7 +159,7 @@ def packages(self): } def test_simple_dependency(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) class TestSimpleDependencyWithDuplicates(object): @@ -180,7 +180,7 @@ def packages(self): } def test_simple_dependency_deps(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) class TestRekeyedDependencyWithSubduplicates(object): @@ -202,7 +202,7 @@ def packages(self): } def test_simple_dependency_deps(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) assert len(os.listdir("dbt_packages")) == 2 @@ -223,7 +223,7 @@ def packages(self): } def deps_run_assert_equality(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) results = run_dbt() assert len(results) == 4 @@ -302,5 +302,5 @@ def dbt_profile_target(self): def test_deps_bad_profile(self, project): del os.environ["PROFILE_TEST_HOST"] - run_dbt(["deps"]) + run_dbt(["deps", "install"]) run_dbt(["clean"]) diff --git a/tests/functional/dependencies/test_simple_dependency_with_configs.py b/tests/functional/dependencies/test_simple_dependency_with_configs.py index 86ab911a2b1..c5000fe1e14 100644 --- a/tests/functional/dependencies/test_simple_dependency_with_configs.py +++ b/tests/functional/dependencies/test_simple_dependency_with_configs.py @@ -63,7 +63,7 @@ def project_config_update(self): } def test_simple_dependency(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) results = run_dbt() assert len(results) == 5 @@ -100,7 +100,7 @@ def project_config_update(self): } def test_simple_dependency(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) results = run_dbt(["run"]) len(results) == 5 diff --git a/tests/functional/deprecations/test_deprecations.py b/tests/functional/deprecations/test_deprecations.py index a70b3687c69..1b5c5a149a7 100644 --- a/tests/functional/deprecations/test_deprecations.py +++ b/tests/functional/deprecations/test_deprecations.py @@ -126,7 +126,7 @@ def packages(self): def test_package_redirect(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == set() - run_dbt(["deps"]) + run_dbt(["deps", "install"]) expected = {"package-redirect"} assert expected == deprecations.active_deprecations @@ -135,7 +135,7 @@ def test_package_redirect_fail(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == set() with pytest.raises(dbt.exceptions.CompilationError) as exc: - run_dbt(["--warn-error", "deps"]) + run_dbt(["--warn-error", "deps", "install"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace expected_msg = "The `fishtown-analytics/dbt_utils` package is deprecated in favor of `dbt-labs/dbt_utils`" assert expected_msg in exc_str diff --git a/tests/functional/duplicates/test_duplicate_model.py b/tests/functional/duplicates/test_duplicate_model.py index 7a53fd6de63..154bd1dcc3d 100644 --- a/tests/functional/duplicates/test_duplicate_model.py +++ b/tests/functional/duplicates/test_duplicate_model.py @@ -106,7 +106,7 @@ def packages(self): return {"packages": [{"local": "local_dependency"}]} def test_duplicate_model_enabled_across_packages(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) message = "dbt found two models with the name" with pytest.raises(DuplicateResourceNameError) as exc: run_dbt(["run"]) @@ -131,7 +131,7 @@ def packages(self): return {"packages": [{"local": "local_dependency"}]} def test_duplicate_model_disabled_across_packages(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) results = run_dbt(["compile"]) assert len(results) == 1 diff --git a/tests/functional/exit_codes/test_exit_codes.py b/tests/functional/exit_codes/test_exit_codes.py index 44672beecae..dc7b69092d5 100644 --- a/tests/functional/exit_codes/test_exit_codes.py +++ b/tests/functional/exit_codes/test_exit_codes.py @@ -78,7 +78,7 @@ def packages(self): } def test_deps(self, project): - results = run_dbt(["deps"]) + results = run_dbt(["deps", "install"]) assert results is None @@ -96,7 +96,7 @@ def packages(self): def test_deps_fail(self, project): with pytest.raises(dbt.exceptions.GitCheckoutError) as exc: - run_dbt(["deps"]) + run_dbt(["deps", "install"]) expected_msg = "Error checking out spec='bad-branch'" assert expected_msg in str(exc.value) diff --git a/tests/functional/graph_selection/test_schema_test_graph_selection.py b/tests/functional/graph_selection/test_schema_test_graph_selection.py index 4cfa5ef51f9..39e12b4c259 100644 --- a/tests/functional/graph_selection/test_schema_test_graph_selection.py +++ b/tests/functional/graph_selection/test_schema_test_graph_selection.py @@ -9,7 +9,7 @@ def run_schema_and_assert(project, include, exclude, expected_tests): # deps must run before seed - run_dbt(["deps"]) + run_dbt(["deps", "install"]) run_dbt(["seed"]) results = run_dbt(["run", "--exclude", "never_selected"]) assert len(results) == 10 diff --git a/tests/functional/macros/test_macros.py b/tests/functional/macros/test_macros.py index e7f25acab3a..b03beedd2f4 100644 --- a/tests/functional/macros/test_macros.py +++ b/tests/functional/macros/test_macros.py @@ -66,7 +66,7 @@ def project_config_update(self): } def test_working_macros(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) results = run_dbt() assert len(results) == 6 @@ -198,7 +198,7 @@ def packages(self): } def test_overrides(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) run_dbt() run_dbt() diff --git a/tests/functional/materializations/test_custom_materialization.py b/tests/functional/materializations/test_custom_materialization.py index 838eb68bb01..fcb0adb261f 100644 --- a/tests/functional/materializations/test_custom_materialization.py +++ b/tests/functional/materializations/test_custom_materialization.py @@ -23,7 +23,7 @@ def packages(self): return {"packages": [{"local": "override-view-adapter-dep"}]} def test_adapter_dependency(self, project, override_view_adapter_dep): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) # this should error because the override is buggy run_dbt(["run"], expect_pass=False) @@ -34,7 +34,7 @@ def packages(self): return {"packages": [{"local": "override-view-default-dep"}]} def test_default_dependency(self, project, override_view_default_dep): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) # this should error because the override is buggy run_dbt(["run"], expect_pass=False) @@ -45,7 +45,7 @@ def packages(self): return {"packages": [{"local": "override-view-adapter-pass-dep"}]} def test_default_dependency(self, project, override_view_adapter_pass_dep): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) # this should pass because the override is ok run_dbt(["run"]) @@ -65,7 +65,7 @@ def project_config_update(self): def test_default_dependency( self, project, override_view_adapter_pass_dep, override_view_adapter_macros ): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) # this should error because the override is buggy run_dbt(["run"], expect_pass=False) @@ -76,6 +76,6 @@ def project_config_update(self): return {"macro-paths": ["override-view-return-no-relation"]} def test_default_dependency(self, project, override_view_return_no_relation): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) results = run_dbt(["run"], expect_pass=False) assert "did not explicitly return a list of relations" in results[0].message diff --git a/tests/functional/minimal_cli/test_minimal_cli.py b/tests/functional/minimal_cli/test_minimal_cli.py index ae8f40dcfcc..e501313609e 100644 --- a/tests/functional/minimal_cli/test_minimal_cli.py +++ b/tests/functional/minimal_cli/test_minimal_cli.py @@ -19,12 +19,12 @@ def test_clean(self, runner, project): assert "logs" in result.output def test_deps(self, runner, project): - result = runner.invoke(cli, ["deps"]) + result = runner.invoke(cli, ["deps", "install"]) assert "dbt-labs/dbt_utils" in result.output assert "1.0.0" in result.output def test_ls(self, runner, project): - runner.invoke(cli, ["deps"]) + runner.invoke(cli, ["deps", "install"]) ls_result = runner.invoke(cli, ["ls"]) assert "1 seed" in ls_result.output assert "1 model" in ls_result.output @@ -32,7 +32,7 @@ def test_ls(self, runner, project): assert "1 snapshot" in ls_result.output def test_build(self, runner, project): - runner.invoke(cli, ["deps"]) + runner.invoke(cli, ["deps", "install"]) result = runner.invoke(cli, ["build"]) # 1 seed, 1 model, 2 tests assert "PASS=4" in result.output @@ -44,7 +44,7 @@ def test_build(self, runner, project): assert "SKIP=1" in result.output def test_docs_generate(self, runner, project): - runner.invoke(cli, ["deps"]) + runner.invoke(cli, ["deps", "install"]) result = runner.invoke(cli, ["docs", "generate"]) assert "Building catalog" in result.output assert "Catalog written" in result.output diff --git a/tests/functional/partial_parsing/test_partial_parsing.py b/tests/functional/partial_parsing/test_partial_parsing.py index f70b2e0f9fa..68f727da4fd 100644 --- a/tests/functional/partial_parsing/test_partial_parsing.py +++ b/tests/functional/partial_parsing/test_partial_parsing.py @@ -441,7 +441,7 @@ def packages(self): def test_parsing_with_dependency(self, project): run_dbt(["clean"]) - run_dbt(["deps"]) + run_dbt(["deps", "install"]) run_dbt(["seed"]) run_dbt(["run"]) diff --git a/tests/functional/schema/test_custom_schema.py b/tests/functional/schema/test_custom_schema.py index 7262a79cce9..2c7566e3969 100644 --- a/tests/functional/schema/test_custom_schema.py +++ b/tests/functional/schema/test_custom_schema.py @@ -149,7 +149,7 @@ def test__postgres__custom_schema_with_prefix_and_dispatch( self, project, macros, project_config_update ): project.run_sql(_VALIDATION_SQL) - run_dbt(["deps"]) + run_dbt(["deps", "install"]) run_dbt(["seed"]) results = run_dbt(["run"]) assert len(results) == 3 diff --git a/tests/functional/schema_tests/test_schema_v2_tests.py b/tests/functional/schema_tests/test_schema_v2_tests.py index 7b80c5d3eb4..e8ebdb09bac 100644 --- a/tests/functional/schema_tests/test_schema_v2_tests.py +++ b/tests/functional/schema_tests/test_schema_v2_tests.py @@ -564,7 +564,7 @@ def test_schema_tests( self, project, ): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) results = run_dbt() assert len(results) == 4 @@ -752,7 +752,7 @@ def packages(self): def test_test_context_tests(self, project): # This test tests the the TestContext and TestMacroNamespace # are working correctly - run_dbt(["deps"]) + run_dbt(["deps", "install"]) results = run_dbt() assert len(results) == 3 @@ -841,7 +841,7 @@ def test_test_context_with_macro_namespace( ): # This test tests the the TestContext and TestMacroNamespace # are working correctly - run_dbt(["deps"]) + run_dbt(["deps", "install"]) results = run_dbt() assert len(results) == 3 @@ -1124,5 +1124,5 @@ def test_macro_resolution_test_namespace( # resolve to 'some_macro' from the 'dbt' namespace during static analysis, # if 'some_macro' also existed in an installed package, # leading to the macro being missing in the TestNamespace - run_dbt(["deps"]) + run_dbt(["deps", "install"]) run_dbt(["parse"]) diff --git a/tests/functional/source_overrides/test_simple_source_override.py b/tests/functional/source_overrides/test_simple_source_override.py index da1b4856e32..ed80726e609 100644 --- a/tests/functional/source_overrides/test_simple_source_override.py +++ b/tests/functional/source_overrides/test_simple_source_override.py @@ -93,7 +93,7 @@ def _set_updated_at_to(self, insert_id, delta, project): def test_source_overrides(self, project): insert_id = 101 - run_dbt(["deps"]) + run_dbt(["deps", "install"]) seed_results = run_dbt(["seed"]) assert len(seed_results) == 5 diff --git a/tests/functional/source_overrides/test_source_overrides_duplicate_model.py b/tests/functional/source_overrides/test_source_overrides_duplicate_model.py index e3cdebe4794..936f46c0ffb 100644 --- a/tests/functional/source_overrides/test_source_overrides_duplicate_model.py +++ b/tests/functional/source_overrides/test_source_overrides_duplicate_model.py @@ -55,7 +55,7 @@ def project_config_update(self): } def test_source_duplicate_overrides(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) with pytest.raises(CompilationError) as exc: run_dbt(["compile"]) diff --git a/tests/unit/test_dbt_runner.py b/tests/unit/test_dbt_runner.py index cc26534b23a..b51ef6c9616 100644 --- a/tests/unit/test_dbt_runner.py +++ b/tests/unit/test_dbt_runner.py @@ -20,7 +20,7 @@ def test_command_mutually_exclusive_option(self, dbt: dbtRunner) -> None: with pytest.raises(dbtUsageException): # dbt.invoke(["--warn-error", "--warn-error-options", '{"include": "all"}', "deps"]) dbt.invoke( - ["--warn-error", "--warn-error-options", '{"include": "all"}', "deps install"] + ["--warn-error", "--warn-error-options", '{"include": "all"}', "deps", "install"] ) def test_invalid_command(self, dbt: dbtRunner) -> None: From 5615910a5f81170127cabbe306b6795cc7f9d36e Mon Sep 17 00:00:00 2001 From: Justin Baldwin <91483530+justbldwn@users.noreply.github.com> Date: Fri, 10 Feb 2023 11:28:38 -0500 Subject: [PATCH 11/12] :white_check_mark: change missing deps to deps install --- tests/functional/context_methods/test_cli_vars.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/functional/context_methods/test_cli_vars.py b/tests/functional/context_methods/test_cli_vars.py index eb2f9d4e4ba..3bf8d560e75 100644 --- a/tests/functional/context_methods/test_cli_vars.py +++ b/tests/functional/context_methods/test_cli_vars.py @@ -149,10 +149,12 @@ def test_cli_vars_in_packages(self, project, packages_config): # Without vars args deps fails with pytest.raises(DbtRuntimeError): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) # With vars arg deps succeeds - results = run_dbt(["deps", "--vars", "path_to_project: dbt_integration_project"]) + results = run_dbt( + ["deps", "install", "--vars", "path_to_project: dbt_integration_project"] + ) assert results is None From 5eb279f8a3230f7397b3d7743e2ad5e080b6d0b8 Mon Sep 17 00:00:00 2001 From: Justin Baldwin <91483530+justbldwn@users.noreply.github.com> Date: Fri, 10 Feb 2023 11:45:05 -0500 Subject: [PATCH 12/12] :white_check_mark: convert adapter tests to deps install from deps --- tests/adapter/dbt/tests/adapter/utils/test_except.py | 2 +- tests/adapter/dbt/tests/adapter/utils/test_intersect.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/adapter/dbt/tests/adapter/utils/test_except.py b/tests/adapter/dbt/tests/adapter/utils/test_except.py index 2c058e91c2c..4235a05c985 100644 --- a/tests/adapter/dbt/tests/adapter/utils/test_except.py +++ b/tests/adapter/dbt/tests/adapter/utils/test_except.py @@ -53,7 +53,7 @@ def models(self): } def test_build_assert_equal(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) run_dbt(["build"]) check_relations_equal( diff --git a/tests/adapter/dbt/tests/adapter/utils/test_intersect.py b/tests/adapter/dbt/tests/adapter/utils/test_intersect.py index 737e317c6f2..477c12e110b 100644 --- a/tests/adapter/dbt/tests/adapter/utils/test_intersect.py +++ b/tests/adapter/dbt/tests/adapter/utils/test_intersect.py @@ -51,7 +51,7 @@ def models(self): } def test_build_assert_equal(self, project): - run_dbt(["deps"]) + run_dbt(["deps", "install"]) run_dbt(["build"]) check_relations_equal(