Check dbt-core version requirements when installing Hub packages

.changes/unreleased/Features-20220815-134312.yaml

@@ -0,0 +1,7 @@
+kind: Features
+body: Check dbt-core version requirements when installing Hub packages
+time: 2022-08-15T13:43:12.965143+01:00
+custom:
+  Author: jtcohen6
+  Issue: "5648"
+  PR: "5651"

core/dbt/clients/registry.py

@@ -14,6 +14,7 @@
 )
 from dbt.utils import memoized, _connection_exception_retry as connection_exception_retry
 from dbt import deprecations
+from dbt import semver
 import os
 
 if os.getenv("DBT_PACKAGE_HUB_URL"):
@@ -125,10 +126,44 @@ def package_version(package_name, version, registry_base_url=None) -> Dict[str,
     return response[version]
 
 
-def get_available_versions(package_name) -> List["str"]:
+def is_compatible_version(package_spec, dbt_version) -> bool:
+    require_dbt_version = package_spec.get("require_dbt_version")
+    if not require_dbt_version:
+        # if version requirements are missing or empty, assume any version is compatible
+        return True
+    else:
+        # determine whether dbt_version satisfies this package's require-dbt-version config
+        if not isinstance(require_dbt_version, list):
+            require_dbt_version = [require_dbt_version]
+        supported_versions = [
+            semver.VersionSpecifier.from_version_string(v) for v in require_dbt_version
+        ]
+        supported_range = semver.reduce_versions(*supported_versions)
+        return semver.versions_compatible(dbt_version, supported_range.start, supported_range.end)
+
+
+def get_compatible_versions(package_name, dbt_version, should_version_check) -> List["str"]:
     # returns a list of all available versions of a package
     response = package(package_name)
-    return list(response)
+
+    # if the user doesn't care about installing compatible versions, just return them all
+    if not should_version_check:
+        return list(response)
+
+    # otherwise, only return versions that are compatible with the installed version of dbt-core
+    else:
+        compatible_versions = [
+            pkg_version
+            for pkg_version, info in response.items()
+            if is_compatible_version(info, dbt_version)
+        ]
+
+        if not compatible_versions:
+            raise Exception(
+                f"No compatible versions of {package_name} found for dbt-core version {dbt_version}"
+            )
+
+        return compatible_versions
 
 
 def _get_index(registry_base_url=None):

core/dbt/deps/registry.py

@@ -3,6 +3,8 @@
 from typing import List
 
 from dbt import semver
+from dbt import flags
+from dbt.version import get_installed_version
 from dbt.clients import registry, system
 from dbt.contracts.project import (
     RegistryPackageMetadata,
@@ -125,12 +127,16 @@ def resolved(self) -> RegistryPinnedPackage:
             new_msg = "Version error for package {}: {}".format(self.name, e)
             raise DependencyException(new_msg) from e
 
-        available = registry.get_available_versions(self.package)
+        should_version_check = bool(flags.VERSION_CHECK)
+        dbt_version = get_installed_version()
+        compatible_versions = registry.get_compatible_versions(
+            self.package, dbt_version, should_version_check
+        )
         prerelease_version_specified = any(bool(version.prerelease) for version in self.versions)
         installable = semver.filter_installable(
-            available, self.install_prerelease or prerelease_version_specified
+            compatible_versions, self.install_prerelease or prerelease_version_specified
         )
-        available_latest = installable[-1]
+        latest_compatible = installable[-1]
 
         # for now, pick a version and then recurse. later on,
         # we'll probably want to traverse multiple options
@@ -140,5 +146,5 @@ def resolved(self) -> RegistryPinnedPackage:
         if not target:
             package_version_not_found(self.package, range_, installable)
         return RegistryPinnedPackage(
-            package=self.package, version=target, version_latest=available_latest
+            package=self.package, version=target, version_latest=latest_compatible
         )

core/dbt/deps/resolver.py

@@ -120,7 +120,6 @@ def resolve_packages(
 ) -> List[PinnedPackage]:
     pending = PackageListing.from_contracts(packages)
     final = PackageListing()
-
     renderer = DbtProjectYamlRenderer(config, config.cli_vars)
 
     while pending:

core/dbt/exceptions.py

@@ -778,9 +778,10 @@ def package_not_found(package_name):
 
 def package_version_not_found(package_name, version_range, available_versions):
     base_msg = (
-        "Could not find a matching version for package {}\n"
+        "Could not find a matching compatible version for package {}\n"
         "  Requested range: {}\n"
-        "  Available versions: {}"
+        "  Compatible versions: {}\n"
+        "  (Not shown: versions incompatible with installed version of dbt-core)"
     )
     raise_dependency_error(base_msg.format(package_name, version_range, available_versions))
 

test/unit/test_deps.py

@@ -6,6 +6,7 @@
 from dbt.deps.git import GitUnpinnedPackage
 from dbt.deps.local import LocalUnpinnedPackage
 from dbt.deps.registry import RegistryUnpinnedPackage
+from dbt.clients.registry import is_compatible_version
 from dbt.deps.resolver import resolve_packages
 from dbt.contracts.project import (
     LocalPackage,
@@ -15,6 +16,7 @@
 
 from dbt.contracts.project import PackageConfig
 from dbt.semver import VersionSpecifier
+from dbt.version import get_installed_version
 
 from dbt.dataclass_schema import ValidationError
 
@@ -114,13 +116,13 @@ def setUp(self):
         self.patcher = mock.patch('dbt.deps.registry.registry')
         self.registry = self.patcher.start()
         self.index_cached = self.registry.index_cached
-        self.get_available_versions = self.registry.get_available_versions
+        self.get_compatible_versions = self.registry.get_compatible_versions
         self.package_version = self.registry.package_version
 
         self.index_cached.return_value = [
             'dbt-labs-test/a',
         ]
-        self.get_available_versions.return_value = [
+        self.get_compatible_versions.return_value = [
             '0.1.2', '0.1.3', '0.1.4a1'
         ]
         self.package_version.return_value = {
@@ -235,9 +237,10 @@ def test_resolve_missing_version(self):
         with self.assertRaises(dbt.exceptions.DependencyException) as exc:
             a.resolved()
         msg = (
-            "Could not find a matching version for package "
+            "Could not find a matching compatible version for package "
             "dbt-labs-test/a\n  Requested range: =0.1.4, =0.1.4\n  "
-            "Available versions: ['0.1.2', '0.1.3']"
+            "Compatible versions: ['0.1.2', '0.1.3']\n  "
+            "(Not shown: versions incompatible with installed version of dbt-core)"
         )
         self.assertEqual(msg, str(exc.exception))
 
@@ -509,12 +512,19 @@ def __init__(self, packages):
     def index_cached(self, registry_base_url=None):
         return sorted(self.packages)
 
-    def get_available_versions(self, name):
+    def package(self, package_name, registry_base_url=None):
         try:
-            pkg = self.packages[name]
+            pkg = self.packages[package_name]
         except KeyError:
             return []
-        return list(pkg)
+        return pkg
+
+    def get_compatible_versions(self, package_name, dbt_version, should_version_check):
+        packages = self.package(package_name)
+        return [
+            pkg_version for pkg_version, info in packages.items()
+            if is_compatible_version(info, dbt_version)
+        ]
 
     def package_version(self, name, version):
         try:
@@ -524,7 +534,13 @@ def package_version(self, name, version):
 
 
 class TestPackageSpec(unittest.TestCase):
-    def setUp(self):
+    def setUp(self):    
+        dbt_version = get_installed_version()
+        next_version = dbt_version
+        next_version.minor = str(int(next_version.minor) + 1)
+        next_version.prerelease = None
+        require_next_version = ">" + next_version.to_version_string()
+
         self.patcher = mock.patch('dbt.deps.registry.registry')
         self.registry = self.patcher.start()
         self.mock_registry = MockRegistry(packages={
@@ -556,6 +572,22 @@ def setUp(self):
                         'extra': 'field',
                     },
                     'newfield': ['another', 'value'],
+                },
+                '0.2.0': {
+                    'id': 'dbt-labs-test/a/0.2.0',
+                    'name': 'a',
+                    'version': '0.2.0',
+                    'packages': [],
+                    '_source': {
+                        'blahblah': 'asdfas',
+                    },
+                    # this one shouldn't be picked!
+                    'require_dbt_version': require_next_version,
+                    'downloads': {
+                        'tarball': 'https://example.com/invalid-url!',
+                        'extra': 'field',
+                    },
+                    'newfield': ['another', 'value'],
                 }
             },
             'dbt-labs-test/b': {
@@ -577,7 +609,7 @@ def setUp(self):
         })
 
         self.registry.index_cached.side_effect = self.mock_registry.index_cached
-        self.registry.get_available_versions.side_effect = self.mock_registry.get_available_versions
+        self.registry.get_compatible_versions.side_effect = self.mock_registry.get_compatible_versions
         self.registry.package_version.side_effect = self.mock_registry.package_version
 
     def tearDown(self):