Package management additions

dbt/clients/registry.py

@@ -1,7 +1,7 @@
 import requests
 
 
-DEFAULT_REGISTRY_BASE_URL = 'http://127.0.0.1:4567/'
+DEFAULT_REGISTRY_BASE_URL = 'http://127.0.0.1:35729/'
 
 
 def _get_url(url, registry_base_url=None):

dbt/clients/system.py

@@ -5,6 +5,8 @@
 import shutil
 import subprocess
 import sys
+import tarfile
+import requests
 
 import dbt.compat
 
@@ -100,10 +102,6 @@ def write_file(path, contents=''):
 
 
 def rmdir(path):
-    """
-    Make a file at `path` assuming that the directory it resides in already
-    exists. The file is saved with contents `contents`
-    """
     return shutil.rmtree(path)
 
 
@@ -133,3 +131,29 @@ def run_cmd(cwd, cmd):
     logger.debug('STDERR: "{}"'.format(err))
 
     return out, err
+
+
+def download(url, path):
+    response = requests.get(url)
+
+    with open(path, 'wb') as handle:
+        for block in response.iter_content(1024*64):
+            handle.write(block)
+
+def rename(from_path, to_path, force=False):
+    if os.path.exists(to_path) and force:
+        rmdir(to_path)
+
+    os.rename(from_path, to_path)
+
+
+def untar_package(tar_path, dest_dir, rename_to=None):
+    tar_dir_name = None
+    with tarfile.open(tar_path, 'r') as tarball:
+        tarball.extractall(dest_dir)
+        tar_dir_name = os.path.commonprefix(tarball.getnames())
+
+    if rename_to:
+        downloaded_path = os.path.join(dest_dir, tar_dir_name)
+        desired_path = os.path.join(dest_dir, rename_to)
+        dbt.clients.system.rename(downloaded_path, desired_path, force=True)

dbt/exceptions.py

@@ -92,6 +92,10 @@ class ParsingException(Exception):
     pass
 
 
+class DependencyException(Exception):
+    pass
+
+
 class VersionsNotCompatibleException(ParsingException):
     def __init__(self, msg=None):
         self.msg = msg
@@ -116,6 +120,10 @@ def raise_database_error(msg, node=None):
     raise DatabaseException(msg, node)
 
 
+def raise_dependency_error(msg):
+    raise DependencyException(msg)
+
+
 def ref_invalid_args(model, args):
     raise_compiler_error(
         "ref() takes at most two arguments ({} given)".format(len(args)),
@@ -221,6 +229,30 @@ def missing_relation(relation_name, model=None):
         model)
 
 
+def package_not_found(package_name):
+    raise_dependency_error(
+        "Package {} was not found in the package index".format(package_name))
+
+
+def package_version_not_found(package_name, version_range, available_versions):
+    base_msg = ('Could not find a matching version for package {}\n'
+                '  Requested range: {}\n'
+                '  Available versions: {}')
+
+    raise_dependency_error(base_msg.format(package_name,
+                                           version_range,
+                                           available_versions))
+
+
+def incompatible_versions(package_name, versions):
+    version_list = ", ".join(versions)
+
+    raise VersionsNotCompatibleException(
+        'Could not find a satisfactory version of {}:\n'
+        'Specified versions: [{}]'
+        .format(package_name, version_list))
+
+
 def invalid_materialization_argument(name, argument):
     raise_compiler_error(
         "materialization '{}' received unknown argument '{}'."

dbt/semver.py

@@ -287,7 +287,7 @@ def is_exact(self):
         return False
 
 
-def reduce_versions(*args):
+def reduce_versions(*args, name=None):
     version_specifiers = []
 
     for version in args:
@@ -322,9 +322,8 @@ def reduce_versions(*args):
         for version_specifier in version_specifiers:
             to_return = to_return.reduce(version_specifier.to_range())
     except VersionsNotCompatibleException as e:
-        raise VersionsNotCompatibleException(
-            'Could not find a satisfactory version from options: {}'
-            .format(str(args)))
+        versions = [VersionSpecifier(arg).to_version_string() for arg in args]
+        dbt.exceptions.incompatible_versions(name, versions)
 
     return to_return
 
@@ -385,7 +384,8 @@ def resolve_dependency_tree(version_index, unmet_dependencies, restrictions):
     for dependency_name, version in unmet_dependencies.items():
         print('resolving path {}'.format(dependency_name))
         dependency_restrictions = reduce_versions(
-            *restrictions.copy().get(dependency_name))
+            *restrictions.copy().get(dependency_name),
+            name=dependency_name)
 
         possible_matches = find_possible_versions(
             dependency_restrictions,
@@ -405,7 +405,8 @@ def resolve_dependency_tree(version_index, unmet_dependencies, restrictions):
                 new_restrictions = restrictions.copy()
                 new_restrictions[dependency_name] = reduce_versions(
                     dependency_restrictions,
-                    possible_match
+                    possible_match,
+                    name=dependency_name
                 ).to_version_string_pair()
 
                 recursive_version_info = version_index.get(dependency_name, {}).get(possible_match)

dbt/task/deps.py

@@ -18,7 +18,7 @@
 class PackageListing(AttrDict):
 
     @classmethod
-    def _convert_version_strings(cls, version_strings):
+    def convert_version_strings(cls, version_strings):
         if not isinstance(version_strings, list):
             version_strings = [version_strings]
 
@@ -61,7 +61,7 @@ def create(cls, parsed_yaml):
                     (package, version_strings) = package.popitem()
                     to_return.incorporate(
                         package,
-                        cls._convert_version_strings(version_strings))
+                        cls.convert_version_strings(version_strings))
 
         return to_return
 
@@ -181,18 +181,22 @@ def __pull_deps_recursive(self, repos, processed_repos=None, i=0):
                     raise e
 
     def run(self):
-        listing = PackageListing.create(self.project['packages'])
+        listing = PackageListing.create(self.project.get('packages', []))
+        visited_listing = self.get_required_listing(listing)
+        self.fetch_required_packages(visited_listing)
+
+    def get_required_listing(self, listing):
         visited_listing = PackageListing.create([])
         index = dbt.clients.registry.index()
 
         while len(listing) > 0:
             (package, version_specifiers) = listing.popitem()
 
             if package not in index:
-                raise Exception('unknown package {}'.format(package))
+                dbt.exceptions.package_not_found(package)
 
-            version_range = dbt.semver.reduce_versions(
-                *version_specifiers)
+            version_range = dbt.semver.reduce_versions(*version_specifiers,
+                                                       name=package)
 
             available_versions = dbt.clients.registry.get_available_versions(
                 package)
@@ -206,38 +210,29 @@ def run(self):
                 available_versions)
 
             if target_version is None:
-                logger.error(
-                    'Could not find a matching version for package {}!'
-                    .format(package))
-                logger.error(
-                    '  Requested range: {}'.format(version_range))
-                logger.error(
-                    '  Available versions: {}'.format(
-                        ', '.join(available_versions)))
-                raise Exception('bad')
-
-            visited_listing.incorporate(
-                package,
-                [VersionSpecifier.from_version_string(target_version)])
+                dbt.exceptions.package_version_not_found(
+                    package, version_range, available_versions)
+
+            version_spec = VersionSpecifier.from_version_string(target_version)
+            visited_listing.incorporate(package, [version_spec])
 
             target_version_metadata = dbt.clients.registry.package_version(
                 package, target_version)
 
             dependencies = target_version_metadata.get('dependencies', {})
 
-            for package, versions in dependencies.items():
-                listing.incorporate(
-                    package,
-                    [VersionSpecifier.from_version_string(version)
-                     for version in versions])
+            for dep_package, dep_versions in dependencies.items():
+                versions = PackageListing.convert_version_strings(dep_versions)
+                listing.incorporate(dep_package, versions)
+
+        return visited_listing
 
+    def fetch_required_packages(self, visited_listing):
         for package, version_specifiers in visited_listing.items():
             version_string = version_specifiers[0].to_version_string(True)
             version_info = dbt.clients.registry.package_version(
                 package, version_string)
 
-            import requests
-
             tar_path = os.path.realpath('{}/downloads/{}.{}.tar.gz'.format(
                 self.project['modules-path'],
                 package,
@@ -249,15 +244,11 @@ def run(self):
             dbt.clients.system.make_directory(
                 os.path.dirname(tar_path))
 
-            response = requests.get(version_info.get('downloads').get('tarball'))
-
-            with open(tar_path, 'wb') as handle:
-                for block in response.iter_content(1024*64):
-                    handle.write(block)
-
-            import tarfile
+            download_url = version_info.get('downloads').get('tarball')
+            dbt.clients.system.download(download_url, tar_path)
 
-            with tarfile.open(tar_path, 'r') as tarball:
-                tarball.extractall(self.project['modules-path'])
+            deps_path = self.project['modules-path']
+            package_name = version_info['name']
+            dbt.clients.system.untar_package(tar_path, deps_path, package_name)
 
             logger.info(" -> Success.")

dbt/utils.py

@@ -190,8 +190,14 @@ def dependency_projects(project):
                 continue
 
             try:
+                dbt_project_yml_path = os.path.join(full_obj, 'dbt_project.yml')
+                if not os.path.exists(dbt_project_yml_path):
+                    msg = "dbt_project.yml file not found, excluding: {}"
+                    logger.debug(msg.format(full_obj))
+                    continue
+
                 yield dbt.project.read_project(
-                    os.path.join(full_obj, 'dbt_project.yml'),
+                    dbt_project_yml_path,
                     project.profiles_dir,
                     profile_to_load=project.profile_to_load,
                     args=project.args)