From a6ede813a9fec927fd84c27d008d1bea6e66e228 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 8 Nov 2017 16:55:11 -0500 Subject: [PATCH 1/3] write packages to correct dir --- dbt/clients/registry.py | 2 +- dbt/semver.py | 1 + dbt/task/deps.py | 21 ++++++++++++++++++--- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/dbt/clients/registry.py b/dbt/clients/registry.py index 8371e68b149..3d4dac91ff4 100644 --- a/dbt/clients/registry.py +++ b/dbt/clients/registry.py @@ -1,7 +1,7 @@ import requests -DEFAULT_REGISTRY_BASE_URL = 'http://127.0.0.1:4567/' +DEFAULT_REGISTRY_BASE_URL = 'http://127.0.0.1:35729/' def _get_url(url, registry_base_url=None): diff --git a/dbt/semver.py b/dbt/semver.py index 083bd67edaf..75284c295e1 100644 --- a/dbt/semver.py +++ b/dbt/semver.py @@ -322,6 +322,7 @@ def reduce_versions(*args): for version_specifier in version_specifiers: to_return = to_return.reduce(version_specifier.to_range()) except VersionsNotCompatibleException as e: + #TODO raise VersionsNotCompatibleException( 'Could not find a satisfactory version from options: {}' .format(str(args))) diff --git a/dbt/task/deps.py b/dbt/task/deps.py index c75edd5a10f..f1bd1ec5f0d 100644 --- a/dbt/task/deps.py +++ b/dbt/task/deps.py @@ -181,14 +181,17 @@ def __pull_deps_recursive(self, repos, processed_repos=None, i=0): raise e def run(self): - listing = PackageListing.create(self.project['packages']) + listing = PackageListing.create(self.project.get('packages', [])) visited_listing = PackageListing.create([]) index = dbt.clients.registry.index() + package_metadata = {} + while len(listing) > 0: (package, version_specifiers) = listing.popitem() if package not in index: + # TODO raise Exception('unknown package {}'.format(package)) version_range = dbt.semver.reduce_versions( @@ -214,6 +217,7 @@ def run(self): logger.error( ' Available versions: {}'.format( ', '.join(available_versions))) + # TODO raise Exception('bad') visited_listing.incorporate( @@ -223,6 +227,9 @@ def run(self): target_version_metadata = dbt.clients.registry.package_version( package, target_version) + # Can i do this? + package_metadata[package] = target_version_metadata + dependencies = target_version_metadata.get('dependencies', {}) for package, versions in dependencies.items(): @@ -236,8 +243,8 @@ def run(self): version_info = dbt.clients.registry.package_version( package, version_string) + # TODO import requests - tar_path = os.path.realpath('{}/downloads/{}.{}.tar.gz'.format( self.project['modules-path'], package, @@ -256,8 +263,16 @@ def run(self): handle.write(block) import tarfile + # TODO + metadata = package_metadata[package] + package_name = metadata['name'] with tarfile.open(tar_path, 'r') as tarball: - tarball.extractall(self.project['modules-path']) + for item in tarball.getmembers(): + orig_path = item.name.split(os.sep) + bare_path = orig_path[1:] + new_path = os.path.join(package_name, *bare_path) + item.name = new_path + tarball.extract(new_path, self.project['modules-path']) logger.info(" -> Success.") From e824f82e8d774a4a063d209e8895403c827b3f37 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 8 Nov 2017 19:29:16 -0500 Subject: [PATCH 2/3] code cleanup, error handling --- dbt/clients/system.py | 32 ++++++++++++++++--- dbt/exceptions.py | 32 +++++++++++++++++++ dbt/semver.py | 14 ++++---- dbt/task/deps.py | 74 +++++++++++++++---------------------------- dbt/utils.py | 8 ++++- 5 files changed, 99 insertions(+), 61 deletions(-) diff --git a/dbt/clients/system.py b/dbt/clients/system.py index f94108f7e66..cb53856bd92 100644 --- a/dbt/clients/system.py +++ b/dbt/clients/system.py @@ -5,6 +5,8 @@ import shutil import subprocess import sys +import tarfile +import requests import dbt.compat @@ -100,10 +102,6 @@ def write_file(path, contents=''): def rmdir(path): - """ - Make a file at `path` assuming that the directory it resides in already - exists. The file is saved with contents `contents` - """ return shutil.rmtree(path) @@ -133,3 +131,29 @@ def run_cmd(cwd, cmd): logger.debug('STDERR: "{}"'.format(err)) return out, err + + +def download(url, path): + response = requests.get(url) + + with open(path, 'wb') as handle: + for block in response.iter_content(1024*64): + handle.write(block) + +def rename(from_path, to_path, force=False): + if os.path.exists(to_path) and force: + rmdir(to_path) + + os.rename(from_path, to_path) + + +def untar_package(tar_path, dest_dir, rename_to=None): + tar_dir_name = None + with tarfile.open(tar_path, 'r') as tarball: + tarball.extractall(dest_dir) + tar_dir_name = os.path.commonprefix(tarball.getnames()) + + if rename_to: + downloaded_path = os.path.join(dest_dir, tar_dir_name) + desired_path = os.path.join(dest_dir, rename_to) + dbt.clients.system.rename(downloaded_path, desired_path, force=True) diff --git a/dbt/exceptions.py b/dbt/exceptions.py index a17a6a74ae9..a203422d6bf 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -92,6 +92,10 @@ class ParsingException(Exception): pass +class DependencyException(Exception): + pass + + class VersionsNotCompatibleException(ParsingException): def __init__(self, msg=None): self.msg = msg @@ -116,6 +120,10 @@ def raise_database_error(msg, node=None): raise DatabaseException(msg, node) +def raise_dependency_error(msg): + raise DependencyException(msg) + + def ref_invalid_args(model, args): raise_compiler_error( "ref() takes at most two arguments ({} given)".format(len(args)), @@ -221,6 +229,30 @@ def missing_relation(relation_name, model=None): model) +def package_not_found(package_name): + raise_dependency_error( + "Package {} was not found in the package index".format(package_name)) + + +def package_version_not_found(package_name, version_range, available_versions): + base_msg = ('Could not find a matching version for package {}!\n' + ' Requested range: {}\n' + ' Available versions: {}') + + raise_dependency_error(base_msg.format(package_name, + version_range, + available_versions)) + + +def incompatible_versions(package_name, versions): + version_list = ", ".join(versions) + + raise VersionsNotCompatibleException( + 'Could not find a satisfactory version of {}:\n' + 'Specified versions: [{}]' + .format(package_name, version_list)) + + def invalid_materialization_argument(name, argument): raise_compiler_error( "materialization '{}' received unknown argument '{}'." diff --git a/dbt/semver.py b/dbt/semver.py index 75284c295e1..bcc6fb3813b 100644 --- a/dbt/semver.py +++ b/dbt/semver.py @@ -287,7 +287,7 @@ def is_exact(self): return False -def reduce_versions(*args): +def reduce_versions(*args, name=None): version_specifiers = [] for version in args: @@ -322,10 +322,8 @@ def reduce_versions(*args): for version_specifier in version_specifiers: to_return = to_return.reduce(version_specifier.to_range()) except VersionsNotCompatibleException as e: - #TODO - raise VersionsNotCompatibleException( - 'Could not find a satisfactory version from options: {}' - .format(str(args))) + versions = [VersionSpecifier(arg).to_version_string() for arg in args] + dbt.exceptions.incompatible_versions(name, versions) return to_return @@ -386,7 +384,8 @@ def resolve_dependency_tree(version_index, unmet_dependencies, restrictions): for dependency_name, version in unmet_dependencies.items(): print('resolving path {}'.format(dependency_name)) dependency_restrictions = reduce_versions( - *restrictions.copy().get(dependency_name)) + *restrictions.copy().get(dependency_name), + name=dependency_name) possible_matches = find_possible_versions( dependency_restrictions, @@ -406,7 +405,8 @@ def resolve_dependency_tree(version_index, unmet_dependencies, restrictions): new_restrictions = restrictions.copy() new_restrictions[dependency_name] = reduce_versions( dependency_restrictions, - possible_match + possible_match, + name=dependency_name ).to_version_string_pair() recursive_version_info = version_index.get(dependency_name, {}).get(possible_match) diff --git a/dbt/task/deps.py b/dbt/task/deps.py index f1bd1ec5f0d..54e2ba7ed8a 100644 --- a/dbt/task/deps.py +++ b/dbt/task/deps.py @@ -18,7 +18,7 @@ class PackageListing(AttrDict): @classmethod - def _convert_version_strings(cls, version_strings): + def convert_version_strings(cls, version_strings): if not isinstance(version_strings, list): version_strings = [version_strings] @@ -61,7 +61,7 @@ def create(cls, parsed_yaml): (package, version_strings) = package.popitem() to_return.incorporate( package, - cls._convert_version_strings(version_strings)) + cls.convert_version_strings(version_strings)) return to_return @@ -182,20 +182,21 @@ def __pull_deps_recursive(self, repos, processed_repos=None, i=0): def run(self): listing = PackageListing.create(self.project.get('packages', [])) + visited_listing = self.get_required_listing(listing) + self.fetch_required_packages(visited_listing) + + def get_required_listing(self, listing): visited_listing = PackageListing.create([]) index = dbt.clients.registry.index() - package_metadata = {} - while len(listing) > 0: (package, version_specifiers) = listing.popitem() if package not in index: - # TODO - raise Exception('unknown package {}'.format(package)) + dbt.exceptions.package_not_found(package) - version_range = dbt.semver.reduce_versions( - *version_specifiers) + version_range = dbt.semver.reduce_versions(*version_specifiers, + name=package) available_versions = dbt.clients.registry.get_available_versions( package) @@ -209,42 +210,29 @@ def run(self): available_versions) if target_version is None: - logger.error( - 'Could not find a matching version for package {}!' - .format(package)) - logger.error( - ' Requested range: {}'.format(version_range)) - logger.error( - ' Available versions: {}'.format( - ', '.join(available_versions))) - # TODO - raise Exception('bad') - - visited_listing.incorporate( - package, - [VersionSpecifier.from_version_string(target_version)]) + dbt.exceptions.package_version_not_found( + package, version_range, available_versions) + + version_spec = VersionSpecifier.from_version_string(target_version) + visited_listing.incorporate(package, [version_spec]) target_version_metadata = dbt.clients.registry.package_version( package, target_version) - # Can i do this? - package_metadata[package] = target_version_metadata - dependencies = target_version_metadata.get('dependencies', {}) - for package, versions in dependencies.items(): - listing.incorporate( - package, - [VersionSpecifier.from_version_string(version) - for version in versions]) + for dep_package, dep_versions in dependencies.items(): + versions = PackageListing.convert_version_strings(dep_versions) + listing.incorporate(dep_package, versions) + + return visited_listing + def fetch_required_packages(self, visited_listing): for package, version_specifiers in visited_listing.items(): version_string = version_specifiers[0].to_version_string(True) version_info = dbt.clients.registry.package_version( package, version_string) - # TODO - import requests tar_path = os.path.realpath('{}/downloads/{}.{}.tar.gz'.format( self.project['modules-path'], package, @@ -256,23 +244,11 @@ def run(self): dbt.clients.system.make_directory( os.path.dirname(tar_path)) - response = requests.get(version_info.get('downloads').get('tarball')) - - with open(tar_path, 'wb') as handle: - for block in response.iter_content(1024*64): - handle.write(block) - - import tarfile - # TODO - metadata = package_metadata[package] - package_name = metadata['name'] + download_url = version_info.get('downloads').get('tarball') + dbt.clients.system.download(download_url, tar_path) - with tarfile.open(tar_path, 'r') as tarball: - for item in tarball.getmembers(): - orig_path = item.name.split(os.sep) - bare_path = orig_path[1:] - new_path = os.path.join(package_name, *bare_path) - item.name = new_path - tarball.extract(new_path, self.project['modules-path']) + deps_path = self.project['modules-path'] + package_name = version_info['name'] + dbt.clients.system.untar_package(tar_path, deps_path, package_name) logger.info(" -> Success.") diff --git a/dbt/utils.py b/dbt/utils.py index cd2b32e73c0..308759aeb7f 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -190,8 +190,14 @@ def dependency_projects(project): continue try: + dbt_project_yml_path = os.path.join(full_obj, 'dbt_project.yml') + if not os.path.exists(dbt_project_yml_path): + msg = "dbt_project.yml file not found, excluding: {}" + logger.debug(msg.format(full_obj)) + continue + yield dbt.project.read_project( - os.path.join(full_obj, 'dbt_project.yml'), + dbt_project_yml_path, project.profiles_dir, profile_to_load=project.profile_to_load, args=project.args) From 9ee480b953a6fd1335b45c2c13a001368db7c0c4 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 8 Nov 2017 19:32:41 -0500 Subject: [PATCH 3/3] curb your enthusiasm --- dbt/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/exceptions.py b/dbt/exceptions.py index a203422d6bf..911b4790881 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -235,7 +235,7 @@ def package_not_found(package_name): def package_version_not_found(package_name, version_range, available_versions): - base_msg = ('Could not find a matching version for package {}!\n' + base_msg = ('Could not find a matching version for package {}\n' ' Requested range: {}\n' ' Available versions: {}')