package management

converter.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+import json
+import yaml
+import sys
+import argparse
+from datetime import datetime, timezone
+import dbt.clients.registry as registry
+
+
+def yaml_type(fname):
+    with open(fname) as f:
+        return yaml.load(f)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--project", type=yaml_type, default="dbt_project.yml")
+    parser.add_argument("--namespace", required=True)
+    return parser.parse_args()
+
+
+def get_full_name(args):
+    return "{}/{}".format(args.namespace, args.project["name"])
+
+
+def init_project_in_packages(args, packages):
+    full_name = get_full_name(args)
+    if full_name not in packages:
+        packages[full_name] = {
+            "name": args.project["name"],
+            "namespace": args.namespace,
+            "latest": args.project["version"],
+            "assets": {},
+            "versions": {},
+        }
+    return packages[full_name]
+
+
+def add_version_to_package(args, project_json):
+    project_json["versions"][args.project["version"]] = {
+        "id": "{}/{}".format(get_full_name(args), args.project["version"]),
+        "name": args.project["name"],
+        "version": args.project["version"],
+        "description": "",
+        "published_at": datetime.now(timezone.utc).astimezone().isoformat(),
+        "packages": args.project.get("packages") or [],
+        "works_with": [],
+        "_source": {
+            "type": "github",
+            "url": "",
+            "readme": "",
+        },
+        "downloads": {
+            "tarball": "",
+            "format": "tgz",
+            "sha1": "",
+        },
+    }
+
+
+def main():
+    args = parse_args()
+    packages = registry.packages()
+    project_json = init_project_in_packages(args, packages)
+    if args.project["version"] in project_json["versions"]:
+        raise Exception("Version {} already in packages JSON"
+                        .format(args.project["version"]),
+                        file=sys.stderr)
+    add_version_to_package(args, project_json)
+    print(json.dumps(packages, indent=2))
+
+if __name__ == "__main__":
+    main()

dbt/adapters/bigquery.py

@@ -372,6 +372,12 @@ def get_columns_in_table(cls, profile, schema_name, table_name,
         raise dbt.exceptions.NotImplementedException(
             '`get_columns_in_table` is not implemented for this adapter!')
 
+    @classmethod
+    def get_columns_in_table(cls, profile, schema_name, table_name,
+                             model_name=None):
+        raise dbt.exceptions.NotImplementedException(
+            '`get_columns_in_table` is not implemented for this adapter!')
+
     @classmethod
     def check_schema_exists(cls, profile, schema, model_name=None):
         conn = cls.get_connection(profile, model_name)

dbt/clients/git.py

@@ -1,3 +1,4 @@
+import re
 import os.path
 
 from dbt.clients.system import run_cmd, rmdir
@@ -29,7 +30,7 @@ def checkout(cwd, repo, branch=None):
     if branch is None:
         branch = 'master'
 
-    logger.info('  Checking out branch {}.'.format(branch))
+    logger.debug('  Checking out branch {}.'.format(branch))
 
     run_cmd(cwd, ['git', 'remote', 'set-branches', 'origin', branch])
     run_cmd(cwd, ['git', 'fetch', '--tags', '--depth', '1', 'origin', branch])
@@ -59,3 +60,32 @@ def get_current_sha(cwd):
 
 def remove_remote(cwd):
     return run_cmd(cwd, ['git', 'remote', 'rm', 'origin'])
+
+
+def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
+                       branch=None):
+    _, err = clone(repo, cwd, dirname=dirname, remove_git_dir=remove_git_dir)
+    exists = re.match("fatal: destination path '(.+)' already exists",
+                      err.decode('utf-8'))
+    directory = None
+    start_sha = None
+    if exists:
+        directory = exists.group(1)
+        logger.debug('Updating existing dependency %s.', directory)
+    else:
+        matches = re.match("Cloning into '(.+)'", err.decode('utf-8'))
+        directory = matches.group(1)
+        logger.debug('Pulling new dependency %s.', directory)
+    full_path = os.path.join(cwd, directory)
+    start_sha = get_current_sha(full_path)
+    checkout(full_path, repo, branch)
+    end_sha = get_current_sha(full_path)
+    if exists:
+        if start_sha == end_sha:
+            logger.debug('  Already at %s, nothing to do.', start_sha[:7])
+        else:
+            logger.debug('  Updated checkout from %s to %s.',
+                         start_sha[:7], end_sha[:7])
+    else:
+        logger.debug('  Checked out at %s.', end_sha[:7])
+    return directory

dbt/clients/registry.py

@@ -0,0 +1,61 @@
+from functools import wraps
+import six
+import requests
+from dbt.exceptions import RegistryException
+from dbt.utils import memoized
+import os
+
+if os.getenv('DBT_PACKAGE_HUB_URL'):
+    DEFAULT_REGISTRY_BASE_URL = os.getenv('DBT_PACKAGE_HUB_URL')
+else:
+    DEFAULT_REGISTRY_BASE_URL = 'https://hub.getdbt.com/'
+
+
+def _get_url(url, registry_base_url=None):
+    if registry_base_url is None:
+        registry_base_url = DEFAULT_REGISTRY_BASE_URL
+
+    return '{}{}'.format(registry_base_url, url)
+
+
+def _wrap_exceptions(fn):
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        try:
+            return fn(*args, **kwargs)
+        except requests.exceptions.ConnectionError as e:
+            six.raise_from(
+                RegistryException('Unable to connect to registry hub'), e)
+    return wrapper
+
+
+@_wrap_exceptions
+def _get(path, registry_base_url=None):
+    url = _get_url(path, registry_base_url)
+    resp = requests.get(url)
+    resp.raise_for_status()
+    return resp.json()
+
+
+def index(registry_base_url=None):
+    return _get('api/v1/index.json', registry_base_url)
+
+
+index_cached = memoized(index)
+
+
+def packages(registry_base_url=None):
+    return _get('api/v1/packages.json', registry_base_url)
+
+
+def package(name, registry_base_url=None):
+    return _get('api/v1/{}.json'.format(name), registry_base_url)
+
+
+def package_version(name, version, registry_base_url=None):
+    return _get('api/v1/{}/{}.json'.format(name, version), registry_base_url)
+
+
+def get_available_versions(name):
+    response = package(name)
+    return list(response['versions'])

dbt/clients/system.py

@@ -5,8 +5,12 @@
 import shutil
 import subprocess
 import sys
+import tarfile
+import requests
+import stat
 
 import dbt.compat
+import dbt.exceptions
 
 from dbt.logger import GLOBAL_LOGGER as logger
 
@@ -92,19 +96,61 @@ def make_file(path, contents='', overwrite=False):
     return False
 
 
+def make_symlink(source, link_path):
+    """
+    Create a symlink at `link_path` referring to `source`.
+    """
+    if not supports_symlinks():
+        dbt.exceptions.system_error('create a symbolic link')
+
+    return os.symlink(source, link_path)
+
+
+def supports_symlinks():
+    return getattr(os, "symlink", None) is not None
+
+
 def write_file(path, contents=''):
     make_directory(os.path.dirname(path))
     dbt.compat.write_file(path, contents)
 
     return True
 
 
+def _windows_rmdir_readonly(func, path, exc):
+    exception_val = exc[1]
+    if exception_val.errno == errno.EACCES:
+        os.chmod(path, stat.S_IWUSR)
+        func(path)
+    else:
+        raise
+
+
 def rmdir(path):
     """
-    Make a file at `path` assuming that the directory it resides in already
-    exists. The file is saved with contents `contents`
+    Recursively deletes a directory. Includes an error handler to retry with
+    different permissions on Windows. Otherwise, removing directories (eg.
+    cloned via git) can cause rmtree to throw a PermissionError exception
     """
-    return shutil.rmtree(path)
+    logger.debug("DEBUG** Window rmdir sys.platform: {}".format(sys.platform))
+    if sys.platform == 'win32':
+        onerror = _windows_rmdir_readonly
+    else:
+        onerror = None
+
+    return shutil.rmtree(path, onerror=onerror)
+
+
+def remove_file(path):
+    return os.remove(path)
+
+
+def path_exists(path):
+    return os.path.lexists(path)
+
+
+def path_is_symlink(path):
+    return os.path.islink(path)
 
 
 def open_dir_cmd():
@@ -133,3 +179,27 @@ def run_cmd(cwd, cmd):
     logger.debug('STDERR: "{}"'.format(err))
 
     return out, err
+
+
+def download(url, path):
+    response = requests.get(url)
+    with open(path, 'wb') as handle:
+        for block in response.iter_content(1024*64):
+            handle.write(block)
+
+
+def rename(from_path, to_path, force=False):
+    if os.path.exists(to_path) and force:
+        rmdir(to_path)
+    os.rename(from_path, to_path)
+
+
+def untar_package(tar_path, dest_dir, rename_to=None):
+    tar_dir_name = None
+    with tarfile.open(tar_path, 'r') as tarball:
+        tarball.extractall(dest_dir)
+        tar_dir_name = os.path.commonprefix(tarball.getnames())
+    if rename_to:
+        downloaded_path = os.path.join(dest_dir, tar_dir_name)
+        desired_path = os.path.join(dest_dir, rename_to)
+        dbt.clients.system.rename(downloaded_path, desired_path, force=True)

dbt/context/common.py

@@ -1,5 +1,6 @@
 import json
 import os
+import pytz
 import voluptuous
 
 from dbt.adapters.factory import get_adapter

dbt/deprecations.py

@@ -11,21 +11,19 @@ def show(self, *args, **kwargs):
             logger.info("* Deprecation Warning: {}\n".format(desc))
             active_deprecations.add(self.name)
 
-# Leaving this as an example. Make sure to add new ones to deprecations_list
-#       - Connor
-#
-# class DBTRunTargetDeprecation(DBTDeprecation):
-#     name = 'run-target'
-#     description = """profiles.yml configuration option 'run-target' is
-#     deprecated. Please use 'target' instead. The 'run-target' option will be
-#     removed (in favor of 'target') in DBT version 0.7.0"""
+
+class DBTRepositoriesDeprecation(DBTDeprecation):
+    name = "repositories"
+    description = """dbt_project.yml configuration option 'repositories' is
+    deprecated. Please use 'packages' instead. The 'repositories' option will
+    be removed in a later version of DBT."""
 
 
 class SeedDropExistingDeprecation(DBTDeprecation):
     name = 'drop-existing'
-    description = """The --drop-existing argument has been deprecated. Please
-  use --full-refresh instead. The --drop-existing option will be removed in a
-  future version of dbt."""
+    description = """The --drop-existing argument to `dbt seed` has been
+  deprecated. Please use --full-refresh instead. The --drop-existing option
+  will be removed in a future version of dbt."""
 
 
 def warn(name, *args, **kwargs):
@@ -44,6 +42,7 @@ def warn(name, *args, **kwargs):
 active_deprecations = set()
 
 deprecations_list = [
+    DBTRepositoriesDeprecation(),
     SeedDropExistingDeprecation()
 ]