Merge branch 'dev/barbara-gittings' of https://github.com/fishtown-an…

…alytics/dbt into dbt-debug-empty-project

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   unit:
     docker: &test_only
-      - image: fishtownjacob/test-container:3
+      - image: fishtownjacob/test-container:4
         environment:
           DBT_INVOCATION_ENV: circle
     steps:
@@ -15,18 +15,20 @@ jobs:
       - run:
           name: Build wheels
           command: |
+            python3.8 -m venv "${PYTHON_ENV}"
+            export PYTHON_BIN="${PYTHON_ENV}/bin/python"
             $PYTHON_BIN -m pip install -U pip setuptools
             $PYTHON_BIN -m pip install -r requirements.txt
             $PYTHON_BIN -m pip install -r dev_requirements.txt
             /bin/bash ./scripts/build-wheels.sh
           environment:
-            PYTHON_BIN: /home/tox/venv3.8/bin/python
+            PYTHON_ENV: /home/tox/build_venv/
       - store_artifacts:
           path: ./dist
           destination: dist
   integration-postgres-py36:
     docker: &test_and_postgres
-      - image: fishtownjacob/test-container:3
+      - image: fishtownjacob/test-container:4
         environment:
           DBT_INVOCATION_ENV: circle
       - image: postgres

Dockerfile

@@ -12,7 +12,8 @@ RUN apt-get update && \
         python python-dev python-pip \
         python3.6 python3.6-dev python3-pip python3.6-venv \
         python3.7 python3.7-dev python3.7-venv \
-        python3.8 python3.8-dev python3.8-venv && \
+        python3.8 python3.8-dev python3.8-venv \
+        python3.9 python3.9-dev python3.9-venv && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 RUN useradd -mU dbt_test_user
@@ -22,27 +23,9 @@ RUN mkdir /home/tox && chown dbt_test_user /home/tox
 WORKDIR /usr/app
 VOLUME /usr/app
 
-RUN pip3 install tox wheel
-
-RUN python2.7 -m pip install virtualenv wheel && \
-    python2.7 -m virtualenv /home/tox/venv2.7 && \
-    /home/tox/venv2.7/bin/python -m pip install -U pip tox && \
-    chown -R dbt_test_user /home/tox/venv2.7
-
-RUN python3.6 -m pip install -U pip wheel && \
-    python3.6 -m venv /home/tox/venv3.6 && \
-    /home/tox/venv3.6/bin/python -m pip install -U pip tox && \
-    chown -R dbt_test_user /home/tox/venv3.6
-
-RUN python3.7 -m pip install -U pip wheel && \
-    python3.7 -m venv /home/tox/venv3.7 && \
-    /home/tox/venv3.7/bin/python -m pip install -U pip tox && \
-    chown -R dbt_test_user /home/tox/venv3.7
-
-RUN python3.8 -m pip install -U pip wheel && \
-    python3.8 -m venv /home/tox/venv3.8 && \
-    /home/tox/venv3.8/bin/python -m pip install -U pip tox && \
-    chown -R dbt_test_user /home/tox/venv3.8
+RUN pip3 install -U "tox==3.14.4" wheel "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools
+# tox fails if the 'python' interpreter (python2) doesn't have `tox` installed
+RUN pip install -U "tox==3.14.4" "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools
 
 USER dbt_test_user
 

core/dbt/contracts/graph/parsed.py

@@ -20,7 +20,7 @@
 from dbt.clients.system import write_file
 import dbt.flags
 from dbt.contracts.graph.unparsed import (
-    UnparsedNode, UnparsedMacro, UnparsedDocumentationFile, Quoting,
+    UnparsedNode, UnparsedMacro, UnparsedDocumentationFile, Quoting, Docs,
     UnparsedBaseNode, FreshnessThreshold, ExternalTable,
     AdditionalPropertiesAllowed, HasYamlMetadata, MacroArgument
 )
@@ -188,6 +188,7 @@ def patch(self, patch: 'ParsedNodePatch'):
         self.description = patch.description
         self.columns = patch.columns
         self.meta = patch.meta
+        self.docs = patch.docs
         if dbt.flags.STRICT_MODE:
             assert isinstance(self, JsonSchemaMixin)
             self.to_dict(validate=True)
@@ -224,6 +225,7 @@ class ParsedNodeDefaults(ParsedNodeMandatory):
     description: str = field(default='')
     columns: Dict[str, ColumnInfo] = field(default_factory=dict)
     meta: Dict[str, Any] = field(default_factory=dict)
+    docs: Docs = field(default_factory=Docs)
     patch_path: Optional[str] = None
     build_path: Optional[str] = None
 
@@ -467,6 +469,7 @@ class ParsedPatch(HasYamlMetadata, Replaceable):
     name: str
     description: str
     meta: Dict[str, Any]
+    docs: Docs
 
 
 # The parsed node update is only the 'patch', not the test. The test became a

core/dbt/contracts/graph/unparsed.py

@@ -58,12 +58,18 @@ class UnparsedRunHook(UnparsedNode):
     index: Optional[int] = None
 
 
+@dataclass
+class Docs(JsonSchemaMixin, Replaceable):
+    show: bool = True
+
+
 @dataclass
 class HasDocs(JsonSchemaMixin, Replaceable):
     name: str
     description: str = ''
     meta: Dict[str, Any] = field(default_factory=dict)
     data_type: Optional[str] = None
+    docs: Docs = field(default_factory=Docs)
 
 
 TestDef = Union[Dict[str, Any], str]

core/dbt/exceptions.py

@@ -673,6 +673,8 @@ def raise_duplicate_resource_name(node_1, node_2):
         get_func = 'ref("{}")'.format(duped_name)
     elif node_1.resource_type == NodeType.Source:
         get_func = 'source("{}", "{}")'.format(node_1.source_name, duped_name)
+    elif node_1.resource_type == NodeType.Documentation:
+        get_func = 'doc("{}")'.format(duped_name)
     elif node_1.resource_type == NodeType.Test and 'schema' in node_1.tags:
         return
 

core/dbt/parser/results.py

@@ -121,7 +121,7 @@ def add_macro(self, source_file: SourceFile, macro: ParsedMacro):
         self.get_file(source_file).macros.append(macro.unique_id)
 
     def add_doc(self, source_file: SourceFile, doc: ParsedDocumentation):
-        # Docs also can be overwritten (should they be?)
+        _check_duplicates(doc, self.docs)
         self.docs[doc.unique_id] = doc
         self.get_file(source_file).docs.append(doc.unique_id)
 

core/dbt/parser/schemas.py

@@ -623,6 +623,7 @@ def parse_patch(
             description=description,
             columns=refs.column_info,
             meta=block.target.meta,
+            docs=block.target.docs,
         )
         self.results.add_patch(self.yaml.file, result)
 
@@ -673,5 +674,6 @@ def parse_patch(
             arguments=block.target.arguments,
             description=description,
             meta=block.target.meta,
+            docs=block.target.docs,
         )
         self.results.add_macro_patch(self.yaml.file, result)

core/dbt/task/rpc/cli.py

@@ -1,5 +1,6 @@
 import abc
 import shlex
+import yaml
 from typing import Type, Optional
 
 
@@ -12,6 +13,7 @@
     Result,
 )
 from dbt.exceptions import InternalException
+from dbt.utils import parse_cli_vars
 
 from .base import RPCTask
 
@@ -36,6 +38,15 @@ def __init__(self, args, config, manifest):
 
     def set_config(self, config):
         super().set_config(config)
+
+        # read any cli vars we got and use it to update cli_vars
+        self.config.cli_vars.update(
+            parse_cli_vars(getattr(self.args, 'vars', '{}'))
+        )
+        # rewrite args.vars to reflect our merged vars
+        self.args.vars = yaml.safe_dump(self.config.cli_vars)
+        self.config.args = self.args
+
         if self.task_type is None:
             raise InternalException('task type not set for set_config')
         if issubclass(self.task_type, RemoteManifestMethod):

dev_requirements.txt

@@ -3,7 +3,9 @@ pytest==4.4.0
 flake8>=3.5.0
 pytz==2017.2
 bumpversion==0.5.3
-tox==2.5.0
+tox==3.14.4
+virtualenv==20.0.3
+six>=1.14.0
 ipdb
 pytest-xdist>=1.28.0,<2
 flaky>=3.5.3,<4

test/integration/029_docs_generate_tests/models/schema.yml

@@ -3,6 +3,8 @@ version: 2
 models:
   - name: model
     description: "The test model"
+    docs:
+      show: false
     columns:
       - name: id
         description: The user ID number