diff --git a/.circleci/config.yml b/.circleci/config.yml index fbdbe95b4f8..69d3dca51a2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: unit: docker: &test_only - - image: fishtownjacob/test-container:3 + - image: fishtownjacob/test-container:4 environment: DBT_INVOCATION_ENV: circle steps: @@ -15,18 +15,20 @@ jobs: - run: name: Build wheels command: | + python3.8 -m venv "${PYTHON_ENV}" + export PYTHON_BIN="${PYTHON_ENV}/bin/python" $PYTHON_BIN -m pip install -U pip setuptools $PYTHON_BIN -m pip install -r requirements.txt $PYTHON_BIN -m pip install -r dev_requirements.txt /bin/bash ./scripts/build-wheels.sh environment: - PYTHON_BIN: /home/tox/venv3.8/bin/python + PYTHON_ENV: /home/tox/build_venv/ - store_artifacts: path: ./dist destination: dist integration-postgres-py36: docker: &test_and_postgres - - image: fishtownjacob/test-container:3 + - image: fishtownjacob/test-container:4 environment: DBT_INVOCATION_ENV: circle - image: postgres diff --git a/Dockerfile b/Dockerfile index 17cc3515913..6a1a49b6568 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,8 @@ RUN apt-get update && \ python python-dev python-pip \ python3.6 python3.6-dev python3-pip python3.6-venv \ python3.7 python3.7-dev python3.7-venv \ - python3.8 python3.8-dev python3.8-venv && \ + python3.8 python3.8-dev python3.8-venv \ + python3.9 python3.9-dev python3.9-venv && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN useradd -mU dbt_test_user @@ -22,27 +23,9 @@ RUN mkdir /home/tox && chown dbt_test_user /home/tox WORKDIR /usr/app VOLUME /usr/app -RUN pip3 install tox wheel - -RUN python2.7 -m pip install virtualenv wheel && \ - python2.7 -m virtualenv /home/tox/venv2.7 && \ - /home/tox/venv2.7/bin/python -m pip install -U pip tox && \ - chown -R dbt_test_user /home/tox/venv2.7 - -RUN python3.6 -m pip install -U pip wheel && \ - python3.6 -m venv /home/tox/venv3.6 && \ - /home/tox/venv3.6/bin/python -m pip install -U pip tox && \ - chown -R dbt_test_user /home/tox/venv3.6 - -RUN python3.7 -m pip install -U pip wheel && \ - python3.7 -m venv /home/tox/venv3.7 && \ - /home/tox/venv3.7/bin/python -m pip install -U pip tox && \ - chown -R dbt_test_user /home/tox/venv3.7 - -RUN python3.8 -m pip install -U pip wheel && \ - python3.8 -m venv /home/tox/venv3.8 && \ - /home/tox/venv3.8/bin/python -m pip install -U pip tox && \ - chown -R dbt_test_user /home/tox/venv3.8 +RUN pip3 install -U "tox==3.14.4" wheel "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools +# tox fails if the 'python' interpreter (python2) doesn't have `tox` installed +RUN pip install -U "tox==3.14.4" "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools USER dbt_test_user diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index 6a80318330f..55102746d3e 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -20,7 +20,7 @@ from dbt.clients.system import write_file import dbt.flags from dbt.contracts.graph.unparsed import ( - UnparsedNode, UnparsedMacro, UnparsedDocumentationFile, Quoting, + UnparsedNode, UnparsedMacro, UnparsedDocumentationFile, Quoting, Docs, UnparsedBaseNode, FreshnessThreshold, ExternalTable, AdditionalPropertiesAllowed, HasYamlMetadata, MacroArgument ) @@ -188,6 +188,7 @@ def patch(self, patch: 'ParsedNodePatch'): self.description = patch.description self.columns = patch.columns self.meta = patch.meta + self.docs = patch.docs if dbt.flags.STRICT_MODE: assert isinstance(self, JsonSchemaMixin) self.to_dict(validate=True) @@ -224,6 +225,7 @@ class ParsedNodeDefaults(ParsedNodeMandatory): description: str = field(default='') columns: Dict[str, ColumnInfo] = field(default_factory=dict) meta: Dict[str, Any] = field(default_factory=dict) + docs: Docs = field(default_factory=Docs) patch_path: Optional[str] = None build_path: Optional[str] = None @@ -467,6 +469,7 @@ class ParsedPatch(HasYamlMetadata, Replaceable): name: str description: str meta: Dict[str, Any] + docs: Docs # The parsed node update is only the 'patch', not the test. The test became a diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 4bb9e8065f9..4d901a27850 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -58,12 +58,18 @@ class UnparsedRunHook(UnparsedNode): index: Optional[int] = None +@dataclass +class Docs(JsonSchemaMixin, Replaceable): + show: bool = True + + @dataclass class HasDocs(JsonSchemaMixin, Replaceable): name: str description: str = '' meta: Dict[str, Any] = field(default_factory=dict) data_type: Optional[str] = None + docs: Docs = field(default_factory=Docs) TestDef = Union[Dict[str, Any], str] diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 86727bf418e..cbbe3c74752 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -673,6 +673,8 @@ def raise_duplicate_resource_name(node_1, node_2): get_func = 'ref("{}")'.format(duped_name) elif node_1.resource_type == NodeType.Source: get_func = 'source("{}", "{}")'.format(node_1.source_name, duped_name) + elif node_1.resource_type == NodeType.Documentation: + get_func = 'doc("{}")'.format(duped_name) elif node_1.resource_type == NodeType.Test and 'schema' in node_1.tags: return diff --git a/core/dbt/parser/results.py b/core/dbt/parser/results.py index f5527511f85..2f523f67910 100644 --- a/core/dbt/parser/results.py +++ b/core/dbt/parser/results.py @@ -121,7 +121,7 @@ def add_macro(self, source_file: SourceFile, macro: ParsedMacro): self.get_file(source_file).macros.append(macro.unique_id) def add_doc(self, source_file: SourceFile, doc: ParsedDocumentation): - # Docs also can be overwritten (should they be?) + _check_duplicates(doc, self.docs) self.docs[doc.unique_id] = doc self.get_file(source_file).docs.append(doc.unique_id) diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 9e91756cb68..5fc7aae1fef 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -623,6 +623,7 @@ def parse_patch( description=description, columns=refs.column_info, meta=block.target.meta, + docs=block.target.docs, ) self.results.add_patch(self.yaml.file, result) @@ -673,5 +674,6 @@ def parse_patch( arguments=block.target.arguments, description=description, meta=block.target.meta, + docs=block.target.docs, ) self.results.add_macro_patch(self.yaml.file, result) diff --git a/core/dbt/task/rpc/cli.py b/core/dbt/task/rpc/cli.py index e3bb477c454..d5bb61b6d96 100644 --- a/core/dbt/task/rpc/cli.py +++ b/core/dbt/task/rpc/cli.py @@ -1,5 +1,6 @@ import abc import shlex +import yaml from typing import Type, Optional @@ -12,6 +13,7 @@ Result, ) from dbt.exceptions import InternalException +from dbt.utils import parse_cli_vars from .base import RPCTask @@ -36,6 +38,15 @@ def __init__(self, args, config, manifest): def set_config(self, config): super().set_config(config) + + # read any cli vars we got and use it to update cli_vars + self.config.cli_vars.update( + parse_cli_vars(getattr(self.args, 'vars', '{}')) + ) + # rewrite args.vars to reflect our merged vars + self.args.vars = yaml.safe_dump(self.config.cli_vars) + self.config.args = self.args + if self.task_type is None: raise InternalException('task type not set for set_config') if issubclass(self.task_type, RemoteManifestMethod): diff --git a/dev_requirements.txt b/dev_requirements.txt index 626ed353b72..835bb9fb1ab 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -3,7 +3,9 @@ pytest==4.4.0 flake8>=3.5.0 pytz==2017.2 bumpversion==0.5.3 -tox==2.5.0 +tox==3.14.4 +virtualenv==20.0.3 +six>=1.14.0 ipdb pytest-xdist>=1.28.0,<2 flaky>=3.5.3,<4 diff --git a/test/integration/029_docs_generate_tests/models/schema.yml b/test/integration/029_docs_generate_tests/models/schema.yml index 6f3fc661976..962a22f8239 100644 --- a/test/integration/029_docs_generate_tests/models/schema.yml +++ b/test/integration/029_docs_generate_tests/models/schema.yml @@ -3,6 +3,8 @@ version: 2 models: - name: model description: "The test model" + docs: + show: false columns: - name: id description: The user ID number diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 17c3dbaf31d..898e48a5c5d 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -950,6 +950,7 @@ def expected_seeded_manifest(self, model_database=None): }, }, 'patch_path': model_schema_yml_path, + 'docs': {'show': False}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -1031,6 +1032,7 @@ def expected_seeded_manifest(self, model_database=None): 'tags': [], }, }, + 'docs': {'show': True}, 'compiled': True, 'compiled_sql': '', 'extra_ctes_injected': True, @@ -1076,6 +1078,7 @@ def expected_seeded_manifest(self, model_database=None): 'tags': ['schema'], 'meta': {}, 'unique_id': 'test.test.not_null_model_id', + 'docs': {'show': True}, 'compiled': True, 'compiled_sql': AnyStringWith('count(*)'), 'extra_ctes_injected': True, @@ -1126,6 +1129,7 @@ def expected_seeded_manifest(self, model_database=None): 'tags': ['schema'], 'meta': {}, 'unique_id': 'test.test.test_nothing_model_', + 'docs': {'show': True}, 'compiled': True, 'compiled_sql': AnyStringWith('select 0'), 'extra_ctes_injected': True, @@ -1176,6 +1180,7 @@ def expected_seeded_manifest(self, model_database=None): 'tags': ['schema'], 'meta': {}, 'unique_id': 'test.test.unique_model_id', + 'docs': {'show': True}, 'compiled': True, 'compiled_sql': AnyStringWith('count(*)'), 'extra_ctes_injected': True, @@ -1337,6 +1342,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'nodes': ['source.test.my_source.my_table'] }, 'description': '', + 'docs': {'show': True}, 'fqn': ['test', 'ephemeral_copy'], 'name': 'ephemeral_copy', 'original_file_path': self.dir('ref_models/ephemeral_copy.sql'), @@ -1398,6 +1404,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'nodes': ['model.test.ephemeral_copy'] }, 'description': 'A summmary table of the ephemeral copy of the seed data', + 'docs': {'show': True}, 'fqn': ['test', 'ephemeral_summary'], 'name': 'ephemeral_summary', 'original_file_path': self.dir('ref_models/ephemeral_summary.sql'), @@ -1461,6 +1468,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'nodes': ['model.test.ephemeral_summary'] }, 'description': 'A view of the summary of the ephemeral copy of the seed data', + 'docs': {'show': True}, 'fqn': ['test', 'view_summary'], 'name': 'view_summary', 'original_file_path': self.dir('ref_models/view_summary.sql'), @@ -1542,6 +1550,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'description': 'The test seed', + 'docs': {'show': True}, 'fqn': ['test', 'seed'], 'name': 'seed', 'original_file_path': self.dir('seed/seed.csv'), @@ -1928,6 +1937,7 @@ def expected_bigquery_complex_manifest(self): }, 'description': 'A clustered and partitioned copy of the test model', 'patch_path': self.dir('bq_models/schema.yml'), + 'docs': {'show': True}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -2006,6 +2016,7 @@ def expected_bigquery_complex_manifest(self): }, 'description': 'A clustered and partitioned copy of the test model, clustered on multiple columns', 'patch_path': self.dir('bq_models/schema.yml'), + 'docs': {'show': True}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -2085,6 +2096,7 @@ def expected_bigquery_complex_manifest(self): }, 'description': 'The test model', 'patch_path': self.dir('bq_models/schema.yml'), + 'docs': {'show': True}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -2128,6 +2140,7 @@ def expected_bigquery_complex_manifest(self): 'unique_id': 'model.test.nested_table', 'columns': {}, 'description': '', + 'docs': {'show': True}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -2209,6 +2222,7 @@ def expected_bigquery_complex_manifest(self): }, }, 'description': 'The test seed', + 'docs': {'show': True}, 'compiled': True, 'compiled_sql': '', 'extra_ctes_injected': True, @@ -2432,6 +2446,7 @@ def expected_redshift_incremental_view_manifest(self): }, }, 'patch_path': self.dir('rs_models/schema.yml'), + 'docs': {'show': True}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -2513,6 +2528,7 @@ def expected_redshift_incremental_view_manifest(self): }, }, 'description': 'The test seed', + 'docs': {'show': True}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -2738,6 +2754,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'nodes': ['seed.test.seed'] }, 'description': 'The test model', + 'docs': {'show': False}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'model'], @@ -2825,6 +2842,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'description': 'The test seed', + 'docs': {'show': True}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'seed'], @@ -2881,6 +2899,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'nodes': ['model.test.model'], }, 'description': '', + 'docs': {'show': True}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'not_null_model_id'], @@ -2941,6 +2960,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'nodes': ['model.test.model'], }, 'description': '', + 'docs': {'show': True}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'test_nothing_model_'], @@ -3001,6 +3021,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'nodes': ['model.test.model'], }, 'description': '', + 'docs': {'show': True}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'unique_model_id'], @@ -3105,6 +3126,7 @@ def expected_postgres_references_run_results(self): 'description': ( 'A summmary table of the ephemeral copy of the seed data' ), + 'docs': {'show': True}, 'extra_ctes': [ {'id': 'model.test.ephemeral_copy', 'sql': cte_sql}, ], @@ -3186,6 +3208,7 @@ def expected_postgres_references_run_results(self): 'A view of the summary of the ephemeral copy of the ' 'seed data' ), + 'docs': {'show': True}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'view_summary'], @@ -3277,6 +3300,7 @@ def expected_postgres_references_run_results(self): 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'description': 'The test seed', + 'docs': {'show': True}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'seed'], diff --git a/test/integration/035_docs_blocks/duplicate_docs/docs.md b/test/integration/035_docs_blocks/duplicate_docs/docs.md new file mode 100644 index 00000000000..8499e541d56 --- /dev/null +++ b/test/integration/035_docs_blocks/duplicate_docs/docs.md @@ -0,0 +1,7 @@ +{% docs my_model_doc %} + a doc string +{% enddocs %} + +{% docs my_model_doc %} + duplicate doc string +{% enddocs %} diff --git a/test/integration/035_docs_blocks/duplicate_docs/model.sql b/test/integration/035_docs_blocks/duplicate_docs/model.sql new file mode 100644 index 00000000000..3397a804a20 --- /dev/null +++ b/test/integration/035_docs_blocks/duplicate_docs/model.sql @@ -0,0 +1 @@ +select 1 as id, 'joe' as first_name \ No newline at end of file diff --git a/test/integration/035_docs_blocks/duplicate_docs/schema.yml b/test/integration/035_docs_blocks/duplicate_docs/schema.yml new file mode 100644 index 00000000000..a04a43e4407 --- /dev/null +++ b/test/integration/035_docs_blocks/duplicate_docs/schema.yml @@ -0,0 +1,5 @@ +version: 2 + +models: + - name: model + description: "{{ doc('my_model_doc') }}" diff --git a/test/integration/035_docs_blocks/test_docs_blocks.py b/test/integration/035_docs_blocks/test_docs_blocks.py index 29b8eb4341e..e02e258399f 100644 --- a/test/integration/035_docs_blocks/test_docs_blocks.py +++ b/test/integration/035_docs_blocks/test_docs_blocks.py @@ -158,3 +158,21 @@ def test_postgres_invalid_doc_ref(self): # The run should fail since we could not find the docs reference. with self.assertRaises(dbt.exceptions.CompilationException): self.run_dbt(expect_pass=False) + +class TestDuplicateDocsBlock(DBTIntegrationTest): + @property + def schema(self): + return 'docs_blocks_035' + + @staticmethod + def dir(path): + return os.path.normpath(path) + + @property + def models(self): + return self.dir("duplicate_docs") + + @use_profile('postgres') + def test_postgres_duplicate_doc_ref(self): + with self.assertRaises(dbt.exceptions.CompilationException): + self.run_dbt(expect_pass=False) diff --git a/test/rpc/test_base.py b/test/rpc/test_base.py index 140741bb267..14694f155a3 100644 --- a/test/rpc/test_base.py +++ b/test/rpc/test_base.py @@ -863,3 +863,25 @@ def test_missing_tag_sighup( querier.sighup() assert querier.wait_for_status('ready') is True + + +def test_rpc_vars( + project_root, profiles_root, postgres_profile, unique_schema +): + project = ProjectDefinition( + models={ + 'my_model.sql': 'select {{ var("param") }} as id', + }, + ) + querier_ctx = get_querier( + project_def=project, + project_dir=project_root, + profiles_dir=profiles_root, + schema=unique_schema, + test_kwargs={}, + ) + + with querier_ctx as querier: + results = querier.async_wait_for_result(querier.cli_args('run --vars "{param: 100}"')) + assert len(results['results']) == 1 + assert results['results'][0]['node']['compiled_sql'] == 'select 100 as id' diff --git a/test/unit/test_contracts_graph_compiled.py b/test/unit/test_contracts_graph_compiled.py index 2bb95fefe97..c30a122079b 100644 --- a/test/unit/test_contracts_graph_compiled.py +++ b/test/unit/test_contracts_graph_compiled.py @@ -14,6 +14,22 @@ class TestCompiledModelNode(ContractTestCase): ContractType = CompiledModelNode + def _minimum(self): + return { + 'name': 'foo', + 'root_path': '/root/', + 'resource_type': str(NodeType.Model), + 'path': '/root/x/path.sql', + 'original_file_path': '/root/path.sql', + 'package_name': 'test', + 'raw_sql': 'select * from wherever', + 'unique_id': 'model.test.foo', + 'fqn': ['test', 'models', 'foo'], + 'database': 'test_db', + 'schema': 'test_schema', + 'alias': 'bar', + } + def test_basic_uncompiled(self): node_dict = { 'name': 'foo', @@ -44,6 +60,7 @@ def test_basic_uncompiled(self): 'tags': [], 'vars': {}, }, + 'docs': {'show': True}, 'columns': {}, 'meta': {}, 'compiled': False, @@ -80,20 +97,7 @@ def test_basic_uncompiled(self): self.assertFalse(node.is_ephemeral) self.assertEqual(node.local_vars(), {}) - minimum = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Model), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from wherever', - 'unique_id': 'model.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'database': 'test_db', - 'schema': 'test_schema', - 'alias': 'bar', - } + minimum = self._minimum() self.assert_from_dict(node, minimum) pickle.loads(pickle.dumps(node)) @@ -127,6 +131,7 @@ def test_basic_compiled(self): 'tags': [], 'vars': {}, }, + 'docs': {'show': True}, 'columns': {}, 'meta': {}, 'compiled': True, @@ -170,28 +175,24 @@ def test_basic_compiled(self): self.assertEqual(node.local_vars(), {}) def test_invalid_extra_fields(self): - bad_extra = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Model), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from wherever', - 'unique_id': 'model.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'database': 'test_db', - 'schema': 'test_schema', - 'alias': 'bar', - 'notvalid': 'nope', - } + bad_extra = self._minimum() + bad_extra['notvalid'] = 'nope' self.assert_fails_validation(bad_extra) def test_invalid_bad_type(self): - bad_type = { + bad_type = self._minimum() + bad_type['resource_type'] = str(NodeType.Macro) + self.assert_fails_validation(bad_type) + + +class TestCompiledTestNode(ContractTestCase): + ContractType = CompiledTestNode + + def _minimum(self): + return { 'name': 'foo', 'root_path': '/root/', - 'resource_type': str(NodeType.Macro), + 'resource_type': str(NodeType.Test), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', 'package_name': 'test', @@ -202,11 +203,6 @@ def test_invalid_bad_type(self): 'schema': 'test_schema', 'alias': 'bar', } - self.assert_fails_validation(bad_type) - - -class TestCompiledTestNode(ContractTestCase): - ContractType = CompiledTestNode def test_basic_uncompiled(self): node_dict = { @@ -239,6 +235,7 @@ def test_basic_uncompiled(self): 'vars': {}, 'severity': 'error', }, + 'docs': {'show': True}, 'columns': {}, 'meta': {}, 'compiled': False, @@ -275,20 +272,7 @@ def test_basic_uncompiled(self): self.assertFalse(node.is_ephemeral) self.assertEqual(node.local_vars(), {}) - minimum = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Test), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from wherever', - 'unique_id': 'model.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'database': 'test_db', - 'schema': 'test_schema', - 'alias': 'bar', - } + minimum = self._minimum() self.assert_from_dict(node, minimum) pickle.loads(pickle.dumps(node)) @@ -323,6 +307,7 @@ def test_basic_compiled(self): 'vars': {}, 'severity': 'warn', }, + 'docs': {'show': True}, 'columns': {}, 'meta': {}, 'compiled': True, @@ -368,36 +353,11 @@ def test_basic_compiled(self): self.assertEqual(node.local_vars(), {}) def test_invalid_extra_fields(self): - bad_extra = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Test), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from wherever', - 'unique_id': 'model.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'database': 'test_db', - 'schema': 'test_schema', - 'alias': 'bar', - 'extra': 'extra value', - } + bad_extra = self._minimum() + bad_extra['extra'] = 'extra value' self.assert_fails_validation(bad_extra) def test_invalid_resource_type(self): - bad_type = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Model), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from wherever', - 'unique_id': 'model.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'database': 'test_db', - 'schema': 'test_schema', - 'alias': 'bar', - } + bad_type = self._minimum() + bad_type['resource_type'] = str(NodeType.Model) self.assert_fails_validation(bad_type) diff --git a/test/unit/test_contracts_graph_parsed.py b/test/unit/test_contracts_graph_parsed.py index 27ab366f3c3..14623635b31 100644 --- a/test/unit/test_contracts_graph_parsed.py +++ b/test/unit/test_contracts_graph_parsed.py @@ -4,9 +4,9 @@ from dbt.contracts.graph.parsed import ( ParsedModelNode, DependsOn, NodeConfig, ColumnInfo, Hook, ParsedTestNode, TestConfig, ParsedSnapshotNode, TimestampSnapshotConfig, All, - GenericSnapshotConfig, CheckSnapshotConfig, SnapshotStrategy, - IntermediateSnapshotNode, ParsedNodePatch, ParsedMacro, - MacroDependsOn, ParsedSourceDefinition, ParsedDocumentation, ParsedHookNode + CheckSnapshotConfig, SnapshotStrategy, IntermediateSnapshotNode, + ParsedNodePatch, ParsedMacro, Docs, MacroDependsOn, ParsedSourceDefinition, + ParsedDocumentation, ParsedHookNode ) from dbt.contracts.graph.unparsed import Quoting @@ -59,8 +59,8 @@ def test_populated(self): class TestParsedModelNode(ContractTestCase): ContractType = ParsedModelNode - def test_ok(self): - node_dict = { + def _model_ok(self): + return { 'name': 'foo', 'root_path': '/root/', 'resource_type': str(NodeType.Model), @@ -89,9 +89,14 @@ def test_ok(self): 'tags': [], 'vars': {}, }, + 'docs': {'show': True}, 'columns': {}, 'meta': {}, } + + + def test_ok(self): + node_dict = self._model_ok() node = self.ContractType( package_name='test', root_path='/root/', @@ -167,6 +172,7 @@ def test_complex(self): 'tags': [], 'vars': {'foo': 100}, }, + 'docs': {'show': True}, 'columns': { 'a': { 'name': 'a', @@ -212,74 +218,14 @@ def test_complex(self): def test_invalid_bad_tags(self): # bad top-level field - bad_tags = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Model), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from wherever', - 'unique_id': 'model.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'refs': [], - 'sources': [], - 'depends_on': {'macros': [], 'nodes': []}, - 'database': 'test_db', - 'description': 'My parsed node', - 'schema': 'test_schema', - 'alias': 'bar', - 'tags': 100, - 'config': { - 'column_types': {}, - 'enabled': True, - 'materialized': None, - 'persist_docs': {}, - 'post-hook': [], - 'pre-hook': [], - 'quoting': {}, - 'tags': [], - 'vars': {}, - }, - 'columns': {}, - 'meta': {}, - } + bad_tags = self._model_ok() + bad_tags['tags'] = 100 self.assert_fails_validation(bad_tags) def test_invalid_bad_materialized(self): # bad nested field - bad_materialized = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Model), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from wherever', - 'unique_id': 'model.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'refs': [], - 'sources': [], - 'depends_on': {'macros': [], 'nodes': []}, - 'database': 'test_db', - 'description': 'My parsed node', - 'schema': 'test_schema', - 'alias': 'bar', - 'tags': ['tag'], - 'config': { - 'column_types': {}, - 'enabled': True, - 'materialized': None, - 'persist_docs': {}, - 'post-hook': [], - 'pre-hook': [], - 'quoting': {}, - 'tags': [], - 'vars': {}, - }, - 'columns': {}, - 'meta': {}, - } + bad_materialized = self._model_ok() + bad_materialized['config']['materialized'] = None self.assert_fails_validation(bad_materialized) def test_patch_ok(self): @@ -311,6 +257,7 @@ def test_patch_ok(self): description='The foo model', original_file_path='/path/to/schema.yml', columns={'a': ColumnInfo(name='a', description='a text field', meta={})}, + docs=Docs(), meta={}, ) @@ -355,6 +302,7 @@ def test_patch_ok(self): 'tags': [], }, }, + 'docs': {'show': True}, } expected = self.ContractType( @@ -379,6 +327,7 @@ def test_patch_ok(self): config=NodeConfig(), patch_path='/path/to/schema.yml', columns={'a': ColumnInfo(name='a', description='a text field', meta={})}, + docs=Docs(), ) self.assert_symmetric(expected, expected_dict) # sanity check self.assertEqual(initial, expected) @@ -413,6 +362,7 @@ def patch_invalid(self): description=None, original_file_path='/path/to/schema.yml', columns={}, + docs=Docs(), ) with self.assertRaises(ValidationError): initial.patch(patch) @@ -421,8 +371,8 @@ def patch_invalid(self): class TestParsedHookNode(ContractTestCase): ContractType = ParsedHookNode - def test_ok(self): - node_dict = { + def _hook_ok(self): + return { 'name': 'foo', 'root_path': '/root/', 'resource_type': str(NodeType.Operation), @@ -451,9 +401,14 @@ def test_ok(self): 'tags': [], 'vars': {}, }, + 'docs': {'show': True}, 'columns': {}, 'meta': {}, + 'index': 10, } + + def test_ok(self): + node_dict = self._hook_ok() node = self.ContractType( package_name='test', root_path='/root/', @@ -473,12 +428,14 @@ def test_ok(self): alias='bar', tags=[], config=NodeConfig(), + index=10, ) self.assert_symmetric(node, node_dict) self.assertFalse(node.empty) self.assertFalse(node.is_refable) self.assertEqual(node.get_materialization(), 'view') + node.index = None minimum = { 'name': 'foo', 'root_path': '/root/', @@ -527,6 +484,7 @@ def test_complex(self): 'tags': [], 'vars': {}, }, + 'docs': {'show': True}, 'columns': { 'a': { 'name': 'a', @@ -572,44 +530,75 @@ def test_complex(self): def test_invalid_index_type(self): # bad top-level field - bad_index = { + bad_index = self._hook_ok() + bad_index['index'] = 'a string!?' + self.assert_fails_validation(bad_index) + + +class TestParsedTestNode(ContractTestCase): + ContractType = ParsedTestNode + + def _minimum(self): + return { 'name': 'foo', 'root_path': '/root/', - 'resource_type': str(NodeType.Operation), + 'resource_type': str(NodeType.Test), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', 'package_name': 'test', 'raw_sql': 'select * from wherever', - 'unique_id': 'model.test.foo', + 'unique_id': 'test.test.foo', + 'fqn': ['test', 'models', 'foo'], + 'database': 'test_db', + 'schema': 'test_schema', + 'alias': 'bar', + 'meta': {}, + } + + def _complex(self): + return { + 'name': 'foo', + 'root_path': '/root/', + 'resource_type': str(NodeType.Test), + 'path': '/root/x/path.sql', + 'original_file_path': '/root/path.sql', + 'package_name': 'test', + 'raw_sql': 'select * from {{ ref("bar") }}', + 'unique_id': 'test.test.foo', 'fqn': ['test', 'models', 'foo'], 'refs': [], 'sources': [], - 'depends_on': {'macros': [], 'nodes': []}, + 'depends_on': {'macros': [], 'nodes': ['model.test.bar']}, 'database': 'test_db', 'description': 'My parsed node', 'schema': 'test_schema', 'alias': 'bar', - 'tags': [], + 'tags': ['tag'], + 'meta': {}, 'config': { - 'column_types': {}, + 'column_types': {'a': 'text'}, 'enabled': True, - 'materialized': None, + 'materialized': 'table', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'tags': [], 'vars': {}, + 'severity': 'WARN', + 'extra_key': 'extra value' }, - 'columns': {}, - 'meta': {}, - 'index': 'a string!?', + 'docs': {'show': False}, + 'columns': { + 'a': { + 'name': 'a', + 'description': 'a text field', + 'meta': {}, + 'tags': [], + }, + }, + 'column_name': 'id', } - self.assert_fails_validation(bad_index) - - -class TestParsedTestNode(ContractTestCase): - ContractType = ParsedTestNode def test_ok(self): node_dict = { @@ -643,6 +632,7 @@ def test_ok(self): 'vars': {}, 'severity': 'error', }, + 'docs': {'show': True}, 'columns': {}, } node = self.ContractType( @@ -672,67 +662,12 @@ def test_ok(self): self.assertFalse(node.is_refable) self.assertEqual(node.get_materialization(), 'view') - minimum = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Test), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from wherever', - 'unique_id': 'test.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'database': 'test_db', - 'schema': 'test_schema', - 'alias': 'bar', - 'meta': {}, - } + minimum = self._minimum() self.assert_from_dict(node, minimum) pickle.loads(pickle.dumps(node)) def test_complex(self): - node_dict = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Test), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from {{ ref("bar") }}', - 'unique_id': 'test.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'refs': [], - 'sources': [], - 'depends_on': {'macros': [], 'nodes': ['model.test.bar']}, - 'database': 'test_db', - 'description': 'My parsed node', - 'schema': 'test_schema', - 'alias': 'bar', - 'tags': ['tag'], - 'meta': {}, - 'config': { - 'column_types': {'a': 'text'}, - 'enabled': True, - 'materialized': 'table', - 'persist_docs': {}, - 'post-hook': [], - 'pre-hook': [], - 'quoting': {}, - 'tags': [], - 'vars': {}, - 'severity': 'WARN', - 'extra_key': 'extra value' - }, - 'columns': { - 'a': { - 'name': 'a', - 'description': 'a text field', - 'meta': {}, - 'tags': [], - }, - }, - 'column_name': 'id', - } + node_dict = self._complex() cfg = TestConfig( column_types={'a': 'text'}, @@ -763,127 +698,28 @@ def test_complex(self): config=cfg, columns={'a': ColumnInfo('a', 'a text field',{})}, column_name='id', + docs=Docs(show=False), ) self.assert_symmetric(node, node_dict) self.assertFalse(node.empty) def test_invalid_column_name_type(self): # bad top-level field - bad_column_name = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Test), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from wherever', - 'unique_id': 'model.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'refs': [], - 'sources': [], - 'depends_on': {'macros': [], 'nodes': []}, - 'database': 'test_db', - 'description': 'My parsed node', - 'schema': 'test_schema', - 'alias': 'bar', - 'tags': 100, - 'config': { - 'column_types': {}, - 'enabled': True, - 'materialized': None, - 'persist_docs': {}, - 'post-hook': [], - 'pre-hook': [], - 'quoting': {}, - 'tags': [], - 'vars': {}, - 'severity': 'ERROR', - }, - 'columns': {}, - 'column_name': {}, - 'meta': {}, - } + bad_column_name = self._complex() + bad_column_name['column_name'] = {} self.assert_fails_validation(bad_column_name) - def test_invalid_missing_severity(self): - # note the typo ('severtiy') - missing_config_value = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Test), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from wherever', - 'unique_id': 'model.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'refs': [], - 'sources': [], - 'depends_on': {'macros': [], 'nodes': []}, - 'database': 'test_db', - 'description': 'My parsed node', - 'schema': 'test_schema', - 'alias': 'bar', - 'tags': ['tag'], - 'config': { - 'column_types': {}, - 'enabled': True, - 'materialized': None, - 'persist_docs': {}, - 'post-hook': [], - 'pre-hook': [], - 'quoting': {}, - 'tags': [], - 'vars': {}, - 'severtiy': 'WARN', - }, - 'columns': {}, - 'meta': {}, - } - self.assert_fails_validation(missing_config_value) - def test_invalid_severity(self): - invalid_config_value = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Test), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from wherever', - 'unique_id': 'model.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'refs': [], - 'sources': [], - 'depends_on': {'macros': [], 'nodes': []}, - 'database': 'test_db', - 'description': 'My parsed node', - 'schema': 'test_schema', - 'alias': 'bar', - 'tags': ['tag'], - 'config': { - 'column_types': {}, - 'enabled': True, - 'materialized': None, - 'persist_docs': {}, - 'post-hook': [], - 'pre-hook': [], - 'quoting': {}, - 'tags': [], - 'vars': {}, - 'severity': 'WERROR', # invalid severity - }, - 'columns': {}, - 'meta': {}, - } + invalid_config_value = self._complex() + invalid_config_value['config']['severity'] = 'WERROR' self.assert_fails_validation(invalid_config_value) class TestTimestampSnapshotConfig(ContractTestCase): ContractType = TimestampSnapshotConfig - def test_basics(self): - cfg_dict = { + def _cfg_basic(self): + return { 'column_types': {}, 'enabled': True, 'materialized': 'view', @@ -899,6 +735,9 @@ def test_basics(self): 'target_database': 'some_snapshot_db', 'target_schema': 'some_snapshot_schema', } + + def test_basics(self): + cfg_dict = self._cfg_basic() cfg = self.ContractType( strategy=SnapshotStrategy.Timestamp, updated_at='last_update', @@ -942,49 +781,22 @@ def test_populated(self): self.assert_symmetric(cfg, cfg_dict) def test_invalid_wrong_strategy(self): - bad_type = { - 'column_types': {}, - 'enabled': True, - 'materialized': 'view', - 'persist_docs': {}, - 'post-hook': [], - 'pre-hook': [], - 'quoting': {}, - 'tags': [], - 'vars': {}, - 'target_database': 'some_snapshot_db', - 'target_schema': 'some_snapshot_schema', - 'unique_key': 'id', - 'strategy': 'check', - 'updated_at': 'last_update', - } + bad_type = self._cfg_basic() + bad_type['strategy'] = 'check' self.assert_fails_validation(bad_type) def test_invalid_missing_updated_at(self): - bad_fields = { - 'column_types': {}, - 'enabled': True, - 'materialized': 'view', - 'persist_docs': {}, - 'post-hook': [], - 'pre-hook': [], - 'quoting': {}, - 'tags': [], - 'vars': {}, - 'target_database': 'some_snapshot_db', - 'target_schema': 'some_snapshot_schema', - 'unique_key': 'id', - 'strategy': 'timestamp', - 'check_cols': 'all' - } + bad_fields = self._cfg_basic() + del bad_fields['updated_at'] + bad_fields['check_cols'] = 'all' self.assert_fails_validation(bad_fields) class TestCheckSnapshotConfig(ContractTestCase): ContractType = CheckSnapshotConfig - def test_basics(self): - cfg_dict = { + def _cfg_ok(self): + return { 'column_types': {}, 'enabled': True, 'materialized': 'view', @@ -1000,6 +812,9 @@ def test_basics(self): 'strategy': 'check', 'check_cols': 'all', } + + def test_basics(self): + cfg_dict = self._cfg_ok() cfg = self.ContractType( strategy=SnapshotStrategy.Check, check_cols=All.All, @@ -1043,68 +858,26 @@ def test_populated(self): self.assert_symmetric(cfg, cfg_dict) def test_invalid_wrong_strategy(self): - wrong_strategy = { - 'column_types': {}, - 'enabled': True, - 'materialized': 'view', - 'persist_docs': {}, - 'post-hook': [], - 'pre-hook': [], - 'quoting': {}, - 'tags': [], - 'vars': {}, - 'target_database': 'some_snapshot_db', - 'target_schema': 'some_snapshot_schema', - 'unique_key': 'id', - 'strategy': 'timestamp', - 'check_cols': 'all', - } + wrong_strategy = self._cfg_ok() + wrong_strategy['strategy'] = 'timestamp' self.assert_fails_validation(wrong_strategy) def test_invalid_missing_check_cols(self): - wrong_fields = { - 'column_types': {}, - 'enabled': True, - 'materialized': 'view', - 'persist_docs': {}, - 'post-hook': [], - 'pre-hook': [], - 'quoting': {}, - 'tags': [], - 'vars': {}, - 'target_database': 'some_snapshot_db', - 'target_schema': 'some_snapshot_schema', - 'unique_key': 'id', - 'strategy': 'check', - 'updated_at': 'last_update' - } + wrong_fields = self._cfg_ok() + del wrong_fields['check_cols'] self.assert_fails_validation(wrong_fields) def test_invalid_check_value(self): - invalid_check_type = { - 'column_types': {}, - 'enabled': True, - 'materialized': 'view', - 'persist_docs': {}, - 'post-hook': [], - 'pre-hook': [], - 'quoting': {}, - 'tags': [], - 'vars': {}, - 'target_database': 'some_snapshot_db', - 'target_schema': 'some_snapshot_schema', - 'unique_key': 'id', - 'strategy': 'timestamp', - 'check_cols': 'some', - } + invalid_check_type = self._cfg_ok() + invalid_check_type['check_cols'] = 'some' self.assert_fails_validation(invalid_check_type) class TestParsedSnapshotNode(ContractTestCase): ContractType = ParsedSnapshotNode - def test_timestamp_ok(self): - node_dict = { + def _ts_ok(self): + return { 'name': 'foo', 'root_path': '/root/', 'resource_type': str(NodeType.Snapshot), @@ -1138,10 +911,14 @@ def test_timestamp_ok(self): 'strategy': 'timestamp', 'updated_at': 'last_update', }, + 'docs': {'show': True}, 'columns': {}, 'meta': {}, } + def test_timestamp_ok(self): + node_dict = self._ts_ok() + node = self.ContractType( package_name='test', root_path='/root/', @@ -1243,6 +1020,7 @@ def test_check_ok(self): 'strategy': 'check', 'check_cols': 'all', }, + 'docs': {'show': True}, 'columns': {}, 'meta': {}, } @@ -1312,43 +1090,8 @@ def test_check_ok(self): self.assertFalse(node.is_ephemeral) def test_invalid_bad_resource_type(self): - bad_resource_type = { - 'name': 'foo', - 'root_path': '/root/', - 'resource_type': str(NodeType.Model), - 'path': '/root/x/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': 'select * from wherever', - 'unique_id': 'model.test.foo', - 'fqn': ['test', 'models', 'foo'], - 'refs': [], - 'sources': [], - 'depends_on': {'macros': [], 'nodes': []}, - 'database': 'test_db', - 'description': '', - 'schema': 'test_schema', - 'alias': 'bar', - 'tags': [], - 'config': { - 'column_types': {}, - 'enabled': True, - 'materialized': 'view', - 'persist_docs': {}, - 'post-hook': [], - 'pre-hook': [], - 'quoting': {}, - 'tags': [], - 'vars': {}, - 'target_database': 'some_snapshot_db', - 'target_schema': 'some_snapshot_schema', - 'unique_key': 'id', - 'strategy': 'timestamp', - 'updated_at': 'last_update', - }, - 'columns': {}, - 'meta': {}, - } + bad_resource_type = self._ts_ok() + bad_resource_type['resource_type'] = str(NodeType.Model) self.assert_fails_validation(bad_resource_type) @@ -1361,6 +1104,7 @@ def test_empty(self): 'description': 'The foo model', 'original_file_path': '/path/to/schema.yml', 'columns': {}, + 'docs': {'show': True}, 'meta': {}, 'yaml_key': 'models', 'package_name': 'test', @@ -1372,6 +1116,7 @@ def test_empty(self): package_name='test', original_file_path='/path/to/schema.yml', columns={}, + docs=Docs(), meta={}, ) self.assert_symmetric(patch, dct) @@ -1389,6 +1134,7 @@ def test_populated(self): 'tags': [], }, }, + 'docs': {'show': False}, 'meta': {'key': ['value']}, 'yaml_key': 'models', 'package_name': 'test', @@ -1401,6 +1147,7 @@ def test_populated(self): meta={'key': ['value']}, yaml_key='models', package_name='test', + docs=Docs(show=False), ) self.assert_symmetric(patch, dct) pickle.loads(pickle.dumps(patch)) @@ -1409,8 +1156,8 @@ def test_populated(self): class TestParsedMacro(ContractTestCase): ContractType = ParsedMacro - def test_ok(self): - macro_dict = { + def _ok_dict(self): + return { 'name': 'foo', 'path': '/root/path.sql', 'original_file_path': '/root/path.sql', @@ -1425,6 +1172,9 @@ def test_ok(self): 'description': 'my macro description', 'arguments': [], } + + def test_ok(self): + macro_dict = self._ok_dict() macro = ParsedMacro( name='foo', path='/root/path.sql', @@ -1445,47 +1195,22 @@ def test_ok(self): pickle.loads(pickle.dumps(macro)) def test_invalid_missing_unique_id(self): - bad_missing_uid = { - 'name': 'foo', - 'path': '/root/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': '{% macro foo() %}select 1 as id{% endmacro %}', - 'root_path': '/root/', - 'resource_type': 'macro', - 'tags': [], - 'depends_on': {'macros': []}, - 'meta': {}, - 'description': 'my macro description', - 'arguments': [], - } + bad_missing_uid = self._ok_dict() + del bad_missing_uid['unique_id'] self.assert_fails_validation(bad_missing_uid) def test_invalid_extra_field(self): - bad_extra_field = { - 'name': 'foo', - 'path': '/root/path.sql', - 'original_file_path': '/root/path.sql', - 'package_name': 'test', - 'raw_sql': '{% macro foo() %}select 1 as id{% endmacro %}', - 'root_path': '/root/', - 'resource_type': 'macro', - 'unique_id': 'macro.test.foo', - 'tags': [], - 'depends_on': {'macros': []}, - 'meta': {}, - 'description': 'my macro description', - 'arguments': [], - 'extra': 'too many fields' - } + + bad_extra_field = self._ok_dict() + bad_extra_field['extra'] = 'too many fields' self.assert_fails_validation(bad_extra_field) class TestParsedDocumentation(ContractTestCase): ContractType = ParsedDocumentation - def test_ok(self): - doc_dict = { + def _ok_dict(self): + return { 'block_contents': 'some doc contents', 'file_contents': '{% doc foo %}some doc contents{% enddoc %}', 'name': 'foo', @@ -1495,6 +1220,9 @@ def test_ok(self): 'root_path': '/root', 'unique_id': 'test.foo', } + + def test_ok(self): + doc_dict = self._ok_dict() doc = self.ContractType( package_name='test', root_path='/root', @@ -1509,37 +1237,37 @@ def test_ok(self): pickle.loads(pickle.dumps(doc)) def test_invalid_missing(self): - bad_missing_contents = { - # 'block_contents': 'some doc contents', - 'file_contents': '{% doc foo %}some doc contents{% enddoc %}', - 'name': 'foo', - 'original_file_path': '/root/docs/doc.md', - 'package_name': 'test', - 'path': '/root/docs', - 'root_path': '/root', - 'unique_id': 'test.foo', - } + bad_missing_contents = self._ok_dict() + del bad_missing_contents['block_contents'] self.assert_fails_validation(bad_missing_contents) def test_invalid_extra(self): - bad_extra_field = { - 'block_contents': 'some doc contents', - 'file_contents': '{% doc foo %}some doc contents{% enddoc %}', - 'name': 'foo', - 'original_file_path': '/root/docs/doc.md', - 'package_name': 'test', - 'path': '/root/docs', - 'root_path': '/root', - 'unique_id': 'test.foo', - - 'extra': 'more', - } + bad_extra_field = self._ok_dict() + bad_extra_field['extra'] = 'more' self.assert_fails_validation(bad_extra_field) class TestParsedSourceDefinition(ContractTestCase): ContractType = ParsedSourceDefinition + def _minimum_dict(self): + return { + 'package_name': 'test', + 'root_path': '/root', + 'path': '/root/models/sources.yml', + 'original_file_path': '/root/models/sources.yml', + 'database': 'some_db', + 'schema': 'some_schema', + 'fqn': ['test', 'source', 'my_source', 'my_source_table'], + 'source_name': 'my_source', + 'name': 'my_source_table', + 'source_description': 'my source description', + 'loader': 'stitch', + 'identifier': 'my_source_table', + 'resource_type': str(NodeType.Source), + 'unique_id': 'test.source.my_source.my_source_table', + } + def test_basic(self): source_def_dict = { 'package_name': 'test', @@ -1584,59 +1312,16 @@ def test_basic(self): tags=[], ) self.assert_symmetric(source_def, source_def_dict) - minimum = { - 'package_name': 'test', - 'root_path': '/root', - 'path': '/root/models/sources.yml', - 'original_file_path': '/root/models/sources.yml', - 'database': 'some_db', - 'schema': 'some_schema', - 'fqn': ['test', 'source', 'my_source', 'my_source_table'], - 'source_name': 'my_source', - 'name': 'my_source_table', - 'source_description': 'my source description', - 'loader': 'stitch', - 'identifier': 'my_source_table', - 'resource_type': str(NodeType.Source), - 'unique_id': 'test.source.my_source.my_source_table', - } + minimum = self._minimum_dict() self.assert_from_dict(source_def, minimum) pickle.loads(pickle.dumps(source_def)) def test_invalid_missing(self): - bad_missing_name = { - 'package_name': 'test', - 'root_path': '/root', - 'path': '/root/models/sources.yml', - 'original_file_path': '/root/models/sources.yml', - 'database': 'some_db', - 'schema': 'some_schema', - 'fqn': ['test', 'source', 'my_source', 'my_source_table'], - 'source_name': 'my_source', - # 'name': 'my_source_table', - 'source_description': 'my source description', - 'loader': 'stitch', - 'identifier': 'my_source_table', - 'resource_type': str(NodeType.Source), - 'unique_id': 'test.source.my_source.my_source_table', - } + bad_missing_name = self._minimum_dict() + del bad_missing_name['name'] self.assert_fails_validation(bad_missing_name) def test_invalid_bad_resource_type(self): - bad_resource_type = { - 'package_name': 'test', - 'root_path': '/root', - 'path': '/root/models/sources.yml', - 'original_file_path': '/root/models/sources.yml', - 'database': 'some_db', - 'schema': 'some_schema', - 'fqn': ['test', 'source', 'my_source', 'my_source_table'], - 'source_name': 'my_source', - 'name': 'my_source_table', - 'source_description': 'my source description', - 'loader': 'stitch', - 'identifier': 'my_source_table', - 'resource_type': str(NodeType.Model), - 'unique_id': 'test.source.my_source.my_source_table', - } + bad_resource_type = self._minimum_dict() + bad_resource_type['resource_type'] = str(NodeType.Model) self.assert_fails_validation(bad_resource_type) diff --git a/test/unit/test_contracts_graph_unparsed.py b/test/unit/test_contracts_graph_unparsed.py index d037a69e7d1..0ae9e83b9dc 100644 --- a/test/unit/test_contracts_graph_unparsed.py +++ b/test/unit/test_contracts_graph_unparsed.py @@ -5,7 +5,7 @@ UnparsedNode, UnparsedRunHook, UnparsedMacro, Time, TimePeriod, FreshnessStatus, FreshnessThreshold, Quoting, UnparsedSourceDefinition, UnparsedSourceTableDefinition, UnparsedDocumentationFile, UnparsedColumn, - UnparsedNodeUpdate + UnparsedNodeUpdate, Docs ) from dbt.node_types import NodeType from .utils import ContractTestCase @@ -312,6 +312,7 @@ def test_table_defaults(self): { 'name': 'table1', 'description': '', + 'docs': {'show': True}, 'tests': [], 'columns': [], 'quoting': {}, @@ -323,6 +324,7 @@ def test_table_defaults(self): { 'name': 'table2', 'description': 'table 2', + 'docs': {'show': True}, 'tests': [], 'columns': [], 'quoting': {'database': True}, @@ -398,6 +400,7 @@ def test_defaults(self): 'package_name': 'test', 'columns': [], 'description': '', + 'docs': {'show': True}, 'tests': [], 'meta': {}, } @@ -430,6 +433,7 @@ def test_contents(self): tags=['a', 'b'], ), ], + docs=Docs(show=False), ) dct = { 'name': 'foo', @@ -443,6 +447,7 @@ def test_contents(self): { 'name': 'x', 'description': 'x description', + 'docs': {'show': True}, 'tests': [], 'meta': {'key2': 'value3'}, 'tags': [], @@ -450,6 +455,7 @@ def test_contents(self): { 'name': 'y', 'description': 'y description', + 'docs': {'show': True}, 'tests': [ 'unique', {'accepted_values': {'values': ['blue', 'green']}} @@ -458,6 +464,7 @@ def test_contents(self): 'tags': ['a', 'b'], }, ], + 'docs': {'show': False}, } self.assert_symmetric(update, dct) pickle.loads(pickle.dumps(update)) @@ -475,12 +482,14 @@ def test_bad_test_type(self): { 'name': 'x', 'description': 'x description', + 'docs': {'show': True}, 'tests': [], 'meta': {'key2': 'value3'}, }, { 'name': 'y', 'description': 'y description', + 'docs': {'show': True}, 'tests': [ 100, {'accepted_values': {'values': ['blue', 'green']}} @@ -490,6 +499,7 @@ def test_bad_test_type(self): 'original_file_path': '/some/fake/path', }, ], + 'docs': {'show': True}, } self.assert_fails_validation(dct) @@ -505,12 +515,15 @@ def test_bad_test_type(self): # column missing a name { 'description': 'x description', + 'docs': {'show': True}, 'tests': [], 'meta': {'key2': 'value3'}, + }, { 'name': 'y', 'description': 'y description', + 'docs': {'show': True}, 'tests': [ 'unique', {'accepted_values': {'values': ['blue', 'green']}} @@ -520,6 +533,7 @@ def test_bad_test_type(self): 'original_file_path': '/some/fake/path', }, ], + 'docs': {'show': True}, } self.assert_fails_validation(dct) @@ -535,12 +549,14 @@ def test_bad_test_type(self): { 'name': 'x', 'description': 'x description', + 'docs': {'show': True}, 'tests': [], 'meta': {'key2': 'value3'}, }, { 'name': 'y', 'description': 'y description', + 'docs': {'show': True}, 'tests': [ 'unique', {'accepted_values': {'values': ['blue', 'green']}} @@ -550,5 +566,6 @@ def test_bad_test_type(self): 'original_file_path': '/some/fake/path', }, ], + 'docs': {'show': True}, } self.assert_fails_validation(dct) diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index ee08b23add4..8b5c0b87011 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -29,7 +29,7 @@ 'alias', 'tags', 'config', 'unique_id', 'refs', 'sources', 'meta', 'depends_on', 'database', 'schema', 'name', 'resource_type', 'package_name', 'root_path', 'path', 'original_file_path', 'raw_sql', - 'description', 'columns', 'fqn', 'build_path', 'patch_path', + 'description', 'columns', 'fqn', 'build_path', 'patch_path', 'docs', }) REQUIRED_COMPILED_NODE_KEYS = frozenset(REQUIRED_PARSED_NODE_KEYS | { diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 3e946cd5247..62632e5b45a 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -28,7 +28,9 @@ ParsedSnapshotNode, TimestampSnapshotConfig, SnapshotStrategy, ParsedAnalysisNode, ParsedDocumentation ) -from dbt.contracts.graph.unparsed import FreshnessThreshold, ExternalTable +from dbt.contracts.graph.unparsed import ( + FreshnessThreshold, ExternalTable, Docs +) from .utils import config_from_parts_or_dicts, normalize @@ -315,6 +317,7 @@ def test__parse_basic_model_tests(self): meta={}, yaml_key='models', package_name='snowplow', + docs=Docs(show=True), ) self.assertEqual(patch, expected_patch)