From ccc8e8ebd8e815f7d973268d02a1ab61b193dc57 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 6 Jul 2018 08:31:57 -0600 Subject: [PATCH 1/4] Allow support for the 'external' relation type in BigQuery --- dbt/adapters/bigquery/relation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index f0bb48c4309..1e696a848ee 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -3,6 +3,7 @@ class BigQueryRelation(DefaultRelation): + External = "external" DEFAULTS = { 'metadata': { @@ -54,7 +55,7 @@ class BigQueryRelation(DefaultRelation): }, }, 'type': { - 'enum': DefaultRelation.RelationTypes + [None], + 'enum': DefaultRelation.RelationTypes + [External, None], }, 'path': PATH_SCHEMA, 'include_policy': POLICY_SCHEMA, From 5dc13a707e958e789f06c5a580061df4b98f1d66 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 6 Jul 2018 09:57:29 -0600 Subject: [PATCH 2/4] Add some rudimentary tests of the bigquery adapter to make sure we accept external data sources --- test/unit/test_bigquery_adapter.py | 86 ++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/test/unit/test_bigquery_adapter.py b/test/unit/test_bigquery_adapter.py index 3f91ed5f6df..1ca78796141 100644 --- a/test/unit/test_bigquery_adapter.py +++ b/test/unit/test_bigquery_adapter.py @@ -4,6 +4,7 @@ import dbt.flags as flags from dbt.adapters.bigquery import BigQueryAdapter +from dbt.adapters.bigquery.relation import BigQueryRelation import dbt.exceptions from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -58,3 +59,88 @@ def test_acquire_connection_service_account_validations(self, mock_open_connecti self.fail('validation failed with unknown exception: {}'.format(str(e))) mock_open_connection.assert_called_once() + + +class TestBigQueryRelation(unittest.TestCase): + def setUp(self): + flags.STRICT_MODE = True + + def test_view_temp_relation(self): + kwargs = { + 'type': None, + 'path': { + 'project': 'test-project', + 'schema': 'test_schema', + 'identifier': 'my_view' + }, + 'table_name': 'my_view__dbt_tmp', + 'quote_policy': { + 'identifier': False + } + } + BigQueryRelation(**kwargs) + + def test_view_relation(self): + kwargs = { + 'type': 'view', + 'path': { + 'project': 'test-project', + 'schema': 'test_schema', + 'identifier': 'my_view' + }, + 'table_name': 'my_view', + 'quote_policy': { + 'identifier': True, + 'schema': True + } + } + BigQueryRelation(**kwargs) + + def test_table_relation(self): + kwargs = { + 'type': 'table', + 'path': { + 'project': 'test-project', + 'schema': 'test_schema', + 'identifier': 'generic_table' + }, + 'table_name': 'generic_table', + 'quote_policy': { + 'identifier': True, + 'schema': True + } + } + BigQueryRelation(**kwargs) + + def test_external_source_relation(self): + kwargs = { + 'type': 'external', + 'path': { + 'project': 'test-project', + 'schema': 'test_schema', + 'identifier': 'sheet' + }, + 'table_name': 'sheet', + 'quote_policy': { + 'identifier': True, + 'schema': True + } + } + BigQueryRelation(**kwargs) + + def test_invalid_relation(self): + kwargs = { + 'type': 'invalid-type', + 'path': { + 'project': 'test-project', + 'schema': 'test_schema', + 'identifier': 'my_invalid_id' + }, + 'table_name': 'my_invalid_id', + 'quote_policy': { + 'identifier': False, + 'schema': True + } + } + with self.assertRaises(dbt.exceptions.ValidationException): + BigQueryRelation(**kwargs) From 2581bbc0c2e3af49c47811608a3e7dceaa8ef929 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 6 Jul 2018 10:05:19 -0600 Subject: [PATCH 3/4] Not sure how this code worked before, but need to import 'logging.handlers' --- dbt/logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/logger.py b/dbt/logger.py index 2ad662aabe8..0f67159edcc 100644 --- a/dbt/logger.py +++ b/dbt/logger.py @@ -1,5 +1,6 @@ import dbt.compat import logging +import logging.handlers import os import sys From 2c935836d46a675ff8a7dbd58e75404ec44e525e Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 11 Jul 2018 16:14:02 -0400 Subject: [PATCH 4/4] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16fa12965cd..b3d09e9d8a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### Changes + - Support external BigQuery relations ([#828](https://github.com/fishtown-analytics/dbt/pull/828)) + - Set the TCP keepalive on redshift ([#826](https://github.com/fishtown-analytics/dbt/pull/826)) - Fix the error handling for profiles.yml validation ([#820](https://github.com/fishtown-analytics/dbt/pull/820)) - Make the `--threads` parameter actually change the number of threads used ([#819](https://github.com/fishtown-analytics/dbt/pull/819)) - Use Mapping instead of dict as the base class for APIObject ([#756](https://github.com/fishtown-analytics/dbt/pull/756))