diff --git a/CHANGELOG.md b/CHANGELOG.md index b5578fd1b1a..97f7945c2f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Added intersection syntax for model selector ([#2167](https://github.com/fishtown-analytics/dbt/issues/2167), [#2417](https://github.com/fishtown-analytics/dbt/pull/2417)) - Extends model selection syntax with at most n-th parent/children `dbt run --models 3+m1+2` ([#2052](https://github.com/fishtown-analytics/dbt/issues/2052), [#2485](https://github.com/fishtown-analytics/dbt/pull/2485)) - Added support for renaming BigQuery relations ([#2520](https://github.com/fishtown-analytics/dbt/issues/2520), [#2521](https://github.com/fishtown-analytics/dbt/pull/2521)) +- Added support for BigQuery authorized views ([#1718](https://github.com/fishtown-analytics/dbt/issues/1718), [#2517](https://github.com/fishtown-analytics/dbt/issues/2517)) ### Fixes - Fixed an error in create_adapter_plugins.py script when -dependency arg not passed ([#2507](https://github.com/fishtown-analytics/dbt/issues/2507), [#2508](https://github.com/fishtown-analytics/dbt/pull/2508)) @@ -18,8 +19,8 @@ Contributors: - [@raalsky](https://github.com/Raalsky) ([#2417](https://github.com/fishtown-analytics/dbt/pull/2417), [#2485](https://github.com/fishtown-analytics/dbt/pull/2485)) - [@alf-mindshift](https://github.com/alf-mindshift) ([#2431](https://github.com/fishtown-analytics/dbt/pull/2431)) - [@scarrucciu](https://github.com/scarrucciu) ([#2508](https://github.com/fishtown-analytics/dbt/pull/2508)) - - [@southpolemonkey](https://github.com/southpolemonkey)([#2511](https://github.com/fishtown-analytics/dbt/issues/2511)) - - [@azhard](https://github.com/azhard) ([#2521](https://github.com/fishtown-analytics/dbt/pull/2521) + - [@southpolemonkey](https://github.com/southpolemonkey) ([#2511](https://github.com/fishtown-analytics/dbt/issues/2511)) + - [@azhard](https://github.com/azhard) ([#2517](https://github.com/fishtown-analytics/dbt/issues/2517), ([#2521](https://github.com/fishtown-analytics/dbt/pull/2521))) ## dbt 0.17.0 (June 08, 2020) diff --git a/core/dbt/include/global_project/macros/adapters/common.sql b/core/dbt/include/global_project/macros/adapters/common.sql index 9dcc71003fe..cb48614f607 100644 --- a/core/dbt/include/global_project/macros/adapters/common.sql +++ b/core/dbt/include/global_project/macros/adapters/common.sql @@ -317,3 +317,4 @@ {% macro set_sql_header(config) -%} {{ config.set('sql_header', caller()) }} {%- endmacro %} + diff --git a/plugins/bigquery/dbt/adapters/bigquery/__init__.py b/plugins/bigquery/dbt/adapters/bigquery/__init__.py index daff48a32ee..c85e79b389a 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/__init__.py +++ b/plugins/bigquery/dbt/adapters/bigquery/__init__.py @@ -2,7 +2,7 @@ from dbt.adapters.bigquery.connections import BigQueryCredentials from dbt.adapters.bigquery.relation import BigQueryRelation # noqa from dbt.adapters.bigquery.column import BigQueryColumn # noqa -from dbt.adapters.bigquery.impl import BigQueryAdapter +from dbt.adapters.bigquery.impl import BigQueryAdapter, GrantTarget # noqa from dbt.adapters.base import AdapterPlugin from dbt.include import bigquery diff --git a/plugins/bigquery/dbt/adapters/bigquery/connections.py b/plugins/bigquery/dbt/adapters/bigquery/connections.py index ec07565ed0f..74e5d7d3690 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/connections.py +++ b/plugins/bigquery/dbt/adapters/bigquery/connections.py @@ -313,6 +313,10 @@ def dataset(database, schema, conn): dataset_ref = conn.handle.dataset(schema, database) return google.cloud.bigquery.Dataset(dataset_ref) + @staticmethod + def dataset_from_id(dataset_id): + return google.cloud.bigquery.Dataset.from_string(dataset_id) + def table_ref(self, database, schema, table_name, conn): dataset = self.dataset(database, schema, conn) return dataset.table(table_name) diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py index 762e579d634..325190aedfb 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py @@ -26,7 +26,7 @@ import google.cloud.exceptions import google.cloud.bigquery -from google.cloud.bigquery import SchemaField +from google.cloud.bigquery import AccessEntry, SchemaField import time import agate @@ -77,6 +77,15 @@ def parse(cls, raw_partition_by) -> Optional['PartitionConfig']: ) +@dataclass +class GrantTarget(JsonSchemaMixin): + dataset: str + project: str + + def render(self): + return f'{self.project}.{self.dataset}' + + def _stub_relation(*args, **kwargs): return BigQueryRelation.create( database='', @@ -94,6 +103,7 @@ class BigqueryConfig(AdapterConfig): kms_key_name: Optional[str] = None labels: Optional[Dict[str, str]] = None partitions: Optional[List[str]] = None + grant_access_to: Optional[List[Dict[str, str]]] = None class BigQueryAdapter(BaseAdapter): @@ -695,3 +705,35 @@ def get_table_options( opts['labels'] = list(labels.items()) return opts + + @available.parse_none + def grant_access_to(self, entity, entity_type, role, grant_target_dict): + """ + Given an entity, grants it access to a permissioned dataset. + """ + conn = self.connections.get_thread_connection() + client = conn.handle + + grant_target = GrantTarget.from_dict(grant_target_dict) + dataset = client.get_dataset( + self.connections.dataset_from_id(grant_target.render()) + ) + + if entity_type == 'view': + entity = self.connections.table_ref( + entity.database, + entity.schema, + entity.identifier, + conn).to_api_repr() + + access_entry = AccessEntry(role, entity_type, entity) + access_entries = dataset.access_entries + + if access_entry in access_entries: + logger.debug(f"Access entry {access_entry} " + f"already exists in dataset") + return + + access_entries.append(AccessEntry(role, entity_type, entity)) + dataset.access_entries = access_entries + client.update_dataset(dataset, ['access_entries']) diff --git a/plugins/bigquery/dbt/include/bigquery/macros/etc.sql b/plugins/bigquery/dbt/include/bigquery/macros/etc.sql index 7bd0cba9086..a10ad1a5bd0 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/etc.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/etc.sql @@ -1,4 +1,7 @@ - {% macro date_sharded_table(base_name) %} {{ return(base_name ~ "[DBT__PARTITION_DATE]") }} {% endmacro %} + +{% macro grant_access_to(entity, entity_type, role, grant_target_dict) -%} + {% do adapter.grant_access_to(entity, entity_type, role, grant_target_dict) %} +{% endmacro %} diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/view.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/view.sql index 3e43abfcf68..8879051293d 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/materializations/view.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/materializations/view.sql @@ -14,6 +14,12 @@ {% set target_relation = this.incorporate(type='view') %} {% do persist_docs(target_relation, model) %} + {% if config.get('grant_access_to') %} + {% for grant_target_dict in config.get('grant_access_to') %} + {% do adapter.grant_access_to(this, 'view', None, grant_target_dict) %} + {% endfor %} + {% endif %} + {% do return(to_return) %} {%- endmaterialization %} diff --git a/test/integration/054_adapter_methods_test/test_adapter_methods.py b/test/integration/054_adapter_methods_test/test_adapter_methods.py index c431550433b..1fcaf5b1a49 100644 --- a/test/integration/054_adapter_methods_test/test_adapter_methods.py +++ b/test/integration/054_adapter_methods_test/test_adapter_methods.py @@ -70,3 +70,52 @@ def test_bigquery_adapter_methods(self): }) self.run_dbt(['run-operation', 'rename_named_relation', '--args', rename_relation_args]) self.run_dbt() + + +class TestGrantAccess(DBTIntegrationTest): + @property + def schema(self): + return "grant_access_054" + + @property + def models(self): + return 'bigquery-models' + + @property + def project_config(self): + return { + 'config-version': 2, + 'source-paths': ['models'] + } + + @use_profile('bigquery') + def test_bigquery_adapter_methods(self): + from dbt.adapters.bigquery import GrantTarget + from google.cloud.bigquery import AccessEntry + + self.run_dbt(['compile']) # trigger any compile-time issues + self.run_sql_file("seed_bq.sql") + self.run_dbt(['seed']) + + ae_role = "READER" + ae_entity = "user@email.com" + ae_entity_type = "userByEmail" + ae_grant_target_dict = { + 'project': self.default_database, + 'dataset': self.unique_schema() + } + self.adapter.grant_access_to(ae_entity, ae_entity_type, ae_role, ae_grant_target_dict) + + conn = self.adapter.connections.get_thread_connection() + client = conn.handle + + grant_target = GrantTarget.from_dict(ae_grant_target_dict) + dataset = client.get_dataset( + self.adapter.connections.dataset_from_id(grant_target.render()) + ) + + expected_access_entry = AccessEntry(ae_role, ae_entity_type, ae_entity) + self.assertTrue(expected_access_entry in dataset.access_entries) + + unexpected_access_entry = AccessEntry(ae_role, ae_entity_type, "unexpected@email.com") + self.assertFalse(unexpected_access_entry in dataset.access_entries)