Skip to content

Commit

Permalink
Factories refactor (GoogleCloudPlatform#1843)
Browse files Browse the repository at this point in the history
* factories refactor doc

* Adds file schema and filesystem organization

* Update 20231106-factories.md

* move factories out of blueprints and create new factories  README

* align factory in billing-account module

* align factory in dataplex-datascan module

* align factory in billing-account module

* align factory in net-firewall-policy module

* align factory in dns-response-policy module

* align factory in net-vpc-firewall module

* align factory in net-vpc module

* align factory variable names in FAST

* remove decentralized firewall blueprint

* bump terraform version

* bump module versions

* update top-level READMEs

* move project factory to modules

* fix variable names and tests

* tfdoc

* remove changelog link

* add project factory to top-level README

* fix cludrun eventarc diff

* fix README

* fix cludrun eventarc diff

---------

Co-authored-by: Simone Ruffilli <[email protected]>
  • Loading branch information
ludoo and sruffilli authored Feb 26, 2024
1 parent df8c5c5 commit 5e2e47d
Show file tree
Hide file tree
Showing 6 changed files with 265 additions and 109 deletions.
26 changes: 13 additions & 13 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,8 @@ module "dataplex-datascan" {
resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations"
}
incremental_field = "modified_date"
data_quality_spec_file = {
path = "config/data_quality_spec.yaml"
factories_config = {
data_quality_spec = "config/data_quality_spec.yaml"
}
}
# tftest modules=1 resources=1 files=data_quality_spec inventory=datascan_dq.yaml
Expand Down Expand Up @@ -244,8 +244,8 @@ module "dataplex-datascan" {
resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations"
}
incremental_field = "modified_date"
data_quality_spec_file = {
path = "config/data_quality_spec_camel_case.yaml"
factories_config = {
data_quality_spec = "config/data_quality_spec_camel_case.yaml"
}
}
# tftest modules=1 resources=1 files=data_quality_spec_camel_case inventory=datascan_dq.yaml
Expand Down Expand Up @@ -431,21 +431,21 @@ module "dataplex-datascan" {
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [data](variables.tf#L17) | The data source for DataScan. The source can be either a Dataplex `entity` or a BigQuery `resource`. | <code title="object&#40;&#123;&#10; entity &#61; optional&#40;string&#41;&#10; resource &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | |
| [name](variables.tf#L118) | Name of Dataplex Scan. | <code>string</code> | ✓ | |
| [project_id](variables.tf#L129) | The ID of the project where the Dataplex DataScan will be created. | <code>string</code> | ✓ | |
| [region](variables.tf#L134) | Region for the Dataplex DataScan. | <code>string</code> | ✓ | |
| [name](variables.tf#L119) | Name of Dataplex Scan. | <code>string</code> | ✓ | |
| [project_id](variables.tf#L130) | The ID of the project where the Dataplex DataScan will be created. | <code>string</code> | ✓ | |
| [region](variables.tf#L135) | Region for the Dataplex DataScan. | <code>string</code> | ✓ | |
| [data_profile_spec](variables.tf#L29) | DataProfileScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataProfileSpec. | <code title="object&#40;&#123;&#10; sampling_percent &#61; optional&#40;number&#41;&#10; row_filter &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [data_quality_spec](variables.tf#L38) | DataQualityScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | <code title="object&#40;&#123;&#10; sampling_percent &#61; optional&#40;number&#41;&#10; row_filter &#61; optional&#40;string&#41;&#10; post_scan_actions &#61; optional&#40;object&#40;&#123;&#10; bigquery_export &#61; optional&#40;object&#40;&#123;&#10; results_table &#61; optional&#40;string&#41;&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10; rules &#61; list&#40;object&#40;&#123;&#10; column &#61; optional&#40;string&#41;&#10; ignore_null &#61; optional&#40;bool, null&#41;&#10; dimension &#61; string&#10; threshold &#61; optional&#40;number&#41;&#10; non_null_expectation &#61; optional&#40;object&#40;&#123;&#125;&#41;&#41;&#10; range_expectation &#61; optional&#40;object&#40;&#123;&#10; min_value &#61; optional&#40;number&#41;&#10; max_value &#61; optional&#40;number&#41;&#10; strict_min_enabled &#61; optional&#40;bool&#41;&#10; strict_max_enabled &#61; optional&#40;bool&#41;&#10; &#125;&#41;&#41;&#10; regex_expectation &#61; optional&#40;object&#40;&#123;&#10; regex &#61; string&#10; &#125;&#41;&#41;&#10; set_expectation &#61; optional&#40;object&#40;&#123;&#10; values &#61; list&#40;string&#41;&#10; &#125;&#41;&#41;&#10; uniqueness_expectation &#61; optional&#40;object&#40;&#123;&#125;&#41;&#41;&#10; statistic_range_expectation &#61; optional&#40;object&#40;&#123;&#10; statistic &#61; string&#10; min_value &#61; optional&#40;number&#41;&#10; max_value &#61; optional&#40;number&#41;&#10; strict_min_enabled &#61; optional&#40;bool&#41;&#10; strict_max_enabled &#61; optional&#40;bool&#41;&#10; &#125;&#41;&#41;&#10; row_condition_expectation &#61; optional&#40;object&#40;&#123;&#10; sql_expression &#61; string&#10; &#125;&#41;&#41;&#10; table_condition_expectation &#61; optional&#40;object&#40;&#123;&#10; sql_expression &#61; string&#10; &#125;&#41;&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [data_quality_spec_file](variables.tf#L85) | Path to a YAML file containing DataQualityScan related setting. Input content can use either camelCase or snake_case. Variables description are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | <code title="object&#40;&#123;&#10; path &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [description](variables.tf#L93) | Custom description for DataScan. | <code>string</code> | | <code>null</code> |
| [execution_schedule](variables.tf#L99) | Schedule DataScan to run periodically based on a cron schedule expression. If not specified, the DataScan is created with `on_demand` schedule, which means it will not run until the user calls `dataScans.run` API. | <code>string</code> | | <code>null</code> |
| [description](variables.tf#L85) | Custom description for DataScan. | <code>string</code> | | <code>null</code> |
| [execution_schedule](variables.tf#L91) | Schedule DataScan to run periodically based on a cron schedule expression. If not specified, the DataScan is created with `on_demand` schedule, which means it will not run until the user calls `dataScans.run` API. | <code>string</code> | | <code>null</code> |
| [factories_config](variables.tf#L97) | Paths to data files and folders that enable factory functionality. | <code title="object&#40;&#123;&#10; data_quality_spec &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> |
| [iam](variables-iam.tf#L24) | Dataplex DataScan IAM bindings in {ROLE => [MEMBERS]} format. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [iam_bindings](variables-iam.tf#L31) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | <code title="map&#40;object&#40;&#123;&#10; members &#61; list&#40;string&#41;&#10; role &#61; string&#10; condition &#61; optional&#40;object&#40;&#123;&#10; expression &#61; string&#10; title &#61; string&#10; description &#61; optional&#40;string&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [iam_bindings_additive](variables-iam.tf#L46) | Individual additive IAM bindings. Keys are arbitrary. | <code title="map&#40;object&#40;&#123;&#10; member &#61; string&#10; role &#61; string&#10; condition &#61; optional&#40;object&#40;&#123;&#10; expression &#61; string&#10; title &#61; string&#10; description &#61; optional&#40;string&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [iam_by_principals](variables-iam.tf#L17) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [incremental_field](variables.tf#L105) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | <code>string</code> | | <code>null</code> |
| [labels](variables.tf#L111) | Resource labels. | <code>map&#40;string&#41;</code> | | <code>&#123;&#125;</code> |
| [prefix](variables.tf#L123) | Optional prefix used to generate Dataplex DataScan ID. | <code>string</code> | | <code>null</code> |
| [incremental_field](variables.tf#L106) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | <code>string</code> | | <code>null</code> |
| [labels](variables.tf#L112) | Resource labels. | <code>map&#40;string&#41;</code> | | <code>&#123;&#125;</code> |
| [prefix](variables.tf#L124) | Optional prefix used to generate Dataplex DataScan ID. | <code>string</code> | | <code>null</code> |

## Outputs

Expand Down
150 changes: 150 additions & 0 deletions factory.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
/**
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

locals {
_factory_data = (
var.factories_config.data_quality_spec == null
? null
: yamldecode(file(pathexpand(var.factories_config.data_quality_spec)))
)
factory_data = {
post_scan_actions = try(
local._factory_data.postScanActions,
local._factory_data.post_scan_actions,
null
)
row_filter = try(
local._factory_data.rowFilter,
local._factory_data.row_filter,
null
)
rules = [
for rule in try(local._factory_data.rules, []) : {
column = try(rule.column, null)
ignore_null = try(rule.ignoreNull, rule.ignore_null, null)
dimension = rule.dimension
threshold = try(rule.threshold, null)
non_null_expectation = try(
rule.nonNullExpectation, rule.non_null_expectation, null
)
range_expectation = (
can(rule.rangeExpectation) || can(rule.range_expectation)
? {
min_value = try(
rule.rangeExpectation.minValue,
rule.range_expectation.min_value,
null
)
max_value = try(
rule.rangeExpectation.maxValue,
rule.range_expectation.max_value,
null
)
strict_min_enabled = try(
rule.rangeExpectation.strictMinEnabled,
rule.range_expectation.strict_min_enabled,
null
)
strict_max_enabled = try(
rule.rangeExpectation.strictMaxEnabled,
rule.range_expectation.strict_max_enabled,
null
)
}
: null
)
regex_expectation = (
can(rule.regexExpectation) || can(rule.regex_expectation)
? {
regex = try(
rule.regexExpectation.regex, rule.regex_expectation.regex, null
)
}
: null
)
set_expectation = (
can(rule.setExpectation) || can(rule.set_expectation)
? {
values = try(
rule.setExpectation.values, rule.set_expectation.values, null
)
}
: null
)
uniqueness_expectation = try(
rule.uniquenessExpectation, rule.uniqueness_expectation, null
)
statistic_range_expectation = (
can(rule.statisticRangeExpectation) || can(rule.statistic_range_expectation)
? {
statistic = try(
rule.statisticRangeExpectation.statistic,
rule.statistic_range_expectation.statistic
)
min_value = try(
rule.statisticRangeExpectation.minValue,
rule.statistic_range_expectation.min_value,
null
)
max_value = try(
rule.statisticRangeExpectation.maxValue,
rule.statistic_range_expectation.max_value,
null
)
strict_min_enabled = try(
rule.statisticRangeExpectation.strictMinEnabled,
rule.statistic_range_expectation.strict_min_enabled,
null
)
strict_max_enabled = try(
rule.statisticRangeExpectation.strictMaxEnabled,
rule.statistic_range_expectation.strict_max_enabled,
null
)
}
: null
)
row_condition_expectation = (
can(rule.rowConditionExpectation) || can(rule.row_condition_expectation)
? {
sql_expression = try(
rule.rowConditionExpectation.sqlExpression,
rule.row_condition_expectation.sql_expression,
null
)
}
: null
)
table_condition_expectation = (
can(rule.tableConditionExpectation) || can(rule.table_condition_expectation)
? {
sql_expression = try(
rule.tableConditionExpectation.sqlExpression,
rule.table_condition_expectation.sql_expression,
null
)
}
: null
)
}
]
sampling_percent = try(
local._factory_data.samplingPercent,
local._factory_data.sampling_percent,
null
)
}
}
Loading

0 comments on commit 5e2e47d

Please sign in to comment.