From a4adca1e38b1be6ebc45109489a9ccbe32a35b7b Mon Sep 17 00:00:00 2001 From: The Magician Date: Fri, 2 Jul 2021 11:15:45 -0500 Subject: [PATCH] add updates for big query kms changes (#4934) (#3406) * add updates for big query kms changes * update docs for bigquery_table Signed-off-by: Modular Magician --- .changelog/4934.txt | 3 + google-beta/resource_bigquery_job.go | 138 +++++++++++++----- google-beta/resource_bigquery_table.go | 21 ++- google-beta/resource_dataproc_cluster_test.go | 3 +- ...esource_gke_hub_feature_membership_test.go | 2 +- website/docs/r/bigquery_job.html.markdown | 9 ++ website/docs/r/bigquery_table.html.markdown | 2 + 7 files changed, 142 insertions(+), 36 deletions(-) create mode 100644 .changelog/4934.txt diff --git a/.changelog/4934.txt b/.changelog/4934.txt new file mode 100644 index 0000000000..c34a9fb066 --- /dev/null +++ b/.changelog/4934.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +bigquery: add `kms_key_version` as an output on `bigquery_table.encryption_configuration` and the `destination_encryption_configuration` blocks of `bigquery_job.query`, `bigquery_job.load`, and `bigquery_copy`. +``` diff --git a/google-beta/resource_bigquery_job.go b/google-beta/resource_bigquery_job.go index 2fbc41872b..7abd8f7b27 100644 --- a/google-beta/resource_bigquery_job.go +++ b/google-beta/resource_bigquery_job.go @@ -114,6 +114,11 @@ Creation, truncation and append actions occur as one atomic update upon job comp Description: `Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key.`, }, + "kms_key_version": { + Type: schema.TypeString, + Computed: true, + Description: `Describes the Cloud KMS encryption key version used to protect destination BigQuery table.`, + }, }, }, }, @@ -407,6 +412,11 @@ Creation, truncation and append actions occur as one atomic update upon job comp Description: `Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key.`, }, + "kms_key_version": { + Type: schema.TypeString, + Computed: true, + Description: `Describes the Cloud KMS encryption key version used to protect destination BigQuery table.`, + }, }, }, }, @@ -645,6 +655,11 @@ or of the form 'projects/{{project}}/datasets/{{dataset_id}}' if not.`, Description: `Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key.`, }, + "kms_key_version": { + Type: schema.TypeString, + Computed: true, + Description: `Describes the Cloud KMS encryption key version used to protect destination BigQuery table.`, + }, }, }, }, @@ -1344,21 +1359,29 @@ func flattenBigQueryJobConfigurationQuerySchemaUpdateOptions(v interface{}, d *s return v } +// KmsKeyName switched from using a key name to a key version, this will separate the key name from the key version and save them +// separately in state. https://github.com/hashicorp/terraform-provider-google/issues/9208 func flattenBigQueryJobConfigurationQueryDestinationEncryptionConfiguration(v interface{}, d *schema.ResourceData, config *Config) interface{} { if v == nil { - return nil + return []map[string]interface{}{} } - original := v.(map[string]interface{}) - if len(original) == 0 { - return nil + + kmsKeyName := v.(map[string]interface{})["kmsKeyName"].(string) + re := regexp.MustCompile(`(projects/.*/locations/.*/keyRings/.*/cryptoKeys/.*)/cryptoKeyVersions/.*`) + paths := re.FindStringSubmatch(kmsKeyName) + + if len(paths) > 0 { + return []map[string]interface{}{ + { + "kms_key_name": paths[0], + "kms_key_version": kmsKeyName, + }, + } } - transformed := make(map[string]interface{}) - transformed["kms_key_name"] = - flattenBigQueryJobConfigurationQueryDestinationEncryptionConfigurationKmsKeyName(original["kmsKeyName"], d, config) - return []interface{}{transformed} -} -func flattenBigQueryJobConfigurationQueryDestinationEncryptionConfigurationKmsKeyName(v interface{}, d *schema.ResourceData, config *Config) interface{} { - return v + + // The key name was returned, no need to set the version + return []map[string]interface{}{{"kms_key_name": kmsKeyName, "kms_key_version": ""}} + } func flattenBigQueryJobConfigurationQueryScriptOptions(v interface{}, d *schema.ResourceData, config *Config) interface{} { @@ -1578,21 +1601,29 @@ func flattenBigQueryJobConfigurationLoadTimePartitioningField(v interface{}, d * return v } +// KmsKeyName switched from using a key name to a key version, this will separate the key name from the key version and save them +// separately in state. https://github.com/hashicorp/terraform-provider-google/issues/9208 func flattenBigQueryJobConfigurationLoadDestinationEncryptionConfiguration(v interface{}, d *schema.ResourceData, config *Config) interface{} { if v == nil { - return nil + return []map[string]interface{}{} } - original := v.(map[string]interface{}) - if len(original) == 0 { - return nil + + kmsKeyName := v.(map[string]interface{})["kmsKeyName"].(string) + re := regexp.MustCompile(`(projects/.*/locations/.*/keyRings/.*/cryptoKeys/.*)/cryptoKeyVersions/.*`) + paths := re.FindStringSubmatch(kmsKeyName) + + if len(paths) > 0 { + return []map[string]interface{}{ + { + "kms_key_name": paths[0], + "kms_key_version": kmsKeyName, + }, + } } - transformed := make(map[string]interface{}) - transformed["kms_key_name"] = - flattenBigQueryJobConfigurationLoadDestinationEncryptionConfigurationKmsKeyName(original["kmsKeyName"], d, config) - return []interface{}{transformed} -} -func flattenBigQueryJobConfigurationLoadDestinationEncryptionConfigurationKmsKeyName(v interface{}, d *schema.ResourceData, config *Config) interface{} { - return v + + // The key name was returned, no need to set the version + return []map[string]interface{}{{"kms_key_name": kmsKeyName, "kms_key_version": ""}} + } func flattenBigQueryJobConfigurationCopy(v interface{}, d *schema.ResourceData, config *Config) interface{} { @@ -1672,21 +1703,29 @@ func flattenBigQueryJobConfigurationCopyWriteDisposition(v interface{}, d *schem return v } +// KmsKeyName switched from using a key name to a key version, this will separate the key name from the key version and save them +// separately in state. https://github.com/hashicorp/terraform-provider-google/issues/9208 func flattenBigQueryJobConfigurationCopyDestinationEncryptionConfiguration(v interface{}, d *schema.ResourceData, config *Config) interface{} { if v == nil { - return nil + return []map[string]interface{}{} } - original := v.(map[string]interface{}) - if len(original) == 0 { - return nil + + kmsKeyName := v.(map[string]interface{})["kmsKeyName"].(string) + re := regexp.MustCompile(`(projects/.*/locations/.*/keyRings/.*/cryptoKeys/.*)/cryptoKeyVersions/.*`) + paths := re.FindStringSubmatch(kmsKeyName) + + if len(paths) > 0 { + return []map[string]interface{}{ + { + "kms_key_name": paths[0], + "kms_key_version": kmsKeyName, + }, + } } - transformed := make(map[string]interface{}) - transformed["kms_key_name"] = - flattenBigQueryJobConfigurationCopyDestinationEncryptionConfigurationKmsKeyName(original["kmsKeyName"], d, config) - return []interface{}{transformed} -} -func flattenBigQueryJobConfigurationCopyDestinationEncryptionConfigurationKmsKeyName(v interface{}, d *schema.ResourceData, config *Config) interface{} { - return v + + // The key name was returned, no need to set the version + return []map[string]interface{}{{"kms_key_name": kmsKeyName, "kms_key_version": ""}} + } func flattenBigQueryJobConfigurationExtract(v interface{}, d *schema.ResourceData, config *Config) interface{} { @@ -2259,6 +2298,13 @@ func expandBigQueryJobConfigurationQueryDestinationEncryptionConfiguration(v int transformed["kmsKeyName"] = transformedKmsKeyName } + transformedKmsKeyVersion, err := expandBigQueryJobConfigurationQueryDestinationEncryptionConfigurationKmsKeyVersion(original["kms_key_version"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedKmsKeyVersion); val.IsValid() && !isEmptyValue(val) { + transformed["kmsKeyVersion"] = transformedKmsKeyVersion + } + return transformed, nil } @@ -2266,6 +2312,10 @@ func expandBigQueryJobConfigurationQueryDestinationEncryptionConfigurationKmsKey return v, nil } +func expandBigQueryJobConfigurationQueryDestinationEncryptionConfigurationKmsKeyVersion(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + func expandBigQueryJobConfigurationQueryScriptOptions(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { l := v.([]interface{}) if len(l) == 0 || l[0] == nil { @@ -2614,6 +2664,13 @@ func expandBigQueryJobConfigurationLoadDestinationEncryptionConfiguration(v inte transformed["kmsKeyName"] = transformedKmsKeyName } + transformedKmsKeyVersion, err := expandBigQueryJobConfigurationLoadDestinationEncryptionConfigurationKmsKeyVersion(original["kms_key_version"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedKmsKeyVersion); val.IsValid() && !isEmptyValue(val) { + transformed["kmsKeyVersion"] = transformedKmsKeyVersion + } + return transformed, nil } @@ -2621,6 +2678,10 @@ func expandBigQueryJobConfigurationLoadDestinationEncryptionConfigurationKmsKeyN return v, nil } +func expandBigQueryJobConfigurationLoadDestinationEncryptionConfigurationKmsKeyVersion(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + func expandBigQueryJobConfigurationCopy(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { l := v.([]interface{}) if len(l) == 0 || l[0] == nil { @@ -2762,6 +2823,13 @@ func expandBigQueryJobConfigurationCopyDestinationEncryptionConfiguration(v inte transformed["kmsKeyName"] = transformedKmsKeyName } + transformedKmsKeyVersion, err := expandBigQueryJobConfigurationCopyDestinationEncryptionConfigurationKmsKeyVersion(original["kms_key_version"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedKmsKeyVersion); val.IsValid() && !isEmptyValue(val) { + transformed["kmsKeyVersion"] = transformedKmsKeyVersion + } + return transformed, nil } @@ -2769,6 +2837,10 @@ func expandBigQueryJobConfigurationCopyDestinationEncryptionConfigurationKmsKeyN return v, nil } +func expandBigQueryJobConfigurationCopyDestinationEncryptionConfigurationKmsKeyVersion(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + func expandBigQueryJobConfigurationExtract(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { l := v.([]interface{}) if len(l) == 0 || l[0] == nil { diff --git a/google-beta/resource_bigquery_table.go b/google-beta/resource_bigquery_table.go index 18c66c3efa..4a8bd4e662 100644 --- a/google-beta/resource_bigquery_table.go +++ b/google-beta/resource_bigquery_table.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log" + "regexp" "sort" "strconv" "strings" @@ -820,6 +821,11 @@ func resourceBigQueryTable() *schema.Resource { Required: true, Description: `The self link or full name of a key which should be used to encrypt this table. Note that the default bigquery service account will need to have encrypt/decrypt permissions on this key - you may want to see the google_bigquery_default_service_account datasource and the google_kms_crypto_key_iam_binding resource.`, }, + "kms_key_version": { + Type: schema.TypeString, + Computed: true, + Description: `The self link or full name of the kms key version used to encrypt this table.`, + }, }, }, }, @@ -1543,7 +1549,20 @@ func expandRangePartitioning(configured interface{}) (*bigquery.RangePartitionin } func flattenEncryptionConfiguration(ec *bigquery.EncryptionConfiguration) []map[string]interface{} { - return []map[string]interface{}{{"kms_key_name": ec.KmsKeyName}} + re := regexp.MustCompile(`(projects/.*/locations/.*/keyRings/.*/cryptoKeys/.*)/cryptoKeyVersions/.*`) + paths := re.FindStringSubmatch(ec.KmsKeyName) + + if len(paths) > 0 { + return []map[string]interface{}{ + { + "kms_key_name": paths[0], + "kms_key_version": ec.KmsKeyName, + }, + } + } + + // The key name was returned, no need to set the version + return []map[string]interface{}{{"kms_key_name": ec.KmsKeyName, "kms_key_version": ""}} } func flattenTimePartitioning(tp *bigquery.TimePartitioning) []map[string]interface{} { diff --git a/google-beta/resource_dataproc_cluster_test.go b/google-beta/resource_dataproc_cluster_test.go index 8f3596cc80..79b887900b 100644 --- a/google-beta/resource_dataproc_cluster_test.go +++ b/google-beta/resource_dataproc_cluster_test.go @@ -13,8 +13,9 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" - dataproc "google.golang.org/api/dataproc/v1beta2" "google.golang.org/api/googleapi" + + dataproc "google.golang.org/api/dataproc/v1beta2" ) func TestDataprocExtractInitTimeout(t *testing.T) { diff --git a/google-beta/resource_gke_hub_feature_membership_test.go b/google-beta/resource_gke_hub_feature_membership_test.go index 32fa84c196..9e1f2ea64d 100644 --- a/google-beta/resource_gke_hub_feature_membership_test.go +++ b/google-beta/resource_gke_hub_feature_membership_test.go @@ -5,7 +5,7 @@ import ( "fmt" "testing" - dcl "github.com/GoogleCloudPlatform/declarative-resource-client-library/dcl" + "github.com/GoogleCloudPlatform/declarative-resource-client-library/dcl" gkehub "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/gkehub/beta" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" diff --git a/website/docs/r/bigquery_job.html.markdown b/website/docs/r/bigquery_job.html.markdown index d2bcf10c67..f0e700e125 100644 --- a/website/docs/r/bigquery_job.html.markdown +++ b/website/docs/r/bigquery_job.html.markdown @@ -539,6 +539,9 @@ The `destination_encryption_configuration` block supports: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. +* `kms_key_version` - + Describes the Cloud KMS encryption key version used to protect destination BigQuery table. + The `script_options` block supports: * `statement_timeout_ms` - @@ -731,6 +734,9 @@ The `destination_encryption_configuration` block supports: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. +* `kms_key_version` - + Describes the Cloud KMS encryption key version used to protect destination BigQuery table. + The `copy` block supports: * `source_tables` - @@ -806,6 +812,9 @@ The `destination_encryption_configuration` block supports: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. +* `kms_key_version` - + Describes the Cloud KMS encryption key version used to protect destination BigQuery table. + The `extract` block supports: * `destination_uris` - diff --git a/website/docs/r/bigquery_table.html.markdown b/website/docs/r/bigquery_table.html.markdown index 5c563cc972..25f4120c25 100644 --- a/website/docs/r/bigquery_table.html.markdown +++ b/website/docs/r/bigquery_table.html.markdown @@ -322,6 +322,8 @@ exported: * `etag` - A hash of the resource. +* `kms_key_version` - The self link or full name of the kms key version used to encrypt this table. + * `last_modified_time` - The time when this table was last modified, in milliseconds since the epoch. * `location` - The geographic location where the table resides. This value is inherited from the dataset.