From 1c356893005b82c2536c4d23e944a6b6c3c2729f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksandra=20Jarmoli=C5=84ska?= Date: Wed, 17 Feb 2021 21:31:39 +0100 Subject: [PATCH] Add EncryptionConfig to Cloud Composer's EnvironmentConfig (#4310) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adds a new section (EncryptionConfig) to Cloud Composer's EnvironmentConfig used for environment creation * Add missing part of the config. * Fixes to how the kms key is passed * Fix testing call * Fix nesting of ForceNew in encryption_config Co-authored-by: Aleksandra Jarmolińska --- .../resource_composer_environment.go.erb | 54 ++++++++++ .../resource_composer_environment_test.go.erb | 101 ++++++++++++++++++ .../docs/r/composer_environment.html.markdown | 11 ++ 3 files changed, 166 insertions(+) diff --git a/mmv1/third_party/terraform/resources/resource_composer_environment.go.erb b/mmv1/third_party/terraform/resources/resource_composer_environment.go.erb index facaaa29f746..71ce132a5e3a 100644 --- a/mmv1/third_party/terraform/resources/resource_composer_environment.go.erb +++ b/mmv1/third_party/terraform/resources/resource_composer_environment.go.erb @@ -56,6 +56,7 @@ var ( <% unless version == "ga" -%> "config.0.database_config", "config.0.web_server_config", + "config.0.encryption_config", <% end -%> } @@ -433,6 +434,24 @@ func resourceComposerEnvironment() *schema.Resource { }, }, }, + "encryption_config": { + Type: schema.TypeList, + Optional: true, + Computed: true, + AtLeastOneOf: composerConfigKeys, + MaxItems: 1, + Description: `The encryption options for the Composer environment and its dependencies.`, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "kms_key_name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: `Optional. Customer-managed Encryption Key available through Google's Key Management Service. Cannot be updated.`, + }, + }, + }, + }, <% end -%> "airflow_uri": { Type: schema.TypeString, @@ -826,6 +845,7 @@ func flattenComposerEnvironmentConfig(envCfg *composer.EnvironmentConfig) interf <% unless version == "ga" -%> transformed["database_config"] = flattenComposerEnvironmentConfigDatabaseConfig(envCfg.DatabaseConfig) transformed["web_server_config"] = flattenComposerEnvironmentConfigWebServerConfig(envCfg.WebServerConfig) + transformed["encryption_config"] = flattenComposerEnvironmentConfigEncryptionConfig(envCfg.EncryptionConfig) <% end -%> return []interface{}{transformed} @@ -876,6 +896,17 @@ func flattenComposerEnvironmentConfigWebServerConfig(webServerCfg *composer.WebS return []interface{}{transformed} } + +func flattenComposerEnvironmentConfigEncryptionConfig(encryptionCfg *composer.EncryptionConfig) interface{} { + if encryptionCfg == nil { + return nil + } + + transformed := make(map[string]interface{}) + transformed["kms_key_name"] = encryptionCfg.KmsKeyName + + return []interface{}{transformed} +} <% end -%> func flattenComposerEnvironmentConfigPrivateEnvironmentConfig(envCfg *composer.PrivateEnvironmentConfig) interface{} { @@ -1006,6 +1037,12 @@ func expandComposerEnvironmentConfig(v interface{}, d *schema.ResourceData, conf } transformed.WebServerConfig = transformedWebServerConfig + transformedEncryptionConfig, err := expandComposerEnvironmentConfigEncryptionConfig(original["encryption_config"], d, config) + if err != nil { + return nil, err + } + transformed.EncryptionConfig = transformedEncryptionConfig + <% end -%> return transformed, nil } @@ -1080,6 +1117,23 @@ func expandComposerEnvironmentConfigWebServerConfig(v interface{}, d *schema.Res <% end -%> +<% unless version == "ga" -%> +func expandComposerEnvironmentConfigEncryptionConfig(v interface{}, d *schema.ResourceData, config *Config) (*composer.EncryptionConfig, error) { + l := v.([]interface{}) + if len(l) == 0 || l[0] == nil { + return nil, nil + } + raw := l[0] + original := raw.(map[string]interface{}) + + transformed := &composer.EncryptionConfig{} + transformed.KmsKeyName = original["kms_key_name"].(string) + + return transformed, nil +} + +<% end -%> + func expandComposerEnvironmentConfigPrivateEnvironmentConfig(v interface{}, d *schema.ResourceData, config *Config) (*composer.PrivateEnvironmentConfig, error) { l := v.([]interface{}) if len(l) == 0 { diff --git a/mmv1/third_party/terraform/tests/resource_composer_environment_test.go.erb b/mmv1/third_party/terraform/tests/resource_composer_environment_test.go.erb index 63330eb3896c..2335472668ab 100644 --- a/mmv1/third_party/terraform/tests/resource_composer_environment_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_composer_environment_test.go.erb @@ -294,6 +294,41 @@ func TestAccComposerEnvironment_withWebServerConfig(t *testing.T) { }, }) } + +func TestAccComposerEnvironment_withEncryptionConfig(t *testing.T) { + t.Parallel() + + kms := BootstrapKMSKeyInLocation(t, "us-central1") + pid := getTestProjectFromEnv() + envName := fmt.Sprintf("%s-%d", testComposerEnvironmentPrefix, randInt(t)) + network := fmt.Sprintf("%s-%d", testComposerNetworkPrefix, randInt(t)) + subnetwork := network + "-1" + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccComposerEnvironmentDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccComposerEnvironment_encryptionCfg(pid, envName, kms.CryptoKey.Name, network, subnetwork), + }, + { + ResourceName: "google_composer_environment.test", + ImportState: true, + ImportStateVerify: true, + }, + // This is a terrible clean-up step in order to get destroy to succeed, + // due to dangling firewall rules left by the Composer Environment blocking network deletion. + // TODO(dzarmola): Remove this check if firewall rules bug gets fixed by Composer. + { + PlanOnly: true, + ExpectNonEmptyPlan: false, + Config: testAccComposerEnvironment_encryptionCfg(pid, envName, kms.CryptoKey.Name, network, subnetwork), + Check: testAccCheckClearComposerEnvironmentFirewalls(t, network), + }, + }, + }) +} <% end -%> // Checks behavior of node config, including dependencies on Compute resources. func TestAccComposerEnvironment_withNodeConfig(t *testing.T) { @@ -735,6 +770,72 @@ resource "google_compute_subnetwork" "test" { `, name, network, subnetwork) } +func testAccComposerEnvironment_encryptionCfg(pid, name, kmsKey, network, subnetwork string) string { + return fmt.Sprintf(` +data "google_project" "project" { + project_id = "%s" +} +resource "google_project_iam_member" "kms-project-binding1" { + project = data.google_project.project.project_id + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:service-${data.google_project.project.number}@cloudcomposer-accounts.iam.gserviceaccount.com" +} +resource "google_project_iam_member" "kms-project-binding2" { + project = data.google_project.project.project_id + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:service-${data.google_project.project.number}@compute-system.iam.gserviceaccount.com" +} +resource "google_project_iam_member" "kms-project-binding3" { + project = data.google_project.project.project_id + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:service-${data.google_project.project.number}@container-engine-robot.iam.gserviceaccount.com" +} +resource "google_project_iam_member" "kms-project-binding4" { + project = data.google_project.project.project_id + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-artifactregistry.iam.gserviceaccount.com" +} +resource "google_project_iam_member" "kms-project-binding5" { + project = data.google_project.project.project_id + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} +resource "google_kms_crypto_key_iam_member" "iam" { + crypto_key_id = "%s" + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:service-${data.google_project.project.number}@gs-project-accounts.iam.gserviceaccount.com" +} +resource "google_composer_environment" "test" { + depends_on = [google_project_iam_member.kms-project-binding1, google_project_iam_member.kms-project-binding2, + google_project_iam_member.kms-project-binding3, google_project_iam_member.kms-project-binding4, + google_project_iam_member.kms-project-binding5, google_kms_crypto_key_iam_member.iam] + name = "%s" + region = "us-central1" + config { + node_config { + network = google_compute_network.test.self_link + subnetwork = google_compute_subnetwork.test.self_link + zone = "us-central1-a" + } + encryption_config { + kms_key_name = "%s" + } + } +} +// use a separate network to avoid conflicts with other tests running in parallel +// that use the default network/subnet +resource "google_compute_network" "test" { + name = "%s" + auto_create_subnetworks = false +} +resource "google_compute_subnetwork" "test" { + name = "%s" + ip_cidr_range = "10.2.0.0/16" + region = "us-central1" + network = google_compute_network.test.self_link +} +`, pid, kmsKey, name, kmsKey, network, subnetwork) +} <% end -%> func testAccComposerEnvironment_update(name, network, subnetwork string) string { return fmt.Sprintf(` diff --git a/mmv1/third_party/terraform/website/docs/r/composer_environment.html.markdown b/mmv1/third_party/terraform/website/docs/r/composer_environment.html.markdown index ae159d2c31c9..b83cb6ca4ba9 100644 --- a/mmv1/third_party/terraform/website/docs/r/composer_environment.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/composer_environment.html.markdown @@ -179,6 +179,10 @@ The `config` block supports: (Optional, [Beta](https://terraform.io/docs/providers/google/guides/provider_versions.html)) The configuration settings for the Airflow web server App Engine instance. +* `encryption_config` - + (Optional, [Beta](https://terraform.io/docs/providers/google/guides/provider_versions.html)) + The encryption options for the Cloud Composer environment and its dependencies. + The `node_config` block supports: * `zone` - @@ -390,6 +394,13 @@ The `web_server_config` block supports: Value custom is returned only in response, if Airflow web server parameters were manually changed to a non-standard values. +The `encryption_config` block supports: + +* `kms_key_name` - + (Required) + Customer-managed Encryption Key available through Google's Key Management Service. It must + be the fully qualified resource name, + i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key. Cannot be updated. ## Attributes Reference