Skip to content

Commit

Permalink
metastore - add support for scheduled backups (#10213) (#7140)
Browse files Browse the repository at this point in the history
* add support for dataproc metastore scheduled backups

* CR comments

* CR comments

* CR comments

* CR comments

* CR comments

* Update mmv1/third_party/terraform/services/dataprocmetastore/resource_dataproc_metastore_service_test.go.erb



* Update mmv1/third_party/terraform/services/dataprocmetastore/resource_dataproc_metastore_service_test.go.erb



* CR comments

---------



[upstream:6f7d670e0f2e9da1ab3e562611c2a3ed25284ab3]

Signed-off-by: Modular Magician <magic-modules@google.com>
  • Loading branch information
modular-magician authored Mar 25, 2024
1 parent 4c98bad commit 836777f
Showing 5 changed files with 334 additions and 0 deletions.
3 changes: 3 additions & 0 deletions .changelog/10213.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:enhancement
metastore: added `scheduled_backup` field to `google_dataproc_metastore_service` resource
```
Original file line number Diff line number Diff line change
@@ -338,6 +338,38 @@ There must be at least one IP address available in the subnet's primary range. T
},
},
},
"scheduled_backup": {
Type: schema.TypeList,
Optional: true,
Description: `The configuration of scheduled backup for the metastore service.`,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"backup_location": {
Type: schema.TypeString,
Required: true,
Description: `A Cloud Storage URI of a folder, in the format gs://<bucket_name>/<path_inside_bucket>. A sub-folder <backup_folder> containing backup files will be stored below it.`,
},
"cron_schedule": {
Type: schema.TypeString,
Optional: true,
Description: `The scheduled interval in Cron format, see https://en.wikipedia.org/wiki/Cron The default is empty: scheduled backup is not enabled. Must be specified to enable scheduled backups.`,
},
"enabled": {
Type: schema.TypeBool,
Computed: true,
Optional: true,
Description: `Defines whether the scheduled backup is enabled. The default value is false.`,
},
"time_zone": {
Type: schema.TypeString,
Computed: true,
Optional: true,
Description: `Specifies the time zone to be used when interpreting cronSchedule. Must be a time zone name from the time zone database (https://en.wikipedia.org/wiki/List_of_tz_database_time_zones), e.g. America/Los_Angeles or Africa/Abidjan. If left unspecified, the default is UTC.`,
},
},
},
},
"telemetry_config": {
Type: schema.TypeList,
Computed: true,
@@ -450,6 +482,12 @@ func resourceDataprocMetastoreServiceCreate(d *schema.ResourceData, meta interfa
} else if v, ok := d.GetOkExists("scaling_config"); !tpgresource.IsEmptyValue(reflect.ValueOf(scalingConfigProp)) && (ok || !reflect.DeepEqual(v, scalingConfigProp)) {
obj["scalingConfig"] = scalingConfigProp
}
scheduledBackupProp, err := expandDataprocMetastoreServiceScheduledBackup(d.Get("scheduled_backup"), d, config)
if err != nil {
return err
} else if v, ok := d.GetOkExists("scheduled_backup"); !tpgresource.IsEmptyValue(reflect.ValueOf(scheduledBackupProp)) && (ok || !reflect.DeepEqual(v, scheduledBackupProp)) {
obj["scheduledBackup"] = scheduledBackupProp
}
maintenanceWindowProp, err := expandDataprocMetastoreServiceMaintenanceWindow(d.Get("maintenance_window"), d, config)
if err != nil {
return err
@@ -629,6 +667,9 @@ func resourceDataprocMetastoreServiceRead(d *schema.ResourceData, meta interface
if err := d.Set("scaling_config", flattenDataprocMetastoreServiceScalingConfig(res["scalingConfig"], d, config)); err != nil {
return fmt.Errorf("Error reading Service: %s", err)
}
if err := d.Set("scheduled_backup", flattenDataprocMetastoreServiceScheduledBackup(res["scheduledBackup"], d, config)); err != nil {
return fmt.Errorf("Error reading Service: %s", err)
}
if err := d.Set("maintenance_window", flattenDataprocMetastoreServiceMaintenanceWindow(res["maintenanceWindow"], d, config)); err != nil {
return fmt.Errorf("Error reading Service: %s", err)
}
@@ -700,6 +741,12 @@ func resourceDataprocMetastoreServiceUpdate(d *schema.ResourceData, meta interfa
} else if v, ok := d.GetOkExists("scaling_config"); !tpgresource.IsEmptyValue(reflect.ValueOf(v)) && (ok || !reflect.DeepEqual(v, scalingConfigProp)) {
obj["scalingConfig"] = scalingConfigProp
}
scheduledBackupProp, err := expandDataprocMetastoreServiceScheduledBackup(d.Get("scheduled_backup"), d, config)
if err != nil {
return err
} else if v, ok := d.GetOkExists("scheduled_backup"); !tpgresource.IsEmptyValue(reflect.ValueOf(v)) && (ok || !reflect.DeepEqual(v, scheduledBackupProp)) {
obj["scheduledBackup"] = scheduledBackupProp
}
maintenanceWindowProp, err := expandDataprocMetastoreServiceMaintenanceWindow(d.Get("maintenance_window"), d, config)
if err != nil {
return err
@@ -757,6 +804,10 @@ func resourceDataprocMetastoreServiceUpdate(d *schema.ResourceData, meta interfa
updateMask = append(updateMask, "scalingConfig")
}

if d.HasChange("scheduled_backup") {
updateMask = append(updateMask, "scheduledBackup")
}

if d.HasChange("maintenance_window") {
updateMask = append(updateMask, "maintenanceWindow")
}
@@ -978,6 +1029,41 @@ func flattenDataprocMetastoreServiceScalingConfigScalingFactor(v interface{}, d
return v
}

func flattenDataprocMetastoreServiceScheduledBackup(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
if v == nil {
return nil
}
original := v.(map[string]interface{})
if len(original) == 0 {
return nil
}
transformed := make(map[string]interface{})
transformed["enabled"] =
flattenDataprocMetastoreServiceScheduledBackupEnabled(original["enabled"], d, config)
transformed["cron_schedule"] =
flattenDataprocMetastoreServiceScheduledBackupCronSchedule(original["cronSchedule"], d, config)
transformed["time_zone"] =
flattenDataprocMetastoreServiceScheduledBackupTimeZone(original["timeZone"], d, config)
transformed["backup_location"] =
flattenDataprocMetastoreServiceScheduledBackupBackupLocation(original["backupLocation"], d, config)
return []interface{}{transformed}
}
func flattenDataprocMetastoreServiceScheduledBackupEnabled(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
return v
}

func flattenDataprocMetastoreServiceScheduledBackupCronSchedule(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
return v
}

func flattenDataprocMetastoreServiceScheduledBackupTimeZone(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
return v
}

func flattenDataprocMetastoreServiceScheduledBackupBackupLocation(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
return v
}

func flattenDataprocMetastoreServiceMaintenanceWindow(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
if v == nil {
return nil
@@ -1300,6 +1386,62 @@ func expandDataprocMetastoreServiceScalingConfigScalingFactor(v interface{}, d t
return v, nil
}

func expandDataprocMetastoreServiceScheduledBackup(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
l := v.([]interface{})
if len(l) == 0 || l[0] == nil {
return nil, nil
}
raw := l[0]
original := raw.(map[string]interface{})
transformed := make(map[string]interface{})

transformedEnabled, err := expandDataprocMetastoreServiceScheduledBackupEnabled(original["enabled"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedEnabled); val.IsValid() && !tpgresource.IsEmptyValue(val) {
transformed["enabled"] = transformedEnabled
}

transformedCronSchedule, err := expandDataprocMetastoreServiceScheduledBackupCronSchedule(original["cron_schedule"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedCronSchedule); val.IsValid() && !tpgresource.IsEmptyValue(val) {
transformed["cronSchedule"] = transformedCronSchedule
}

transformedTimeZone, err := expandDataprocMetastoreServiceScheduledBackupTimeZone(original["time_zone"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedTimeZone); val.IsValid() && !tpgresource.IsEmptyValue(val) {
transformed["timeZone"] = transformedTimeZone
}

transformedBackupLocation, err := expandDataprocMetastoreServiceScheduledBackupBackupLocation(original["backup_location"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedBackupLocation); val.IsValid() && !tpgresource.IsEmptyValue(val) {
transformed["backupLocation"] = transformedBackupLocation
}

return transformed, nil
}

func expandDataprocMetastoreServiceScheduledBackupEnabled(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
return v, nil
}

func expandDataprocMetastoreServiceScheduledBackupCronSchedule(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
return v, nil
}

func expandDataprocMetastoreServiceScheduledBackupTimeZone(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
return v, nil
}

func expandDataprocMetastoreServiceScheduledBackupBackupLocation(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
return v, nil
}

func expandDataprocMetastoreServiceMaintenanceWindow(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
l := v.([]interface{})
if len(l) == 0 || l[0] == nil {
Original file line number Diff line number Diff line change
@@ -428,6 +428,67 @@ resource "google_dataproc_metastore_service" "dpms2_scaling_factor_lt1" {
`, context)
}

func TestAccDataprocMetastoreService_dataprocMetastoreServiceScheduledBackupExample(t *testing.T) {
t.Parallel()

context := map[string]interface{}{
"random_suffix": acctest.RandString(t, 10),
}

acctest.VcrTest(t, resource.TestCase{
PreCheck: func() { acctest.AccTestPreCheck(t) },
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
CheckDestroy: testAccCheckDataprocMetastoreServiceDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccDataprocMetastoreService_dataprocMetastoreServiceScheduledBackupExample(context),
},
{
ResourceName: "google_dataproc_metastore_service.backup",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"service_id", "location", "labels", "terraform_labels"},
},
},
})
}

func testAccDataprocMetastoreService_dataprocMetastoreServiceScheduledBackupExample(context map[string]interface{}) string {
return acctest.Nprintf(`
resource "google_dataproc_metastore_service" "backup" {
service_id = "backup%{random_suffix}"
location = "us-central1"
port = 9080
tier = "DEVELOPER"
maintenance_window {
hour_of_day = 2
day_of_week = "SUNDAY"
}
hive_metastore_config {
version = "2.3.6"
}
scheduled_backup {
enabled = true
cron_schedule = "0 0 * * *"
time_zone = "UTC"
backup_location = "gs://${google_storage_bucket.bucket.name}"
}
labels = {
env = "test"
}
}
resource "google_storage_bucket" "bucket" {
name = "backup%{random_suffix}"
location = "us-central1"
}
`, context)
}

func testAccCheckDataprocMetastoreServiceDestroyProducer(t *testing.T) func(s *terraform.State) error {
return func(s *terraform.State) error {
for name, rs := range s.RootModule().Resources {
Original file line number Diff line number Diff line change
@@ -54,6 +54,34 @@ resource "google_dataproc_metastore_service" "my_metastore" {
`, name, tier)
}

func TestAccDataprocMetastoreService_dataprocMetastoreServiceScheduledBackupExampleUpdate(t *testing.T) {
t.Parallel()

context := map[string]interface{}{
"random_suffix": acctest.RandString(t, 10),
}

acctest.VcrTest(t, resource.TestCase{
PreCheck: func() { acctest.AccTestPreCheck(t) },
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
CheckDestroy: testAccCheckDataprocMetastoreServiceDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccDataprocMetastoreService_dataprocMetastoreServiceScheduledBackupExample(context),
},
{
ResourceName: "google_dataproc_metastore_service.backup",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"service_id", "location", "labels", "terraform_labels"},
},
{
Config: testAccDataprocMetastoreService_dataprocMetastoreServiceScheduledBackupExampleUpdate(context),
},
},
})
}

func TestAccDataprocMetastoreService_PrivateServiceConnect(t *testing.T) {
t.Skip("Skipping due to https://github.com/hashicorp/terraform-provider-google/issues/13710")
t.Parallel()
@@ -105,3 +133,39 @@ resource "google_dataproc_metastore_service" "default" {
}
`, context)
}

func testAccDataprocMetastoreService_dataprocMetastoreServiceScheduledBackupExampleUpdate(context map[string]interface{}) string {
return acctest.Nprintf(`
resource "google_dataproc_metastore_service" "backup" {
service_id = "tf-test-backup%{random_suffix}"
location = "us-central1"
port = 9080
tier = "DEVELOPER"
maintenance_window {
hour_of_day = 2
day_of_week = "SUNDAY"
}
hive_metastore_config {
version = "2.3.6"
}
scheduled_backup {
enabled = true
cron_schedule = "0 0 * * 0"
time_zone = "America/Los_Angeles"
backup_location = "gs://${google_storage_bucket.bucket.name}"
}
labels = {
env = "test"
}
}
resource "google_storage_bucket" "bucket" {
name = "tf-test-backup%{random_suffix}"
location = "us-central1"
}
`, context)
}
Loading

0 comments on commit 836777f

Please sign in to comment.