From bf4a252cf1748b5c27f50a6a62232f10de532944 Mon Sep 17 00:00:00 2001 From: Jeremy Mathevet Date: Wed, 28 Oct 2020 16:33:43 +0800 Subject: [PATCH 1/8] Add BigTable source format in BigQuery --- products/bigquery/api.yaml | 3 +- .../resources/resource_bigquery_table.go | 4 +-- .../tests/resource_bigquery_table_test.go | 31 +++++++++++++++++++ .../docs/r/bigquery_table.html.markdown | 2 +- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/products/bigquery/api.yaml b/products/bigquery/api.yaml index 44b8d7ddf864..891b0fd9b876 100644 --- a/products/bigquery/api.yaml +++ b/products/bigquery/api.yaml @@ -722,7 +722,8 @@ objects: description: | The format of the data files. For CSV files, specify "CSV". For datastore backups, specify "DATASTORE_BACKUP". For newline-delimited JSON, specify "NEWLINE_DELIMITED_JSON". For Avro, specify "AVRO". For parquet, specify "PARQUET". - For orc, specify "ORC". The default value is CSV. + For orc, specify "ORC". [Beta] For Bigtable, specify "BIGTABLE". + The default value is CSV. default_value: 'CSV' - !ruby/object:Api::Type::Boolean name: 'allowJaggedRows' diff --git a/third_party/terraform/resources/resource_bigquery_table.go b/third_party/terraform/resources/resource_bigquery_table.go index 4ddd8c954b45..7e025737ea76 100644 --- a/third_party/terraform/resources/resource_bigquery_table.go +++ b/third_party/terraform/resources/resource_bigquery_table.go @@ -153,9 +153,9 @@ func resourceBigQueryTable() *schema.Resource { "source_format": { Type: schema.TypeString, Required: true, - Description: `The data format. Supported values are: "CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "PARQUET", and "DATSTORE_BACKUP". To use "GOOGLE_SHEETS" the scopes must include "googleapis.com/auth/drive.readonly".`, + Description: `The data format. Supported values are: "CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "PARQUET", "DATSTORE_BACKUP", and "BIGTABLE". To use "GOOGLE_SHEETS" the scopes must include "googleapis.com/auth/drive.readonly".`, ValidateFunc: validation.StringInSlice([]string{ - "CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "DATSTORE_BACKUP", "PARQUET", + "CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "DATSTORE_BACKUP", "PARQUET", "BIGTABLE", }, false), }, // SourceURIs [Required] The fully-qualified URIs that point to your data in Google Cloud. diff --git a/third_party/terraform/tests/resource_bigquery_table_test.go b/third_party/terraform/tests/resource_bigquery_table_test.go index 3da6eba393be..72d1d3cae587 100644 --- a/third_party/terraform/tests/resource_bigquery_table_test.go +++ b/third_party/terraform/tests/resource_bigquery_table_test.go @@ -1048,6 +1048,37 @@ func testAccBigQueryTableFromSheet(context map[string]interface{}) string { `, context) } +func testAccBigQueryTableFromBigTable(context map[string]interface{}) string { + return Nprintf(` + resource "google_bigquery_table" "table" { + dataset_id = google_bigquery_dataset.dataset.dataset_id + table_id = "tf_test_bigtable_%{random_suffix}" + + external_data_configuration { + autodetect = true + source_format = "BIGTABLE" + ignore_unknown_values = true + + source_uris = [ + "https://https://googleapis.com/bigtable/projects/project_id/instances/instance_id/tables/table_name", + ] + } + } + + resource "google_bigquery_dataset" "dataset" { + dataset_id = "tf_test_ds_%{random_suffix}" + friendly_name = "test" + description = "This is a test description" + location = "EU" + default_table_expiration_ms = 3600000 + + labels = { + env = "default" + } + } +`, context) +} + var TEST_CSV = `lifelock,LifeLock,,web,Tempe,AZ,1-May-07,6850000,USD,b lifelock,LifeLock,,web,Tempe,AZ,1-Oct-06,6000000,USD,a lifelock,LifeLock,,web,Tempe,AZ,1-Jan-08,25000000,USD,c diff --git a/third_party/terraform/website/docs/r/bigquery_table.html.markdown b/third_party/terraform/website/docs/r/bigquery_table.html.markdown index 3e6dae8a3aae..62ea40e60cc1 100644 --- a/third_party/terraform/website/docs/r/bigquery_table.html.markdown +++ b/third_party/terraform/website/docs/r/bigquery_table.html.markdown @@ -180,7 +180,7 @@ The `external_data_configuration` block supports: * `source_format` (Required) - The data format. Supported values are: "CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "PARQUET", - and "DATSTORE_BACKUP". To use "GOOGLE_SHEETS" + "DATSTORE_BACKUP", and "BIGTABLE". To use "GOOGLE_SHEETS" the `scopes` must include "https://www.googleapis.com/auth/drive.readonly". From 828eb0a8314971fb9cd79a79be95745e4d57f10b Mon Sep 17 00:00:00 2001 From: Jeremy Mathevet Date: Wed, 28 Oct 2020 18:08:02 +0800 Subject: [PATCH 2/8] Add tests --- .../tests/resource_bigquery_table_test.go | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/third_party/terraform/tests/resource_bigquery_table_test.go b/third_party/terraform/tests/resource_bigquery_table_test.go index 72d1d3cae587..c86daa408330 100644 --- a/third_party/terraform/tests/resource_bigquery_table_test.go +++ b/third_party/terraform/tests/resource_bigquery_table_test.go @@ -416,6 +416,30 @@ func TestAccBigQueryDataTable_sheet(t *testing.T) { }) } +func TestAccBigQueryDataTable_bigtable(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": randString(t, 10), + } + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckBigQueryTableDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccBigQueryTableFromBigtable(context), + }, + { + ResourceName: "google_bigquery_table.table", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + func testAccCheckBigQueryExtData(t *testing.T, expectedQuoteChar string) resource.TestCheckFunc { return func(s *terraform.State) error { for _, rs := range s.RootModule().Resources { From f50376211dea6d1bed491ecad42ed0cdb75e0880 Mon Sep 17 00:00:00 2001 From: Jeremy Mathevet Date: Thu, 29 Oct 2020 09:35:57 +0800 Subject: [PATCH 3/8] Fix test config typos --- third_party/terraform/tests/resource_bigquery_table_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/terraform/tests/resource_bigquery_table_test.go b/third_party/terraform/tests/resource_bigquery_table_test.go index c86daa408330..f1ee1678a299 100644 --- a/third_party/terraform/tests/resource_bigquery_table_test.go +++ b/third_party/terraform/tests/resource_bigquery_table_test.go @@ -1072,7 +1072,7 @@ func testAccBigQueryTableFromSheet(context map[string]interface{}) string { `, context) } -func testAccBigQueryTableFromBigTable(context map[string]interface{}) string { +func testAccBigQueryTableFromBigtable(context map[string]interface{}) string { return Nprintf(` resource "google_bigquery_table" "table" { dataset_id = google_bigquery_dataset.dataset.dataset_id @@ -1084,7 +1084,7 @@ func testAccBigQueryTableFromBigTable(context map[string]interface{}) string { ignore_unknown_values = true source_uris = [ - "https://https://googleapis.com/bigtable/projects/project_id/instances/instance_id/tables/table_name", + "https://googleapis.com/bigtable/projects/project_id/instances/instance_id/tables/table_name", ] } } From 0fdd96bc3f0f9dc7b80a0b26583bb165cb51f503 Mon Sep 17 00:00:00 2001 From: Jeremy Mathevet Date: Mon, 2 Nov 2020 15:22:46 +0800 Subject: [PATCH 4/8] Add bigtable external source in the bigquery tests --- .../tests/resource_bigquery_table_test.go | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/third_party/terraform/tests/resource_bigquery_table_test.go b/third_party/terraform/tests/resource_bigquery_table_test.go index f1ee1678a299..3acc85ce8b9b 100644 --- a/third_party/terraform/tests/resource_bigquery_table_test.go +++ b/third_party/terraform/tests/resource_bigquery_table_test.go @@ -421,6 +421,7 @@ func TestAccBigQueryDataTable_bigtable(t *testing.T) { context := map[string]interface{}{ "random_suffix": randString(t, 10), + "project": getTestProjectFromEnv(), } vcrTest(t, resource.TestCase{ @@ -1074,6 +1075,31 @@ func testAccBigQueryTableFromSheet(context map[string]interface{}) string { func testAccBigQueryTableFromBigtable(context map[string]interface{}) string { return Nprintf(` + resource "google_bigtable_instance" "instance" { + name = "tf_test_bigtable_instance_%{random_suffix}" + + cluster { + cluster_id = "%{random_suffix}" + zone = "us-central1-b" + } + + instance_type = "DEVELOPMENT" + deletion_protection = false + } + + resource "google_bigtable_table" "table" { + name = "%{random_suffix}" + instance_name = google_bigtable_instance.instance.name + + column_family { + family = cf-"%{random_suffix}-first" + } + + column_family { + family = "cf-%{random_suffix}-second" + } + } + resource "google_bigquery_table" "table" { dataset_id = google_bigquery_dataset.dataset.dataset_id table_id = "tf_test_bigtable_%{random_suffix}" @@ -1084,7 +1110,7 @@ func testAccBigQueryTableFromBigtable(context map[string]interface{}) string { ignore_unknown_values = true source_uris = [ - "https://googleapis.com/bigtable/projects/project_id/instances/instance_id/tables/table_name", + "https://googleapis.com/bigtable/${google_bigtable_table.table.id}", ] } } From d9a0a51b39555f4c4bbc9ae4ce024952004a93e4 Mon Sep 17 00:00:00 2001 From: Jeremy Mathevet Date: Fri, 13 Nov 2020 15:23:02 +0800 Subject: [PATCH 5/8] typo --- third_party/terraform/tests/resource_bigquery_table_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/terraform/tests/resource_bigquery_table_test.go b/third_party/terraform/tests/resource_bigquery_table_test.go index 3acc85ce8b9b..cabaf20d9ae2 100644 --- a/third_party/terraform/tests/resource_bigquery_table_test.go +++ b/third_party/terraform/tests/resource_bigquery_table_test.go @@ -1092,7 +1092,7 @@ func testAccBigQueryTableFromBigtable(context map[string]interface{}) string { instance_name = google_bigtable_instance.instance.name column_family { - family = cf-"%{random_suffix}-first" + family = "cf-%{random_suffix}-first" } column_family { From 05336e518903804e2d2d7a6191cd3b0db84d13ca Mon Sep 17 00:00:00 2001 From: Jeremy Mathevet Date: Tue, 24 Nov 2020 19:14:06 +0800 Subject: [PATCH 6/8] Shrink resource name --- third_party/terraform/tests/resource_bigquery_table_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/terraform/tests/resource_bigquery_table_test.go b/third_party/terraform/tests/resource_bigquery_table_test.go index cabaf20d9ae2..76ecf264a053 100644 --- a/third_party/terraform/tests/resource_bigquery_table_test.go +++ b/third_party/terraform/tests/resource_bigquery_table_test.go @@ -1076,7 +1076,7 @@ func testAccBigQueryTableFromSheet(context map[string]interface{}) string { func testAccBigQueryTableFromBigtable(context map[string]interface{}) string { return Nprintf(` resource "google_bigtable_instance" "instance" { - name = "tf_test_bigtable_instance_%{random_suffix}" + name = "tf_test_bigtable_inst_%{random_suffix}" cluster { cluster_id = "%{random_suffix}" From 552aa0e7be3faf4787168bc36f4a8d68ca65b306 Mon Sep 17 00:00:00 2001 From: Jeremy Mathevet Date: Wed, 25 Nov 2020 10:16:22 +0800 Subject: [PATCH 7/8] Fix master merge --- third_party/terraform/resources/resource_bigquery_table.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/third_party/terraform/resources/resource_bigquery_table.go b/third_party/terraform/resources/resource_bigquery_table.go index 31e265512241..9ecec2af6eee 100644 --- a/third_party/terraform/resources/resource_bigquery_table.go +++ b/third_party/terraform/resources/resource_bigquery_table.go @@ -153,13 +153,9 @@ func resourceBigQueryTable() *schema.Resource { "source_format": { Type: schema.TypeString, Required: true, - Description: `The data format. Supported values are: "CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "PARQUET", "DATSTORE_BACKUP", and "BIGTABLE". To use "GOOGLE_SHEETS" the scopes must include "googleapis.com/auth/drive.readonly".`, - ValidateFunc: validation.StringInSlice([]string{ - "CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "DATSTORE_BACKUP", "PARQUET", "BIGTABLE", Description: `The data format. Supported values are: "CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "PARQUET", "ORC" and "DATASTORE_BACKUP". To use "GOOGLE_SHEETS" the scopes must include "googleapis.com/auth/drive.readonly".`, ValidateFunc: validation.StringInSlice([]string{ "CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "DATASTORE_BACKUP", "PARQUET", "ORC", "BIGTABLE", - }, false), }, // SourceURIs [Required] The fully-qualified URIs that point to your data in Google Cloud. From 2973a68bd14e16e23a5eeed2a512c1d367c2ce47 Mon Sep 17 00:00:00 2001 From: Jeremy Mathevet Date: Thu, 15 Apr 2021 15:48:06 +0800 Subject: [PATCH 8/8] Ignore deletion_protection in TestAccBigQueryDataTable_bigtable --- .../terraform/tests/resource_bigquery_table_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_bigquery_table_test.go b/mmv1/third_party/terraform/tests/resource_bigquery_table_test.go index 9299852d1006..cc3c93520c48 100644 --- a/mmv1/third_party/terraform/tests/resource_bigquery_table_test.go +++ b/mmv1/third_party/terraform/tests/resource_bigquery_table_test.go @@ -429,7 +429,7 @@ func TestAccBigQueryDataTable_bigtable(t *testing.T) { t.Parallel() context := map[string]interface{}{ - "random_suffix": randString(t, 10), + "random_suffix": randString(t, 8), "project": getTestProjectFromEnv(), } @@ -442,9 +442,10 @@ func TestAccBigQueryDataTable_bigtable(t *testing.T) { Config: testAccBigQueryTableFromBigtable(context), }, { - ResourceName: "google_bigquery_table.table", - ImportState: true, - ImportStateVerify: true, + ResourceName: "google_bigquery_table.table", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"deletion_protection"}, }, }, }) @@ -1436,7 +1437,7 @@ func testAccBigQueryTableFromBigtable(context map[string]interface{}) string { resource "google_bigtable_instance" "instance" { name = "tf-test-bigtable-inst-%{random_suffix}" cluster { - cluster_id = "%{random_suffix}" + cluster_id = "tf-test-bigtable-%{random_suffix}" zone = "us-central1-b" } instance_type = "DEVELOPMENT" @@ -1453,6 +1454,7 @@ func testAccBigQueryTableFromBigtable(context map[string]interface{}) string { } } resource "google_bigquery_table" "table" { + deletion_protection = false dataset_id = google_bigquery_dataset.dataset.dataset_id table_id = "tf_test_bigtable_%{random_suffix}" external_data_configuration {