diff --git a/google/resource_bigquery_table.go b/google/resource_bigquery_table.go index 1e7874b9a51..39c8374cb99 100644 --- a/google/resource_bigquery_table.go +++ b/google/resource_bigquery_table.go @@ -1,3 +1,4 @@ +// package google import ( @@ -62,6 +63,184 @@ func resourceBigQueryTable() *schema.Resource { Computed: true, }, + // ExternalDataConfiguration [Optional] Describes the data format, + // location, and other properties of a table stored outside of BigQuery. + // By defining these properties, the data source can then be queried as + // if it were a standard BigQuery table. + "external_data_configuration": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + // Autodetect : [Required] If true, let BigQuery try to autodetect the + // schema and format of the table. + "autodetect": { + Type: schema.TypeBool, + Required: true, + }, + // BigtableOptions: [Optional] Additional options if sourceFormat is set to BIGTABLE. + "bigtable_options": { + Removed: "This field is in beta. Use it in the the google-beta provider instead. See https://terraform.io/docs/providers/google/provider_versions.html for more details.", + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + // ColumnFamilies: [Optional] List of column families to expose in the + // table schema along with their types. + "column_families": { + Type: schema.TypeString, + Optional: true, + Computed: true, + ValidateFunc: validation.ValidateJsonString, + StateFunc: func(v interface{}) string { + json, _ := structure.NormalizeJsonString(v) + return json + }, + }, + // IgnoreUnspecifiedColumnFamilies: [Optional] If field is true, then + // the column families that are not specified in columnFamilies list are + // not exposed in the table schema. + "ignore_unspecified_column_families": { + Type: schema.TypeBool, + Optional: true, + }, + // ReadRowkeyAsString: [Optional] If field is true, then the rowkey + // column families will be read and converted to string. + "read_rowkey_as_string": { + Type: schema.TypeBool, + Optional: true, + }, + }, + }, + }, + // Compression: [Optional] The compression type of the data source. + "compression": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringInSlice([]string{"NONE", "GZIP"}, false), + }, + // CsvOptions: [Optional] Additional properties to set if + // sourceFormat is set to CSV. + "csv_options": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + // AllowJaggedRows: [Optional] Indicates if BigQuery should + // accept rows that are missing trailing optional columns. + "allow_jagged_rows": { + Type: schema.TypeBool, + Optional: true, + }, + // AllowQuotedNewlines: [Optional] Indicates if BigQuery + // should allow quoted data sections that contain newline + // characters in a CSV file. The default value is false. + "allow_quoted_newlines": { + Type: schema.TypeBool, + Optional: true, + }, + // Encoding: [Optional] The character encoding of the data. + // The supported values are UTF-8 or ISO-8859-1. + "encoding": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringInSlice([]string{"ISO-8859-1", "UTF-8"}, false), + }, + // FieldDelimiter: [Optional] The separator for fields in a CSV file. + "field_delimiter": { + Type: schema.TypeString, + Optional: true, + }, + // Quote: [Optional] The value that is used to quote data + // sections in a CSV file. + "quote": { + Type: schema.TypeString, + Optional: true, + }, + // SkipLeadingRows: [Optional] The number of rows at the top + // of a CSV file that BigQuery will skip when reading the data. + "skip_leading_rows": { + Type: schema.TypeInt, + Optional: true, + }, + }, + }, + }, + // GoogleSheetsOptions: [Optional] Additional options if sourceFormat is set to GOOGLE_SHEETS. + "google_sheets_options": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + // Range: [Optional] Range of a sheet to query from. Only used when non-empty. + // Typical format: !: + "range": { + Removed: "This field is in beta. Use it in the the google-beta provider instead. See https://terraform.io/docs/providers/google/provider_versions.html for more details.", + Type: schema.TypeString, + Optional: true, + }, + // SkipLeadingRows: [Optional] The number of rows at the top + // of the scheet that BigQuery will skip when reading the data. + "skip_leading_rows": { + Type: schema.TypeInt, + Optional: true, + }, + }, + }, + }, + + // IgnoreUnknownValues: [Optional] Indicates if BigQuery should + // allow extra values that are not represented in the table schema. + // If true, the extra values are ignored. If false, records with + // extra columns are treated as bad records, and if there are too + // many bad records, an invalid error is returned in the job result. + // The default value is false. + "ignore_unknown_values": { + Type: schema.TypeBool, + Optional: true, + }, + // MaxBadRecords: [Optional] The maximum number of bad records that + // BigQuery can ignore when reading data. + "max_bad_records": { + Type: schema.TypeInt, + Optional: true, + }, + // Schema: [Optional] The schema for the data. + // Schema is required for CSV and JSON formats + // schema is disallowed for Google Cloud Bigtable, + // Cloud Datastore backups, and Avro formats. + "schema": { + Type: schema.TypeString, + Optional: true, + Computed: true, + ValidateFunc: validation.ValidateJsonString, + StateFunc: func(v interface{}) string { + json, _ := structure.NormalizeJsonString(v) + return json + }, + }, + // SourceFormat [Required] The data format. + "source_format": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringInSlice([]string{ + "CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "DATSTORE_BACKUP", + }, false), + }, + // SourceURIs [Required] The fully-qualified URIs that point to your data in Google Cloud. + "source_uris": { + Type: schema.TypeList, + Required: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + }, + }, + }, + // FriendlyName: [Optional] A descriptive name for this table. "friendly_name": { Type: schema.TypeString, @@ -258,6 +437,15 @@ func resourceTable(d *schema.ResourceData, meta interface{}) (*bigquery.Table, e table.ExpirationTime = int64(v.(int)) } + if v, ok := d.GetOk("external_data_configuration"); ok { + externalDataConfiguration, err := expandExternalDataConfiguration(v) + if err != nil { + return nil, err + } + + table.ExternalDataConfiguration = externalDataConfiguration + } + if v, ok := d.GetOk("friendly_name"); ok { table.FriendlyName = v.(string) } @@ -349,6 +537,15 @@ func resourceBigQueryTableRead(d *schema.ResourceData, meta interface{}) error { d.Set("self_link", res.SelfLink) d.Set("type", res.Type) + if res.ExternalDataConfiguration != nil { + externalDataConfiguration, err := flattenExternalDataConfiguration(res.ExternalDataConfiguration) + if err != nil { + return err + } + + d.Set("external_data_configuration", externalDataConfiguration) + } + if res.TimePartitioning != nil { if err := d.Set("time_partitioning", flattenTimePartitioning(res.TimePartitioning)); err != nil { return err @@ -413,6 +610,274 @@ func resourceBigQueryTableDelete(d *schema.ResourceData, meta interface{}) error return nil } +func expandExternalDataConfiguration(cfg interface{}) (*bigquery.ExternalDataConfiguration, error) { + raw := cfg.([]interface{})[0].(map[string]interface{}) + + edc := &bigquery.ExternalDataConfiguration{ + Autodetect: raw["autodetect"].(bool), + } + + sourceUris := []string{} + for _, rawSourceUri := range raw["source_uris"].([]interface{}) { + sourceUris = append(sourceUris, rawSourceUri.(string)) + } + if len(sourceUris) > 0 { + edc.SourceUris = sourceUris + } + + if v, ok := raw["bigtable_options"]; ok { + bigtableOpts, err := expandBigtableOptions(v) + if err != nil { + return nil, err + } + edc.BigtableOptions = bigtableOpts + } + if v, ok := raw["compression"]; ok { + edc.Compression = v.(string) + } + if v, ok := raw["csv_options"]; ok { + edc.CsvOptions = expandCsvOptions(v) + } + if v, ok := raw["google_sheets_options"]; ok { + edc.GoogleSheetsOptions = expandGoogleSheetsOptions(v) + } + if v, ok := raw["ignore_unknown_values"]; ok { + edc.IgnoreUnknownValues = v.(bool) + } + if v, ok := raw["max_bad_records"]; ok { + edc.MaxBadRecords = int64(v.(int)) + } + if v, ok := raw["schema"]; ok { + schema, err := expandSchema(v) + if err != nil { + return nil, err + } + + edc.Schema = schema + } + if v, ok := raw["source_format"]; ok { + edc.SourceFormat = v.(string) + } + + return edc, nil + +} + +func flattenExternalDataConfiguration(edc *bigquery.ExternalDataConfiguration) (map[string]interface{}, error) { + result := map[string]interface{}{} + + result["autodetect"] = edc.Autodetect + result["source_uris"] = edc.SourceUris + + if edc.BigtableOptions != nil { + columnFamilies, err := flattenBigtableOptions(edc.BigtableOptions) + if err != nil { + return result, err + } + result["bigtable_options"] = columnFamilies + } + + if edc.Compression != "" { + result["compression"] = edc.Compression + } + + if edc.CsvOptions != nil { + result["csv_options"] = flattenCsvOptions(edc.CsvOptions) + } + + if edc.GoogleSheetsOptions != nil { + result["google_sheets_options"] = flattenGoogleSheetsOptions(edc.GoogleSheetsOptions) + } + + if edc.IgnoreUnknownValues == true { + result["ignore_unknown_values"] = edc.IgnoreUnknownValues + } + if edc.MaxBadRecords != 0 { + result["max_bad_records"] = edc.MaxBadRecords + } + + if edc.Schema != nil { + schema, err := flattenSchema(edc.Schema) + if err != nil { + return result, err + } + result["schema"] = schema + } + + if edc.SourceFormat != "" { + result["source_format"] = edc.SourceFormat + } + + return result, nil +} + +func expandBigtableOptions(configured interface{}) (*bigquery.BigtableOptions, error) { + if len(configured.([]interface{})) == 0 { + return nil, nil + } + + raw := configured.([]interface{})[0].(map[string]interface{}) + opts := &bigquery.BigtableOptions{} + + if v, ok := raw["column_families"]; ok { + columnFamilies, err := expandColumnFamilies(v) + if err != nil { + return nil, err + } + + opts.ColumnFamilies = columnFamilies + } + + if v, ok := raw["ignore_unspecified_column_families"]; ok { + opts.IgnoreUnspecifiedColumnFamilies = v.(bool) + } + + if v, ok := raw["read_rowkey_as_string"]; ok { + opts.ReadRowkeyAsString = v.(bool) + } + + return opts, nil +} + +func flattenBigtableOptions(opts *bigquery.BigtableOptions) ([]map[string]interface{}, error) { + result := map[string]interface{}{} + + if opts.ColumnFamilies != nil { + columnFamilies, err := flattenColumnFamilies(opts.ColumnFamilies) + if err != nil { + return []map[string]interface{}{result}, err + } + result["column_families"] = columnFamilies + } + + if opts.IgnoreUnspecifiedColumnFamilies == true { + result["ignore_unspecified_column_families"] = opts.IgnoreUnspecifiedColumnFamilies + } + + if opts.ReadRowkeyAsString == true { + result["read_rowkey_as_string"] = opts.ReadRowkeyAsString + } + + return []map[string]interface{}{result}, nil +} + +func expandCsvOptions(configured interface{}) *bigquery.CsvOptions { + if len(configured.([]interface{})) == 0 { + return nil + } + + raw := configured.([]interface{})[0].(map[string]interface{}) + opts := &bigquery.CsvOptions{} + + if v, ok := raw["allow_jagged_rows"]; ok { + opts.AllowJaggedRows = v.(bool) + } + + if v, ok := raw["allow_quoted_newlines"]; ok { + opts.AllowQuotedNewlines = v.(bool) + } + + if v, ok := raw["encoding"]; ok { + opts.Encoding = v.(string) + } + + if v, ok := raw["field_delimiter"]; ok { + opts.FieldDelimiter = v.(string) + } + + if v, ok := raw["skip_leading_rows"]; ok { + opts.SkipLeadingRows = int64(v.(int)) + } + + if v, ok := raw["quote"]; ok { + quote := v.(string) + opts.Quote = "e + } + + return opts +} + +func flattenCsvOptions(opts *bigquery.CsvOptions) []map[string]interface{} { + result := map[string]interface{}{} + + if opts.AllowJaggedRows == true { + result["allow_jagged_rows"] = opts.AllowJaggedRows + } + + if opts.AllowQuotedNewlines == true { + result["allow_quoted_newlines"] = opts.AllowQuotedNewlines + } + + if opts.Encoding != "" { + result["encoding"] = opts.Encoding + } + + if opts.FieldDelimiter != "" { + result["field_delimiter"] = opts.FieldDelimiter + } + + if opts.SkipLeadingRows != 0 { + result["skip_leading_rows"] = opts.SkipLeadingRows + } + + if *opts.Quote != "" { + result["quote"] = *opts.Quote + } + + return []map[string]interface{}{result} +} + +func expandGoogleSheetsOptions(configured interface{}) *bigquery.GoogleSheetsOptions { + if len(configured.([]interface{})) == 0 { + return nil + } + + raw := configured.([]interface{})[0].(map[string]interface{}) + opts := &bigquery.GoogleSheetsOptions{} + + if v, ok := raw["range"]; ok { + opts.Range = v.(string) + } + + if v, ok := raw["skip_leading_rows"]; ok { + opts.SkipLeadingRows = int64(v.(int)) + } + return opts +} + +func flattenGoogleSheetsOptions(opts *bigquery.GoogleSheetsOptions) []map[string]interface{} { + result := map[string]interface{}{} + + if opts.Range != "" { + result["range"] = opts.Range + } + + if opts.SkipLeadingRows != 0 { + result["skip_leading_rows"] = opts.SkipLeadingRows + } + + return []map[string]interface{}{result} +} + +func expandColumnFamilies(raw interface{}) ([]*bigquery.BigtableColumnFamily, error) { + var columnFamilies []*bigquery.BigtableColumnFamily + + if err := json.Unmarshal([]byte(raw.(string)), &columnFamilies); err != nil { + return nil, err + } + + return columnFamilies, nil +} + +func flattenColumnFamilies(columnFamilies []*bigquery.BigtableColumnFamily) (string, error) { + cf, err := json.Marshal(columnFamilies) + if err != nil { + return "", err + } + + return string(cf), nil +} + func expandSchema(raw interface{}) (*bigquery.TableSchema, error) { var fields []*bigquery.TableFieldSchema diff --git a/website/docs/r/bigquery_table.html.markdown b/website/docs/r/bigquery_table.html.markdown index 6e69197941c..37426025875 100644 --- a/website/docs/r/bigquery_table.html.markdown +++ b/website/docs/r/bigquery_table.html.markdown @@ -40,7 +40,40 @@ resource "google_bigquery_table" "default" { env = "default" } - schema = "${file("schema.json")}" + schema = <