diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ffc9672f554..5bb8cd90def 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -510,6 +510,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix GCP Project ID being ingested as `cloud.account.id` in `gcp.billing` module {issue}26357[26357] {pull}26412[26412] - Fix memory leak in SQL module when database is not available. {issue}25840[25840] {pull}26607[26607] - Fix aws metric tags with resourcegroupstaggingapi paginator. {issue}26385[26385] {pull}26443[26443] +- Fix quoting in GCP billing table name {issue}26855[26855] {pull}26870[26870] *Packetbeat* diff --git a/x-pack/metricbeat/module/gcp/billing/billing.go b/x-pack/metricbeat/module/gcp/billing/billing.go index a14d263ac6b..cecc6b4a0a4 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing.go +++ b/x-pack/metricbeat/module/gcp/billing/billing.go @@ -205,26 +205,12 @@ func getTables(ctx context.Context, client *bigquery.Client, datasetID string, t func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, tableMeta tableMeta, month string, costType string) ([]mb.Event, error) { var events []mb.Event - query := fmt.Sprintf(` - SELECT - invoice.month, - project.id, - project.name, - billing_account_id, - cost_type, - (SUM(CAST(cost * 1000000 AS int64)) - + SUM(IFNULL((SELECT SUM(CAST(c.amount * 1000000 as int64)) FROM UNNEST(credits) c), 0))) / 1000000 - AS total_exact - FROM %s - WHERE project.id IS NOT NULL - AND invoice.month = '%s' - AND cost_type = '%s' - GROUP BY 1, 2, 3, 4, 5 - ORDER BY 1 ASC, 2 ASC, 3 ASC, 4 ASC, 5 ASC;`, tableMeta.tableFullID, month, costType) - q := client.Query(query) + query := generateQuery(tableMeta.tableFullID, month, costType) m.logger.Debug("bigquery query = ", query) + q := client.Query(query) + // Location must match that of the dataset(s) referenced in the query. q.Location = tableMeta.location @@ -308,3 +294,30 @@ func generateEventID(currentDate string, rowItems []bigquery.Value) string { prefix := hex.EncodeToString(h.Sum(nil)) return prefix[:20] } + +// generateQuery returns the query to be used by the BigQuery client to retrieve monthly +// cost types breakdown. +func generateQuery(tableName, month, costType string) string { + // The table name is user provided, so it may contains special characters. + // In order to allow any character in the table identifier, use the Quoted identifier format. + // See https://github.com/elastic/beats/issues/26855 + // NOTE: is not possible to escape backtics (`) in a multiline string + escapedTableName := fmt.Sprintf("`%s`", tableName) + query := fmt.Sprintf(` +SELECT + invoice.month, + project.id, + project.name, + billing_account_id, + cost_type, + (SUM(CAST(cost * 1000000 AS int64)) + + SUM(IFNULL((SELECT SUM(CAST(c.amount * 1000000 as int64)) FROM UNNEST(credits) c), 0))) / 1000000 + AS total_exact +FROM %s +WHERE project.id IS NOT NULL +AND invoice.month = '%s' +AND cost_type = '%s' +GROUP BY 1, 2, 3, 4, 5 +ORDER BY 1 ASC, 2 ASC, 3 ASC, 4 ASC, 5 ASC;`, escapedTableName, month, costType) + return query +} diff --git a/x-pack/metricbeat/module/gcp/billing/billing_test.go b/x-pack/metricbeat/module/gcp/billing/billing_test.go index 885a2fc9786..0b257b3a6eb 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing_test.go +++ b/x-pack/metricbeat/module/gcp/billing/billing_test.go @@ -5,6 +5,8 @@ package billing import ( + "io/ioutil" + "log" "strconv" "testing" @@ -16,3 +18,17 @@ func TestGetCurrentMonth(t *testing.T) { _, err := strconv.ParseInt(currentMonth, 0, 64) assert.NoError(t, err) } + +func TestGenerateQuery(t *testing.T) { + log.SetOutput(ioutil.Discard) + + query := generateQuery("my-table", "jan", "cost") + log.Println(query) + + // verify that table name quoting is in effect + assert.Contains(t, query, "`my-table`") + // verify the group by is preserved + assert.Contains(t, query, "GROUP BY 1, 2, 3, 4, 5") + // verify the order by is preserved + assert.Contains(t, query, "ORDER BY 1 ASC, 2 ASC, 3 ASC, 4 ASC, 5 ASC") +}