diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c85bf81e5f3..533e8cb6c13 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -665,6 +665,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641] - Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391] - Request prometheus endpoints to be gzipped by default {pull}20766[20766] +- Add billing metricset into googlecloud module. {pull}20812[20812] {issue}20738[20738] *Packetbeat* diff --git a/NOTICE.txt b/NOTICE.txt index e4b49dbfb05..d7f5b0d8b14 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -284,6 +284,218 @@ Contents of probable licence file $GOMODCACHE/cloud.google.com/go@v0.51.0/LICENS limitations under the License. +-------------------------------------------------------------------------------- +Dependency : cloud.google.com/go/bigquery +Version: v1.0.1 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/cloud.google.com/go/bigquery@v1.0.1/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/pubsub Version: v1.0.1 @@ -17121,218 +17333,6 @@ File fuse_kernel.go: */ --------------------------------------------------------------------------------- -Dependency : cloud.google.com/go/bigquery -Version: v1.0.1 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/cloud.google.com/go/bigquery@v1.0.1/LICENSE: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/datastore Version: v1.0.0 diff --git a/go.mod b/go.mod index 50331b7a4b0..b832a3ac892 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.14 require ( 4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b cloud.google.com/go v0.51.0 + cloud.google.com/go/bigquery v1.0.1 cloud.google.com/go/pubsub v1.0.1 cloud.google.com/go/storage v1.0.0 code.cloudfoundry.org/go-diodes v0.0.0-20190809170250-f77fb823c7ee // indirect diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 66619eaa374..ef1d1c70457 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -19812,6 +19812,48 @@ type: object -- +[float] +=== billing + +Google Cloud Billing metrics + + +*`googlecloud.billing.cost_type`*:: ++ +-- +Cost types include regular, tax, adjustment, and rounding_error. + +type: keyword + +-- + +*`googlecloud.billing.invoice_month`*:: ++ +-- +Billing report month. + +type: keyword + +-- + +*`googlecloud.billing.project_id`*:: ++ +-- +Project ID of the billing report belongs to. + +type: keyword + +-- + +*`googlecloud.billing.total`*:: ++ +-- +Total billing amount. + +type: float + +-- + [float] === compute diff --git a/metricbeat/docs/images/metricbeat-googlecloud-billing-overview.png b/metricbeat/docs/images/metricbeat-googlecloud-billing-overview.png new file mode 100644 index 00000000000..b697c285a11 Binary files /dev/null and b/metricbeat/docs/images/metricbeat-googlecloud-billing-overview.png differ diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index dc30097ea87..ce450aee4ca 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -149,9 +149,20 @@ For example, if Compute Metricset fetches 14 metrics (which is the number of met [float] == Metricsets -Currently, we have `compute`, `loadbalancing`, `pubsub`, `metrics` and +Currently, we have `billing`, `compute`, `loadbalancing`, `pubsub`, `metrics` and `storage` metricset in `googlecloud` module. +[float] +=== `billing` +This metricset fetches billing metrics from https://cloud.google.com/bigquery[GCP BigQuery] +Cloud Billing allows users to export billing data into BigQuery automatically +throughout the day. This metricset gets access to the daily cost detail table +periodically to export billing metrics for further analysis. + +The `billing` metricset comes with a predefined dashboard: + +image::./images/metricbeat-googlecloud-billing-overview.png[] + [float] === `compute` This metricset fetches metrics from https://cloud.google.com/compute/[Compute Engine] @@ -274,6 +285,16 @@ metricbeat.modules: - "instance/cpu/usage_time" - "instance/cpu/utilization" - "instance/uptime" + +- module: googlecloud + metricsets: + - billing + period: 24h + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + dataset_id: "dataset id" + table_pattern: "table pattern" + cost_type: "regular" ---- [float] @@ -281,6 +302,8 @@ metricbeat.modules: The following metricsets are available: +* <> + * <> * <> @@ -291,6 +314,8 @@ The following metricsets are available: * <> +include::googlecloud/billing.asciidoc[] + include::googlecloud/compute.asciidoc[] include::googlecloud/loadbalancing.asciidoc[] diff --git a/metricbeat/docs/modules/googlecloud/billing.asciidoc b/metricbeat/docs/modules/googlecloud/billing.asciidoc new file mode 100644 index 00000000000..cfb91238840 --- /dev/null +++ b/metricbeat/docs/modules/googlecloud/billing.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-googlecloud-billing]] +[role="xpack"] +=== Google Cloud Platform billing metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/googlecloud/billing/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/googlecloud/billing/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 72bf027a51e..0fabf48383c 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -111,7 +111,8 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.5+| .5+| |<> beta[] +.6+| .6+| |<> beta[] +|<> beta[] |<> beta[] |<> beta[] |<> beta[] diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 108d14802aa..b0b66085b2b 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -33,6 +33,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/coredns" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/coredns/stats" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/billing" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/metrics" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/ibmmq" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/iis" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 070e73c851b..2c38df1fffc 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -555,6 +555,16 @@ metricbeat.modules: - "instance/cpu/utilization" - "instance/uptime" +- module: googlecloud + metricsets: + - billing + period: 24h + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + dataset_id: "dataset id" + table_pattern: "table pattern" + cost_type: "regular" + #------------------------------- Graphite Module ------------------------------- - module: graphite metricsets: ["server"] diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index d73737d93c5..b6f24962b29 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -40,3 +40,13 @@ - "instance/cpu/usage_time" - "instance/cpu/utilization" - "instance/uptime" + +- module: googlecloud + metricsets: + - billing + period: 24h + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + dataset_id: "dataset id" + table_pattern: "table pattern" + cost_type: "regular" diff --git a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc index 00846498770..d2e74d0122f 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc @@ -139,9 +139,20 @@ For example, if Compute Metricset fetches 14 metrics (which is the number of met [float] == Metricsets -Currently, we have `compute`, `loadbalancing`, `pubsub`, `metrics` and +Currently, we have `billing`, `compute`, `loadbalancing`, `pubsub`, `metrics` and `storage` metricset in `googlecloud` module. +[float] +=== `billing` +This metricset fetches billing metrics from https://cloud.google.com/bigquery[GCP BigQuery] +Cloud Billing allows users to export billing data into BigQuery automatically +throughout the day. This metricset gets access to the daily cost detail table +periodically to export billing metrics for further analysis. + +The `billing` metricset comes with a predefined dashboard: + +image::./images/metricbeat-googlecloud-billing-overview.png[] + [float] === `compute` This metricset fetches metrics from https://cloud.google.com/compute/[Compute Engine] diff --git a/x-pack/metricbeat/module/googlecloud/_meta/kibana/7/dashboard/Metricbeat-googlecloud-billing-overview.json b/x-pack/metricbeat/module/googlecloud/_meta/kibana/7/dashboard/Metricbeat-googlecloud-billing-overview.json new file mode 100644 index 00000000000..4387e827748 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/_meta/kibana/7/dashboard/Metricbeat-googlecloud-billing-overview.json @@ -0,0 +1,773 @@ +{ + "objects": [ + { + "attributes": { + "description": "Overview of Google Cloud Billing Metrics", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "title": "Filters" + }, + "gridData": { + "h": 14, + "i": "2552123b-6ad6-4d63-89c3-0672ab428580", + "w": 8, + "x": 0, + "y": 0 + }, + "panelIndex": "2552123b-6ad6-4d63-89c3-0672ab428580", + "panelRefName": "panel_0", + "title": "Filters", + "version": "7.9.0" + }, + { + "embeddableConfig": { + "title": "Cost Per Month" + }, + "gridData": { + "h": 20, + "i": "2d3d3b79-0656-45c2-b051-4489484b625c", + "w": 10, + "x": 8, + "y": 0 + }, + "panelIndex": "2d3d3b79-0656-45c2-b051-4489484b625c", + "panelRefName": "panel_1", + "title": "Cost Per Month", + "version": "7.9.0" + }, + { + "embeddableConfig": { + "title": "Cost Per Project ID" + }, + "gridData": { + "h": 20, + "i": "b737e597-cc4d-4437-859c-6d491679599d", + "w": 30, + "x": 18, + "y": 0 + }, + "panelIndex": "b737e597-cc4d-4437-859c-6d491679599d", + "panelRefName": "panel_2", + "title": "Cost Per Project ID", + "version": "7.9.0" + }, + { + "embeddableConfig": { + "title": "Total Number Of Projects" + }, + "gridData": { + "h": 6, + "i": "9eedb0c7-2089-4e0f-af98-721034203aad", + "w": 8, + "x": 0, + "y": 14 + }, + "panelIndex": "9eedb0c7-2089-4e0f-af98-721034203aad", + "panelRefName": "panel_3", + "title": "Total Number Of Projects", + "version": "7.9.0" + }, + { + "embeddableConfig": { + "title": "Cost Per Invoice Month" + }, + "gridData": { + "h": 15, + "i": "f4d0ebcb-ac15-4c31-ab57-7f22e0c3e02a", + "w": 48, + "x": 0, + "y": 20 + }, + "panelIndex": "f4d0ebcb-ac15-4c31-ab57-7f22e0c3e02a", + "panelRefName": "panel_4", + "title": "Cost Per Invoice Month", + "version": "7.9.0" + }, + { + "embeddableConfig": { + "title": "Total Billing Cost" + }, + "gridData": { + "h": 15, + "i": "991e60a8-68eb-4c2b-ac9a-b553e90dd49d", + "w": 48, + "x": 0, + "y": 35 + }, + "panelIndex": "991e60a8-68eb-4c2b-ac9a-b553e90dd49d", + "panelRefName": "panel_5", + "title": "Total Billing Cost", + "version": "7.9.0" + } + ], + "timeRestore": false, + "title": "[Metricbeat Googlecloud] Billing Overview", + "version": 1 + }, + "id": "76c9e920-e890-11ea-bf8c-d13ebf358a78", + "migrationVersion": { + "dashboard": "7.3.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "5d2f9160-e88e-11ea-bf8c-d13ebf358a78", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "520c6f10-ec8a-11ea-a0ed-7fe6b565d158", + "name": "panel_1", + "type": "lens" + }, + { + "id": "057de170-e88d-11ea-bf8c-d13ebf358a78", + "name": "panel_2", + "type": "lens" + }, + { + "id": "dd835300-e88f-11ea-bf8c-d13ebf358a78", + "name": "panel_3", + "type": "lens" + }, + { + "id": "e6933020-e88d-11ea-bf8c-d13ebf358a78", + "name": "panel_4", + "type": "lens" + }, + { + "id": "73346db0-e88d-11ea-bf8c-d13ebf358a78", + "name": "panel_5", + "type": "lens" + } + ], + "type": "dashboard", + "updated_at": "2020-09-01T19:41:37.582Z", + "version": "WzYwNDMsOF0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Account ID Filter [Metricbeat Googlecloud]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "cloud.provider", + "id": "1598550838945", + "indexPatternRefName": "control_0_index_pattern", + "label": "Cloud Provider ", + "options": { + "dynamicOptions": true, + "multiselect": false, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + }, + { + "fieldName": "cloud.account.id", + "id": "1598893530938", + "indexPatternRefName": "control_1_index_pattern", + "label": "Account ID", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "1598550838945", + "type": "list" + }, + { + "fieldName": "googlecloud.billing.invoice_month", + "id": "1598988595566", + "indexPatternRefName": "control_2_index_pattern", + "label": "Invoice Month", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": false, + "useTimeFilter": false + }, + "title": "Account ID Filter [Metricbeat Googlecloud]", + "type": "input_control_vis" + } + }, + "id": "5d2f9160-e88e-11ea-bf8c-d13ebf358a78", + "migrationVersion": { + "visualization": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_1_index_pattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_2_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2020-09-01T19:30:18.073Z", + "version": "WzU3NDMsOF0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"e12171da-25a4-41ea-86d3-8fd71205c263\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true aggConfigs=\"[{\\\"id\\\":\\\"6011e524-4646-410b-8d1c-06c281e8f7ed\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"googlecloud.billing.invoice_month\\\",\\\"orderBy\\\":\\\"f8ab301c-f139-4573-b233-ed8a3f717e24\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":12,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"f8ab301c-f139-4573-b233-ed8a3f717e24\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"sum\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"googlecloud.billing.total\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-6011e524-4646-410b-8d1c-06c281e8f7ed\\\":{\\\"label\\\":\\\"Invoice Month\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"googlecloud.billing.invoice_month\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":12,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"f8ab301c-f139-4573-b233-ed8a3f717e24\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"6011e524-4646-410b-8d1c-06c281e8f7ed\\\"},\\\"col-1-f8ab301c-f139-4573-b233-ed8a3f717e24\\\":{\\\"label\\\":\\\"Total Billing Cost\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"sum\\\",\\\"sourceField\\\":\\\"googlecloud.billing.total\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"f8ab301c-f139-4573-b233-ed8a3f717e24\\\"}}\"}\n| lens_datatable \n columns={lens_datatable_columns columnIds=\"6011e524-4646-410b-8d1c-06c281e8f7ed\" columnIds=\"f8ab301c-f139-4573-b233-ed8a3f717e24\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "e12171da-25a4-41ea-86d3-8fd71205c263": { + "columnOrder": [ + "6011e524-4646-410b-8d1c-06c281e8f7ed", + "f8ab301c-f139-4573-b233-ed8a3f717e24" + ], + "columns": { + "6011e524-4646-410b-8d1c-06c281e8f7ed": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Invoice Month", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "f8ab301c-f139-4573-b233-ed8a3f717e24", + "type": "column" + }, + "orderDirection": "desc", + "size": 12 + }, + "scale": "ordinal", + "sourceField": "googlecloud.billing.invoice_month" + }, + "f8ab301c-f139-4573-b233-ed8a3f717e24": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Billing Cost", + "operationType": "sum", + "scale": "ratio", + "sourceField": "googlecloud.billing.total" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "columns": [ + "6011e524-4646-410b-8d1c-06c281e8f7ed", + "f8ab301c-f139-4573-b233-ed8a3f717e24" + ], + "layerId": "e12171da-25a4-41ea-86d3-8fd71205c263" + } + ] + } + }, + "title": "Total Cost Table [Metricbeat Googlecloud]", + "visualizationType": "lnsDatatable" + }, + "id": "520c6f10-ec8a-11ea-a0ed-7fe6b565d158", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-09-01T19:35:41.441Z", + "version": "WzU5MDEsOF0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"4ca843af-63d7-46b9-a719-51a81eebf1f7\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true aggConfigs=\"[{\\\"id\\\":\\\"2477291e-9021-4eb2-9fce-8da1ee792c49\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"googlecloud.billing.project_id\\\",\\\"orderBy\\\":\\\"10b91492-efef-490d-bc7a-c2074b2eae84\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":20,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"10b91492-efef-490d-bc7a-c2074b2eae84\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"max\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"googlecloud.billing.total\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-2477291e-9021-4eb2-9fce-8da1ee792c49\\\":{\\\"label\\\":\\\"Cost Per Project ID\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"googlecloud.billing.project_id\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":20,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"10b91492-efef-490d-bc7a-c2074b2eae84\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"2477291e-9021-4eb2-9fce-8da1ee792c49\\\"},\\\"col-1-10b91492-efef-490d-bc7a-c2074b2eae84\\\":{\\\"label\\\":\\\"Maximum of googlecloud.billing.total\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"max\\\",\\\"sourceField\\\":\\\"googlecloud.billing.total\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"id\\\":\\\"10b91492-efef-490d-bc7a-c2074b2eae84\\\"}}\"}\n| lens_pie shape=\"pie\" hideLabels=false groups=\"2477291e-9021-4eb2-9fce-8da1ee792c49\" metric=\"10b91492-efef-490d-bc7a-c2074b2eae84\" numberDisplay=\"percent\" categoryDisplay=\"default\" legendDisplay=\"default\" percentDecimals=3 nestedLegend=false", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "4ca843af-63d7-46b9-a719-51a81eebf1f7": { + "columnOrder": [ + "2477291e-9021-4eb2-9fce-8da1ee792c49", + "10b91492-efef-490d-bc7a-c2074b2eae84" + ], + "columns": { + "10b91492-efef-490d-bc7a-c2074b2eae84": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of googlecloud.billing.total", + "operationType": "max", + "scale": "ratio", + "sourceField": "googlecloud.billing.total" + }, + "2477291e-9021-4eb2-9fce-8da1ee792c49": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Cost Per Project ID", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "10b91492-efef-490d-bc7a-c2074b2eae84", + "type": "column" + }, + "orderDirection": "desc", + "size": 20 + }, + "scale": "ordinal", + "sourceField": "googlecloud.billing.project_id" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "groups": [ + "2477291e-9021-4eb2-9fce-8da1ee792c49" + ], + "layerId": "4ca843af-63d7-46b9-a719-51a81eebf1f7", + "legendDisplay": "default", + "metric": "10b91492-efef-490d-bc7a-c2074b2eae84", + "nestedLegend": false, + "numberDisplay": "percent" + } + ], + "shape": "pie" + } + }, + "title": "Cost Per Project ID [Metricbeat Googlecloud]", + "visualizationType": "lnsPie" + }, + "id": "057de170-e88d-11ea-bf8c-d13ebf358a78", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-08-31T02:45:59.906Z", + "version": "WzUxODksOF0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"4cb00ce3-c62e-46f3-90ce-b69c876b9605\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true aggConfigs=\"[{\\\"id\\\":\\\"2f66b924-5392-4e5e-93fe-5b23a87068c1\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"cardinality\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"googlecloud.billing.project_id\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-2f66b924-5392-4e5e-93fe-5b23a87068c1\\\":{\\\"label\\\":\\\"\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"cardinality\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"sourceField\\\":\\\"googlecloud.billing.project_id\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"2f66b924-5392-4e5e-93fe-5b23a87068c1\\\"}}\"}\n| lens_metric_chart title=\"\" accessor=\"2f66b924-5392-4e5e-93fe-5b23a87068c1\" mode=\"full\"", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "4cb00ce3-c62e-46f3-90ce-b69c876b9605": { + "columnOrder": [ + "2f66b924-5392-4e5e-93fe-5b23a87068c1" + ], + "columns": { + "2f66b924-5392-4e5e-93fe-5b23a87068c1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "", + "operationType": "cardinality", + "scale": "ratio", + "sourceField": "googlecloud.billing.project_id" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "2f66b924-5392-4e5e-93fe-5b23a87068c1", + "layerId": "4cb00ce3-c62e-46f3-90ce-b69c876b9605" + } + }, + "title": "Total Number Of Projects [Metricbeat Googlecloud]", + "visualizationType": "lnsMetric" + }, + "id": "dd835300-e88f-11ea-bf8c-d13ebf358a78", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-08-31T02:50:27.115Z", + "version": "WzUzMTgsOF0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"4ca843af-63d7-46b9-a719-51a81eebf1f7\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true aggConfigs=\"[{\\\"id\\\":\\\"e25f49de-f161-4be8-a8fc-519188a7776c\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"googlecloud.billing.project_id\\\",\\\"orderBy\\\":\\\"af747bf6-66e9-4760-bbd8-3dae9c97159d\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":10,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"b92edf5e-58bc-4382-9cd5-19db2c332c93\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"googlecloud.billing.invoice_month\\\",\\\"orderBy\\\":\\\"_key\\\",\\\"order\\\":\\\"asc\\\",\\\"size\\\":5,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"af747bf6-66e9-4760-bbd8-3dae9c97159d\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"max\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"googlecloud.billing.total\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-e25f49de-f161-4be8-a8fc-519188a7776c\\\":{\\\"label\\\":\\\"Monthly Cost\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"googlecloud.billing.project_id\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":10,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"af747bf6-66e9-4760-bbd8-3dae9c97159d\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"e25f49de-f161-4be8-a8fc-519188a7776c\\\"},\\\"col-2-b92edf5e-58bc-4382-9cd5-19db2c332c93\\\":{\\\"label\\\":\\\"Invoice Month\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"googlecloud.billing.invoice_month\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":5,\\\"orderBy\\\":{\\\"type\\\":\\\"alphabetical\\\"},\\\"orderDirection\\\":\\\"asc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"b92edf5e-58bc-4382-9cd5-19db2c332c93\\\"},\\\"col-3-af747bf6-66e9-4760-bbd8-3dae9c97159d\\\":{\\\"label\\\":\\\"Total Billing Cost\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"max\\\",\\\"sourceField\\\":\\\"googlecloud.billing.total\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"af747bf6-66e9-4760-bbd8-3dae9c97159d\\\"}}\"}\n| lens_xy_chart xTitle=\"Monthly Cost\" yTitle=\"Total Billing Cost\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"4ca843af-63d7-46b9-a719-51a81eebf1f7\" hide=false xAccessor=\"e25f49de-f161-4be8-a8fc-519188a7776c\" yScaleType=\"linear\" xScaleType=\"ordinal\" isHistogram=false splitAccessor=\"b92edf5e-58bc-4382-9cd5-19db2c332c93\" seriesType=\"bar_stacked\" accessors=\"af747bf6-66e9-4760-bbd8-3dae9c97159d\" columnToLabel=\"{\\\"af747bf6-66e9-4760-bbd8-3dae9c97159d\\\":\\\"Total Billing Cost\\\",\\\"b92edf5e-58bc-4382-9cd5-19db2c332c93\\\":\\\"Invoice Month\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "4ca843af-63d7-46b9-a719-51a81eebf1f7": { + "columnOrder": [ + "e25f49de-f161-4be8-a8fc-519188a7776c", + "b92edf5e-58bc-4382-9cd5-19db2c332c93", + "af747bf6-66e9-4760-bbd8-3dae9c97159d" + ], + "columns": { + "af747bf6-66e9-4760-bbd8-3dae9c97159d": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Billing Cost", + "operationType": "max", + "scale": "ratio", + "sourceField": "googlecloud.billing.total" + }, + "b92edf5e-58bc-4382-9cd5-19db2c332c93": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Invoice Month", + "operationType": "terms", + "params": { + "orderBy": { + "type": "alphabetical" + }, + "orderDirection": "asc", + "size": 5 + }, + "scale": "ordinal", + "sourceField": "googlecloud.billing.invoice_month" + }, + "e25f49de-f161-4be8-a8fc-519188a7776c": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Monthly Cost", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "af747bf6-66e9-4760-bbd8-3dae9c97159d", + "type": "column" + }, + "orderDirection": "desc", + "size": 10 + }, + "scale": "ordinal", + "sourceField": "googlecloud.billing.project_id" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "af747bf6-66e9-4760-bbd8-3dae9c97159d" + ], + "layerId": "4ca843af-63d7-46b9-a719-51a81eebf1f7", + "seriesType": "bar_stacked", + "splitAccessor": "b92edf5e-58bc-4382-9cd5-19db2c332c93", + "xAccessor": "e25f49de-f161-4be8-a8fc-519188a7776c" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked" + } + }, + "title": "Monthly Cost Per Project [Metricbeat Googlecloud]", + "visualizationType": "lnsXY" + }, + "id": "e6933020-e88d-11ea-bf8c-d13ebf358a78", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-08-31T17:12:47.088Z", + "version": "WzU1NjksOF0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"4ca843af-63d7-46b9-a719-51a81eebf1f7\"\n layerIds=\"325e60ce-0fbd-42b0-82f6-b10df31fef6c\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"1164563d-d2b3-4067-bc7b-d694179182ed\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"1d\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"10b91492-efef-490d-bc7a-c2074b2eae84\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"sum\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"googlecloud.billing.total\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-1164563d-d2b3-4067-bc7b-d694179182ed\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"1d\\\"},\\\"id\\\":\\\"1164563d-d2b3-4067-bc7b-d694179182ed\\\"},\\\"col-1-10b91492-efef-490d-bc7a-c2074b2eae84\\\":{\\\"label\\\":\\\"Total Billing Cost\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"sum\\\",\\\"sourceField\\\":\\\"googlecloud.billing.total\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"10b91492-efef-490d-bc7a-c2074b2eae84\\\"}}\"}\n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"faaaaf23-f362-4a00-be9e-8a155208a39e\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"googlecloud.billing.project_id\\\",\\\"orderBy\\\":\\\"3041fc1b-ceb8-4188-b55d-d354819f267e\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":10,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"c4bc659c-3e7c-41f2-bc38-32d9edee95e8\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"1d\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"3041fc1b-ceb8-4188-b55d-d354819f267e\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"max\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"googlecloud.billing.total\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-faaaaf23-f362-4a00-be9e-8a155208a39e\\\":{\\\"sourceField\\\":\\\"googlecloud.billing.project_id\\\",\\\"isBucketed\\\":true,\\\"dataType\\\":\\\"string\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"operationType\\\":\\\"terms\\\",\\\"label\\\":\\\"Project ID\\\",\\\"customLabel\\\":true,\\\"params\\\":{\\\"size\\\":10,\\\"orderBy\\\":{\\\"columnId\\\":\\\"3041fc1b-ceb8-4188-b55d-d354819f267e\\\",\\\"type\\\":\\\"column\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"id\\\":\\\"faaaaf23-f362-4a00-be9e-8a155208a39e\\\"},\\\"col-2-c4bc659c-3e7c-41f2-bc38-32d9edee95e8\\\":{\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"dataType\\\":\\\"date\\\",\\\"scale\\\":\\\"interval\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"label\\\":\\\"@timestamp\\\",\\\"params\\\":{\\\"interval\\\":\\\"1d\\\"},\\\"id\\\":\\\"c4bc659c-3e7c-41f2-bc38-32d9edee95e8\\\"},\\\"col-3-3041fc1b-ceb8-4188-b55d-d354819f267e\\\":{\\\"sourceField\\\":\\\"googlecloud.billing.total\\\",\\\"isBucketed\\\":false,\\\"dataType\\\":\\\"number\\\",\\\"scale\\\":\\\"ratio\\\",\\\"operationType\\\":\\\"max\\\",\\\"label\\\":\\\"Total Billing\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"3041fc1b-ceb8-4188-b55d-d354819f267e\\\"}}\"}\n| lens_xy_chart xTitle=\"@timestamp [1]\" yTitle=\"Total Billing\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"325e60ce-0fbd-42b0-82f6-b10df31fef6c\" hide=false xAccessor=\"c4bc659c-3e7c-41f2-bc38-32d9edee95e8\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"faaaaf23-f362-4a00-be9e-8a155208a39e\" seriesType=\"bar_stacked\" accessors=\"3041fc1b-ceb8-4188-b55d-d354819f267e\" columnToLabel=\"{\\\"3041fc1b-ceb8-4188-b55d-d354819f267e\\\":\\\"Total Billing\\\",\\\"faaaaf23-f362-4a00-be9e-8a155208a39e\\\":\\\"Project ID\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "325e60ce-0fbd-42b0-82f6-b10df31fef6c": { + "columnOrder": [ + "faaaaf23-f362-4a00-be9e-8a155208a39e", + "c4bc659c-3e7c-41f2-bc38-32d9edee95e8", + "3041fc1b-ceb8-4188-b55d-d354819f267e" + ], + "columns": { + "3041fc1b-ceb8-4188-b55d-d354819f267e": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Billing", + "operationType": "max", + "scale": "ratio", + "sourceField": "googlecloud.billing.total" + }, + "c4bc659c-3e7c-41f2-bc38-32d9edee95e8": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "1d" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "faaaaf23-f362-4a00-be9e-8a155208a39e": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Project ID", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "3041fc1b-ceb8-4188-b55d-d354819f267e", + "type": "column" + }, + "orderDirection": "desc", + "size": 10 + }, + "scale": "ordinal", + "sourceField": "googlecloud.billing.project_id" + } + }, + "indexPatternId": "metricbeat-*" + }, + "4ca843af-63d7-46b9-a719-51a81eebf1f7": { + "columnOrder": [ + "1164563d-d2b3-4067-bc7b-d694179182ed", + "10b91492-efef-490d-bc7a-c2074b2eae84" + ], + "columns": { + "10b91492-efef-490d-bc7a-c2074b2eae84": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Billing Cost", + "operationType": "sum", + "scale": "ratio", + "sourceField": "googlecloud.billing.total" + }, + "1164563d-d2b3-4067-bc7b-d694179182ed": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "1d" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "e25f49de-f161-4be8-a8fc-519188a7776c": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Cost", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "10b91492-efef-490d-bc7a-c2074b2eae84", + "type": "column" + }, + "orderDirection": "desc", + "size": 15 + }, + "scale": "ordinal", + "sourceField": "googlecloud.billing.project_id" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "3041fc1b-ceb8-4188-b55d-d354819f267e" + ], + "layerId": "325e60ce-0fbd-42b0-82f6-b10df31fef6c", + "seriesType": "bar_stacked", + "splitAccessor": "faaaaf23-f362-4a00-be9e-8a155208a39e", + "xAccessor": "c4bc659c-3e7c-41f2-bc38-32d9edee95e8" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked" + } + }, + "title": "Total Cost Bar Chart [Metricbeat Googlecloud]", + "visualizationType": "lnsXY" + }, + "id": "73346db0-e88d-11ea-bf8c-d13ebf358a78", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-09-01T19:36:43.994Z", + "version": "WzU5MzgsOF0=" + } + ], + "version": "7.9.0" +} diff --git a/x-pack/metricbeat/module/googlecloud/billing/_meta/data.json b/x-pack/metricbeat/module/googlecloud/billing/_meta/data.json new file mode 100644 index 00000000000..92ee77fb15b --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/billing/_meta/data.json @@ -0,0 +1,30 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "cloud": { + "account": { + "id": "elastic-bi", + "name": "elastic-bi" + }, + "provider": "googlecloud" + }, + "event": { + "dataset": "googlecloud.billing", + "duration": 115000, + "module": "googlecloud" + }, + "googlecloud": { + "billing": { + "cost_type": "regular", + "invoice_month": "202008", + "project_id": "elastic-fin-bi", + "total": 170.811692 + } + }, + "metricset": { + "name": "billing", + "period": 10000 + }, + "service": { + "type": "googlecloud" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/googlecloud/billing/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/billing/_meta/docs.asciidoc new file mode 100644 index 00000000000..ccf26a17600 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/billing/_meta/docs.asciidoc @@ -0,0 +1,39 @@ +`billing` metricset is designed for collecting billing metrics from Google Cloud +BigQuery daily cost detail table. BigQuery is a fully-managed, serverless data +warehouse. +Cloud Billing export to BigQuery enables you to export detailed Google Cloud +billing data (such as usage, cost estimates, and pricing data) automatically +throughout the day to a BigQuery dataset that you specify. Then you can access +your Cloud Billing data from BigQuery for detailed analysis using Metricbeat. +Please see https://cloud.google.com/billing/docs/how-to/export-data-bigquery[export +cloud billing data to BigQuery] for more details on how to export billing data. + +In BigQuery dataset, detailed Google Cloud daily cost data is loaded into a data +table named `gcp_billing_export_v1_`. There is a defined +schema for Google Cloud daily cost data that is exported to BigQuery. Please see +https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables#data-schema[ +daily cost detail data schema] for more details. + +[float] +=== Metricset-specific configuration notes +* *dataset_id*: (Required) Dataset ID that points to the top-level container which contains +the actual billing tables. +* *table_pattern*: (Optional) Daily cost detail billing table name prefix. +Default to `gcp_billing_export_v1`. +* *cost_type*: (Optional) The type of cost this line item represents: regular, +tax, adjustment, or rounding error. Default to `regular`. + +[float] +=== Configuration example +[source,yaml] +---- +- module: googlecloud + metricsets: + - billing + period: 24h + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + dataset_id: "dataset id" + table_pattern: "table pattern" + cost_type: "regular" +---- diff --git a/x-pack/metricbeat/module/googlecloud/billing/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/billing/_meta/fields.yml new file mode 100644 index 00000000000..56608ad00a7 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/billing/_meta/fields.yml @@ -0,0 +1,17 @@ +- name: billing + release: beta + type: group + description: Google Cloud Billing metrics + fields: + - name: cost_type + type: keyword + description: Cost types include regular, tax, adjustment, and rounding_error. + - name: invoice_month + type: keyword + description: Billing report month. + - name: project_id + type: keyword + description: Project ID of the billing report belongs to. + - name: total + type: float + description: Total billing amount. diff --git a/x-pack/metricbeat/module/googlecloud/billing/billing.go b/x-pack/metricbeat/module/googlecloud/billing/billing.go new file mode 100644 index 00000000000..a314df78100 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/billing/billing.go @@ -0,0 +1,305 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package billing + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "time" + + "cloud.google.com/go/bigquery" + "google.golang.org/api/iterator" + "google.golang.org/api/option" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud" +) + +const ( + // metricsetName is the name of this metricset + metricsetName = "billing" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet(googlecloud.ModuleName, metricsetName, New) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + config config + logger *logp.Logger +} + +type config struct { + Period time.Duration `config:"period" validate:"required"` + ProjectID string `config:"project_id" validate:"required"` + CredentialsFilePath string `config:"credentials_file_path" validate:"required"` + DatasetID string `config:"dataset_id" validate:"required"` + TablePattern string `config:"table_pattern"` + CostType string `config:"cost_type"` +} + +// Validate checks for deprecated config options +func (c config) Validate() error { + if c.CostType != "" { + // cost_type can only be regular, tax, adjustment, or rounding error + costTypes := []string{"regular", "tax", "adjustment", "rounding error"} + if stringInSlice(c.CostType, costTypes) { + return nil + } + return fmt.Errorf("given cost_type %s is not in supported list %s", c.CostType, costTypes) + } + + if c.Period.Hours() < 24 { + return fmt.Errorf("collection period for billing metricset %s cannot be less than 24 hours", c.Period) + } + return nil +} + +func stringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The gcp '%s' metricset is beta.", metricsetName) + + m := &MetricSet{ + BaseMetricSet: base, + logger: logp.NewLogger(metricsetName), + } + + if err := base.Module().UnpackConfig(&m.config); err != nil { + return nil, fmt.Errorf("unpack billing config failed: %w", err) + } + + m.Logger().Debugf("metricset config: %v", m.config) + return m, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) (err error) { + // find current month + month := getCurrentMonth() + + opt := []option.ClientOption{option.WithCredentialsFile(m.config.CredentialsFilePath)} + client, err := bigquery.NewClient(ctx, m.config.ProjectID, opt...) + if err != nil { + return fmt.Errorf("gerror creating bigquery client: %w", err) + } + + defer client.Close() + + // default table_pattern for query is "gcp_billing_export_v1" + if m.config.TablePattern == "" { + m.logger.Warn("table_pattern is not set in config, \"gcp_billing_export_v1\" will be used by default.") + m.config.TablePattern = "gcp_billing_export_v1" + } + + // default cost_type for query is "regular" + if m.config.CostType == "" { + m.logger.Warn("cost_type is not set in config, \"regular\" will be used by default.") + m.config.CostType = "regular" + } + + tableMetas, err := getTables(ctx, client, m.config.DatasetID, m.config.TablePattern) + if err != nil { + return fmt.Errorf("getTables failed: %w", err) + } + + var events []mb.Event + for _, tableMeta := range tableMetas { + eventsPerQuery, err := m.queryBigQuery(ctx, client, tableMeta, month, m.config.CostType) + if err != nil { + return fmt.Errorf("queryBigQuery failed: %w", err) + } + + events = append(events, eventsPerQuery...) + } + + m.Logger().Debugf("Total %d of events are created for billing", len(events)) + for _, event := range events { + reporter.Event(event) + } + return nil +} + +func getCurrentMonth() string { + currentTime := time.Now() + return fmt.Sprintf("%04d%02d", currentTime.Year(), int(currentTime.Month())) +} + +type tableMeta struct { + tableFullID string + location string +} + +func getTables(ctx context.Context, client *bigquery.Client, datasetID string, tablePattern string) ([]tableMeta, error) { + dit := client.Datasets(ctx) + var tables []tableMeta + + for { + dataset, err := dit.Next() + if err == iterator.Done { + break + } + if err != nil { + return tables, err + } + + meta, err := client.Dataset(dataset.DatasetID).Metadata(ctx) + if err != nil { + return tables, err + } + + // compare with given dataset_id + if dataset.DatasetID != datasetID { + continue + } + + tit := dataset.Tables(ctx) + for { + var tableMeta tableMeta + table, err := tit.Next() + if err == iterator.Done { + break + } + if err != nil { + return tables, err + } + + // make sure table ID fits the given table_pattern + if strings.HasPrefix(table.TableID, tablePattern) { + tableMeta.tableFullID = table.ProjectID + "." + table.DatasetID + "." + table.TableID + tableMeta.location = meta.Location + tables = append(tables, tableMeta) + } + } + } + return tables, nil +} + +func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, tableMeta tableMeta, month string, costType string) ([]mb.Event, error) { + var events []mb.Event + query := fmt.Sprintf(` + SELECT + invoice.month, + project.id, + cost_type, + (SUM(CAST(cost * 1000000 AS int64)) + + SUM(IFNULL((SELECT SUM(CAST(c.amount * 1000000 as int64)) FROM UNNEST(credits) c), 0))) / 1000000 + AS total_exact + FROM %s + WHERE project.id IS NOT NULL + AND invoice.month = '%s' + AND cost_type = '%s' + GROUP BY 1, 2, 3 + ORDER BY 1 ASC, 2 ASC, 3 ASC;`, tableMeta.tableFullID, month, costType) + + q := client.Query(query) + m.logger.Debug("bigquery query = ", query) + + // Location must match that of the dataset(s) referenced in the query. + q.Location = tableMeta.location + + // Run the query and print results when the query job is completed. + job, err := q.Run(ctx) + if err != nil { + err = fmt.Errorf("bigquery Run failed: %w", err) + m.logger.Error(err) + return events, err + } + + status, err := job.Wait(ctx) + if err != nil { + err = fmt.Errorf("bigquery Wait failed: %w", err) + m.logger.Error(err) + return events, err + } + + if err := status.Err(); err != nil { + err = fmt.Errorf("bigquery status error: %w", err) + m.logger.Error(err) + return events, err + } + + it, err := job.Read(ctx) + for { + var row []bigquery.Value + err := it.Next(&row) + if err == iterator.Done { + break + } + + if err != nil { + err = fmt.Errorf("bigquery RowIterator Next failed: %w", err) + m.logger.Error(err) + return events, err + } + + if len(row) == 4 { + events = append(events, createEvents(row, m.config.ProjectID)) + } + } + return events, nil +} + +func createEvents(rowItems []bigquery.Value, accountID string) mb.Event { + event := mb.Event{} + event.MetricSetFields = common.MapStr{ + "invoice_month": rowItems[0], + "project_id": rowItems[1], + "cost_type": rowItems[2], + "total": rowItems[3], + } + + event.RootFields = common.MapStr{ + "cloud.provider": "googlecloud", + "cloud.account.id": accountID, + "cloud.account.name": accountID, + } + + // create eventID for each current_date + invoice_month + project_id + cost_type + currentDate := getCurrentDate() + event.ID = generateEventID(currentDate, rowItems) + return event +} + +func getCurrentDate() string { + currentTime := time.Now() + return fmt.Sprintf("%04d%02d%02d", currentTime.Year(), int(currentTime.Month()), currentTime.Day()) +} + +func generateEventID(currentDate string, rowItems []bigquery.Value) string { + // create eventID using hash of current_date + invoice_month + project_id + cost_type + // This will prevent more than one billing metric getting collected in the same day. + eventID := currentDate + rowItems[0].(string) + rowItems[1].(string) + rowItems[2].(string) + h := sha256.New() + h.Write([]byte(eventID)) + prefix := hex.EncodeToString(h.Sum(nil)) + return prefix[:20] +} diff --git a/x-pack/metricbeat/module/googlecloud/billing/billing_integration_test.go b/x-pack/metricbeat/module/googlecloud/billing/billing_integration_test.go new file mode 100644 index 00000000000..0a4af506057 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/billing/billing_integration_test.go @@ -0,0 +1,24 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration +// +build googlecloud + +package billing + +import ( + "testing" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/metrics" +) + +func TestData(t *testing.T) { + config := metrics.GetConfigForTest(t, "billing") + config["period"] = "24h" + config["dataset_id"] = "master_gcp" + + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/googlecloud/billing/billing_test.go b/x-pack/metricbeat/module/googlecloud/billing/billing_test.go new file mode 100644 index 00000000000..885a2fc9786 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/billing/billing_test.go @@ -0,0 +1,18 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package billing + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetCurrentMonth(t *testing.T) { + currentMonth := getCurrentMonth() + _, err := strconv.ParseInt(currentMonth, 0, 64) + assert.NoError(t, err) +} diff --git a/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go b/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go index a91c26948d0..99f76e47c20 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go @@ -11,9 +11,10 @@ import ( "fmt" "testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/metrics" + "github.com/elastic/beats/v7/libbeat/common" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/metrics" ) func TestData(t *testing.T) { diff --git a/x-pack/metricbeat/module/googlecloud/constants.go b/x-pack/metricbeat/module/googlecloud/constants.go index 19b7e27c53d..7840b3bffc4 100644 --- a/x-pack/metricbeat/module/googlecloud/constants.go +++ b/x-pack/metricbeat/module/googlecloud/constants.go @@ -42,8 +42,9 @@ const ( ECSCloudRegion = "region" - ECSCloudAccount = "account" - ECSCloudAccountID = "id" + ECSCloudAccount = "account" + ECSCloudAccountID = "id" + ECSCloudAccountName = "name" ECSCloudInstance = "instance" ECSCloudInstanceKey = ECSCloud + "." + ECSCloudInstance diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index ad15833fa63..d74009bff14 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJzcXN1u2zgWvu9THPSm7SJ1gd27YjFAxsXOFGh2AyTtrUBRxzbXFKnyJ1736RckJVmyJVmWJSWZzs3EtsjvO//nUNJH2OL+M6ylXHOkXNrkDYBhhuNnePeH/xSW7mO458SspErfvQFQyJFo/AwxGvIGIEFNFcsMk+Iz/PYGAOCP5T2kMrEc3wCsGPJEf/ZffARBUjze0v0z+8x9rqTN8k+q11Wv5SRGrsuPi0tl/F+kpvJxA67iX87N7w+pFMxIxcQaUjSKUX26xTGWKh6rUS3+VvuqFZP7Fz6Mwi+2uN9JlTQunKIhCTFkqsUd1UnW1nttMB1t6WLZtyXm8N/b8zZQWzeRNvYW2fBtlJIsY2Kd//RtbfEOS7rLLcZsiAGFxiqBCayUTKHmQLf3X+GnRbVfnNCiMs2sqeI69rA6yaqPnKCrbboMKxe67mnPTGhDBMVG7R1v3rZYdcEVU7gjnJ/8oGvRroWriydKZhkmUbw3qCMqrTCLJ8LtMfz6jlyKdcsPauL8KqhMXWTwyxebQbwHs8EuYqcAM0K3aCaEmG/QG2Rpf5mdRDMKNaonTCIqFeoejE/8s5Xzv20aowK5Ar92uRVI4TlvpDbuW/f/LeZcx2oN4+wXcauPCvTRKUAR6v4qABHOJSUGE1jefw+Bg2mgVikUhu+BCZdSCir94GuyxsiwFEdF/90tCyupHOZc1EyARipFolsNKmF6O5FFkakcfenWcxoKju52ClG8hcwRKJlNCMlBCIi+/gdkhsrb6an8q6h2ihmcR1ZuK4MCjDwvrABremn5fc6Iq3SerNNxOmHUIPwpd/533m9/3MGGaIgRBSgrBBPrmz7OI9DspJrKfyiyp8mS5YkPhd2CHzmZtHNrwDhVvixRFvlyGE6NwswjR7cTyCdUl2GbTX798B36jlSq/SJ2WVCKacycpJFmv/rkwr6kXR430hAOJC34BybOp4O/L+Bxw3RebLuULgXfA3kijJOYhzz64y5vFWIMkUL6i/HvsCIp45XuoI2Y1ZiMSOwukDhUH279OTnpHckiJiZyJae3E435VMpEjnNtUZvg+8xokDvhMYHOCMWZ+Es7VSxpFECRscuAF2Rg5OQSKGc4kiQx4URQVuMxVuf7TZIEfi82uLAB3hiTHSfo9qh0BYg2IFUwsYuyIokU/nRK6mUm/UsWZx6ibKYq+Ya4vOh31MFM/nx8vP/04BUHQXMu4MsCnz618jYG02Av0eadYLwvobmvm+C3Q35uYVPOUBjt5DsM+GwyvhSezqTQ/RqTsQQbtjxjxrnEF40Bgf9jwYRBJcjxFGXmsFAAwrVCrXtIsUOG/STohfb12++FMx1kBe9d6H9c3sOKy50GZt5p8FgO8zIpgGQZZ9S3YKCNQpL6DPLh2EiOqPWpXoeRq5WtHfTyaY0j14aViXn1kCMzsgZ5Gj0U3OZURCO/Zpc0NIu05lGm5P/2LyVXUy61n3cKgX7k1z717B/cqrPOct1Qbu1QIRhUKRN+nuibsMfl/aeHh2/gJdMeinuHkGFYj033x13Ftaz28+pugP2daxyEB+P7cdcPocDdzLqmCi9WtMxQjAxzGbrFii9La7QhInFSq0NX0q43PpK24P3YWvT7w+lTT5zuFKwQWWZjbeMJ2pJ7Gz/Y+MJ2RNu4XG9YpHuw5R+OijehYRGO0G2UovbHDWO55NKmlhPDnjCkrdCo+j2021DIHcdkHerN28PfZTl6EyQQfpAgZ0+o9h5Ad1PC5TpElzFKUD8ccquBZr+wqB6sqBEoab0ni+2CLAoY5RcfgAkgNZV3xB+bRhXHKxQzbiAqIeeCRV8j1SG+05BZvQEUSSaZMDcQWwNCGtijqamwm4wV5SbTkJlUHZInrvXyexwIRGQ9xtnC7Rrh/eH44ENhYGHTFmJtvC6ilVnOo6rfl0cqk0aACp+SzuEwp+byCrXlZgH/kgoIJLhighUHrk2XaqzE40+HgFwTyKdUJp52giThTGAr//OSG7srPyevxsCYS6kb7bw6dlu+auXOoVgvpIEa1ZuZIOoNEGMwzZohwnfB2RY9F33jA5e/xs+RFLA045iiMKE9TSRqnz1iYujG319XRuQFPEhAQjeFSMI8mkphCBOu08XaBQvfdFY3U84cQk+MSknlbMYltDV7QlG7FijxfTMSBanlhmUcwbAUO4adNZm7SktQNkoS+8K0USy2he17SoUMyo18jkgZVbJIFBdZjD84nLW+8/1XvD9yWSMLd3WJO5/NDSnzwqTDFUfPlMYeCgD3zo3/AkmtQaJzhJhy23o8hB0zGxBSfHSxZ18TMEsuC5dHzOa1kyN+cxnHP6lM8LdBJtJXjuXMf1bzyIf+FxmAYxxxNAbVvEEwszFnehNaK4cCAgowMmP0Ig6t2puFyG4jNUKxN+yIBpslft4Q7+FOJmy1v6XbL8UPrunge4eyaTifkunns/35zBFWe6rkHPa89VXoijD0PObugBv73wLQgDa+nUsU7yOF665bhZ+L1Q3ESm7RldA7cSircrQDJhdzcm2ZYYxF8kihEw/+mudMQ+yxCfeoahmJwUC1aMSZKkjEbWeT2rswfN7EU+8oTgP4wPRzjuQLKvJPOYdji4v4OnuPqNSjcwlPnlS1kCLRVgWfCWEn3FPHdLjb0sjiJ/DTSkNqT6O0HzZLsWLrKNRX4x2QNuklbGUDIaAbIlwoWEkVJjH1KFDRQSkEvz8QkZxVS5GDZgtzbUcC18S38iYFV74PO7J7dJdeeVj3ahsxjSKZfRTtW7BXO5WZ4gbLLjlN0ya8sNI6b7//ojX1NexeclF6Da9Rq4LrSoFKEeBTrS8NOk4enrseuDbzFx7i1D3NgUk9wvt9vO94RXx4JSXJpdZdZghBMr2Rx2PbvncQ5VdfWZHkQXHWwVDL1CQndDaAP1Pg7kY9NLrNc9NT27DgrMhr+ObwtDNIB4pZ2HTie7EuFWwV0ahy7Y1toChfW1I7tF5SkfUU73R5CCtfeA8pyVjvwN/96EmfJqPv8wBfkBty0MDt/VegxN8hUpG681P3TYpmIxOPohC9by2BygSbcx6xZvPrSt7Ve195FBONSZS/N4hQino8ozySRjn9yl8t5AzN+5TI31sEMbo+e62IcBk5oAEtOfI9JBZdMZn/8nb5raNydMQORepofMLbQ5xel98qRfBxIOg+5ctFrTOkbMVo5LCm1ow7EDiSfDHgSklSFWKBoUGa3S9vGG5yF72wYTDfo9c1HD/Kf603wpC3JlxJpvGdBGOFldPgfp2Wcxsf7WlIX+tUns3wy2vIUEFs6RZNTRA5G6Cc6KId9SDAP/ad96tSUPQrJGR/4wXlBFj+TmEWnkwhJp8JkzTjLjz5ByefCC8eBZHW+EsT0vHMin/VgreTKK+PX7XVH+iMV3UXZSzhvNRw/tqGF6bk/wcAAP//nHNPVA==" + return "eJzcXF9v2zgSf++nGOxLt4fUBe7eisMCXRe3G6C5C5BsXwWKGtvcUKTKP/Gqn/5AUpIlWZJlW1KabZ9iW5zfb/5xZijpPTxh/hG2Um45Ui5t8gbAMMPxI7z9zX8Ka/cx3HNiNlKlb98AKORINH6EGA15A5CgpoplhknxEX55AwDw2/oeUplYjm8ANgx5oj/6L96DICm2Rbp/Js/c50rarPikfl39Wk5i5Lr6uLxUxn8iNbWPO3CV/wpuXj6kUjAjFRNbSNEoRvWxiDaWOh6rUa3+0fiqF5P7Fz6Mwi+eMN9LlXQunKIhCTFkrsUd1VnW1rk2mE62dLnsTxXm8P+n0z7QWDeRNvYe2fFtlJIsY2Jb/PSnxuIDnnRXeIzZEQMKjVUCE9gomUIjgD7d38I3iypfHdGKGedMbGvLtiOsSbIeI0foGkJ/DSuXth7pz1TqoJNO83XZvYVhLbXxv9bABOU2QVC4tZyoGzDkrxsgyZ9WmxSFuQEiElDSisQpH5WSatWJiolnyShGqRRmdymyUiEKM6kM+LW6xWVKet9g7fVGy7oPK8DtZ5AbMDssDV1Kj5FLsdVgZDcEIw3hndI3XJJ2BDVkP7pLK3kklVaYY8ejMs2swRkcbx1WPtPxmNCGCNrtd23hfYvVF9wwhXvC20ocXnRo4friiZJZhkkU5wZ1RL2Knwm3bfhNic7kPT9oqPNWUJk64/nlS2EQ596TBogdA8wIfUIzI8RCwGiQlf9ldhbLKNSonjGJqFSoRzA+2hh6Of/XpjEqF9B+7UoUSOE571zqK8K9x52bWK1hnH0nbvVJgT46AyhC3V8lIMK5pMRgAuv7P8KOxTRQqxQKw3NgwtUyJZVx8DXZYmRYipOi/8MtCxupHOZC1UyARipFonsdKmH6aSaPInMF+tqt5ywUAt1JCuVDD5kWKJnNCMlBCIhu/wcyQ+X99Fj/dVR7xQwuoysnyqAAI08rK8CaX1tezgl1VcGTDQbOIIwGhN/l3v/Ox+3XO9gRDTGiAGWFYGJ7MyZ4BJq9VHPFD0X2PNtmeRRDQVqII6eTfm4dGOfaLyuU5X55GU6NwiyjRycJ5DOq87Atpr9x+A4NbypVvordLijFPG5O0kiz72P2wrGk3T7uG4Giinf8AxMX0yHeV/C4Y7oott2WLgXPgTwTxknMwz769a7oUUPX4XKmuxj/CRuSMp6vThKzGpMJid0FEofqw62/JCe9J1nExEyh5Ox2ZDG/lTJR4Nxa1CbEPjMa5F54TKAzQnEh/tLOlUs6FVDu2FXCCzowcnYNVMNDSZKYcCLoPCOXL5Ik8Gsp4MwGeGdM1t6g+7PSFSD6gNTBxC7LiiRS+M0ZaZSbjC9ZnHuIqpmq7TfE7Yteog5u8vvj4/2HB284CJZzCV+W+PSxl/cxmAd7hbboBOO8gua+7oLfD/mllU05Q2G00+9lwBfT8bnwdCaFHteYTKXYIPKEGxca7x4B8n+tmDCoRM8gcLG0UALCrUKtR2hxQIfjNOiVdvvl1zKYDrqCn13qf1zfw4bLvQZm3mrwWA7zMimAZBln1LdgoI1Ckvod5F3bSVrUxlSvl5FrlK0D9IppjSPXh5WJZe1QIDOyAXkeO5TcljREJ7+eqTzNIq15lCn5V/6j7NWUS+3nnUKgH/n1Tz3HJ7f6rLNaN5Rbe1QIBlXKhJ8n+ibscX3/4eHhC3jN9Kfi0SnkMqxt1/16Vwstq/28ehjg+OCaBuHB+b7ejUMocL+wranCsw0tMxQTw1yHbrEWy9IabYg/PWxBV9Judz6T9uB931v0+7sijiNxvlOw6tTRxtrGM7Ql9zZ+sPGZ7Yi2cbXeZZnuwVZ/OCrehS7LcIQ+RSlqf9wwVUiubWo5MewZw7YVGlUvQzuBQu45JttQb346/F2VozdBA+EHCXL2jCr3AIabEi63IbtMUYKGo97cIGj2HcvqwYoGgYrWz2T1tCKrEkb1xTtgAkjD5AP5x6ZRLfBKw0ybiCrIhWLR10hNiG81ZFbvAEWSSSbMDcTWgJAGcjQNEw6TsaISMg+ZWc0heeJaLy/jQCAi2ynOFj5tEX4+HB+8Kx0sCO0h1sfrLFqZ5Tyqx311pDJrBqjxqegcDnMaIa9QW25W8B+pgECCGyZYeeDadanGWj7+cEjIDYV8SGXiaSdIEs4E9vI/rbmpu/JT+upMjIWWhtEua2Mn8lUbdwnDeiVdaFG9Wwii3gExBtOsGyL8ITh7Qs9F3/jE5a/xcyQFLM04pihMaE8TidrvHjExdOdv7Kwy8goeJCChu1IlYR5NpTCECdfpYuOClW8668KUc4fQE/v73JzPuA1ty55RNK4FSnzfjERBarlhGUcwLMWBYWdD567SEpRNsol9ZtooFtvS9z2lUgeVIL9HpIwqWW4UZ3mMPzhctL7z/Vect0LWyDJc3cZdzOYuKfPCpMMVRy+0jT2UAO5dGP8NNrUOjS6RYiqxzXwIe2Z2IKR473JP3lAwS85Lly1my/pJi99SzvFvKhP85SIXGavHaua/qHsUQ/+zHMAxjjgag2rZJJjZmDO9C62VQwEBBRiZMXoWh17rLUJkv5MaoZQNe6LBZomfN8Q53MmEbfJP9Olz+YNrOvjRqWwezsdkxsXseD5LpNWRJjmFvWh9FboiDD2PpTvgzv63BHRBG9/PJYrzSOF26Fbhl2J1A7GST+hK6L04lFUF2gsmF0ty7ZlhTEWyZdCZB3/dc6ZL/LEL96RmmYjBhWbRiAtVkIhPg03q6MLwZTeeZkdxnMAv3H5OkfyBivxjzuHY4iy+zt8jKvXkXMKTJ3UrpEi0VSFmQtoJ99QxHe62NLL8CXyz0pDG0yj9h81SbNg2CvXVdAekXXYJomwgBHRHhEsFG6nCJKaZBWo2qJTg5YdnDE+YpdyDFktzfUcC1+S3w6ODGaOXHdk9ukuvPKx7tY2YRpEsPor2LdirncrMcYPlkJ7maRN+sNK6aL//pjX1Nex+5KL0Gl6TVgXXlQK1IsBvtb40GDh5eOl64Nqdv4wQZ+55DkyaGd7L8bHjDfHulZQk53p3tUMIkumdbI9tx95BVFx9ZUVSJMVFB0M9U5OC0MkE/kKJexj1pdltmZue+oYFJ1XewLdEpJ1AeqGahU1nvhfrXMXWEU2q19HYLlTla9vUDq2XVGQ7xztdHsLKZ95DSjI2OvEPP3oypskY+zzAZ+SGHCzw6f4WKPF3iNS07uLUfZOi2cnEoyhV71tLoDLB7j2PWLP7fiXv+r2vPIqJxiQqXlhFKEU9nVO2tFFNv4p3WjlH8zElihdmQYyuz94qItyOHNCAlhx5DolFV0wWv/y0/jJQOTpihyJ1Mj7h7SHOrusvtSK4nQiGT/kKVesMKdswGjmsqTXTDgRami8HXClJ6kosMXRoc/jlDZe73FkvbLiYb+t1De1H+a+NRrjkrQlXkul8J8FUaeU4uV9n5cLHJ3sa0tc6tWcz/PIaMlQQW/qEpqGIgg1QTnTZjnoQ4B/7LvpVKSj6FRKSh7fFOQVWv1OYhSdTiClmwiTN/FvP/IOTz4SXj4JIa/ylCRl4ZsW/asH7SVTUx6/a6w90pqu6yzKWcF5ZuHhtww9m5P8HAAD//wY8Aj8=" } diff --git a/x-pack/metricbeat/module/googlecloud/loadbalancing/loadbalancing_integration_test.go b/x-pack/metricbeat/module/googlecloud/loadbalancing/loadbalancing_integration_test.go index 0c50b96f863..e95dede2a29 100644 --- a/x-pack/metricbeat/module/googlecloud/loadbalancing/loadbalancing_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/loadbalancing/loadbalancing_integration_test.go @@ -11,9 +11,10 @@ import ( "fmt" "testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/metrics" + "github.com/elastic/beats/v7/libbeat/common" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/metrics" ) func TestData(t *testing.T) { diff --git a/x-pack/metricbeat/module/googlecloud/metrics/integration.go b/x-pack/metricbeat/module/googlecloud/metrics/integration.go index 2e79918d84e..cfb8b38a828 100644 --- a/x-pack/metricbeat/module/googlecloud/metrics/integration.go +++ b/x-pack/metricbeat/module/googlecloud/metrics/integration.go @@ -18,9 +18,7 @@ func GetConfigForTest(t *testing.T, metricSetName string) map[string]interface{} credentialsFilePath, okCredentialsFilePath := os.LookupEnv("GCP_CREDENTIALS_FILE_PATH") config := map[string]interface{}{} - if !okRegion || region == "" { - t.Fatal("$GCP_REGION not set or set to empty") - } else if !okProjectID || projectID == "" { + if !okProjectID || projectID == "" { t.Fatal("$GCP_PROJECT_ID not set or set to empty") } else if !okCredentialsFilePath || credentialsFilePath == "" { t.Fatal("$GCP_CREDENTIALS_FILE_PATH not set or set to empty") @@ -31,7 +29,10 @@ func GetConfigForTest(t *testing.T, metricSetName string) map[string]interface{} "metricsets": []string{metricSetName}, "project_id": projectID, "credentials_file_path": credentialsFilePath, - "region": region, + } + + if okRegion { + config["region"] = region } if metricSetName == "metrics" { diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go b/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go index 7ccb71e93d3..e391f22b8f8 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go @@ -11,9 +11,10 @@ import ( "fmt" "testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/metrics" + "github.com/elastic/beats/v7/libbeat/common" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/metrics" ) func TestData(t *testing.T) { diff --git a/x-pack/metricbeat/module/googlecloud/storage/storage_integration_test.go b/x-pack/metricbeat/module/googlecloud/storage/storage_integration_test.go index a057d9642c1..035df4fdd61 100644 --- a/x-pack/metricbeat/module/googlecloud/storage/storage_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/storage/storage_integration_test.go @@ -11,9 +11,10 @@ import ( "fmt" "testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/metrics" + "github.com/elastic/beats/v7/libbeat/common" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/metrics" ) func TestData(t *testing.T) { diff --git a/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go index fe14aff5966..d309d78714b 100644 --- a/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go +++ b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go @@ -55,7 +55,8 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i ecs := common.MapStr{ ECSCloud: common.MapStr{ ECSCloudAccount: common.MapStr{ - ECSCloudAccountID: accountID, + ECSCloudAccountID: accountID, + ECSCloudAccountName: accountID, }, ECSCloudProvider: "googlecloud", }, diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index 266b2b0cd12..4dd52de0a0b 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -43,3 +43,13 @@ - "instance/cpu/usage_time" - "instance/cpu/utilization" - "instance/uptime" + +- module: googlecloud + metricsets: + - billing + period: 24h + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + dataset_id: "dataset id" + table_pattern: "table pattern" + cost_type: "regular"