From 33f6bccc89f5bbc5cb63f7b10059b3d33dd6d031 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 13 Jan 2020 20:10:47 +0100 Subject: [PATCH 01/15] Pubsub atomic commit --- .../docs/modules/googlecloud/pubsub.asciidoc | 24 +++++++++++++++++ .../module/googlecloud/pubsub/_meta/data.json | 0 .../googlecloud/pubsub/_meta/docs.asciidoc | 0 .../googlecloud/pubsub/_meta/fields.yml | 1 + .../module/googlecloud/pubsub/manifest.yml | 27 +++++++++++++++++++ 5 files changed, 52 insertions(+) create mode 100644 metricbeat/docs/modules/googlecloud/pubsub.asciidoc create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml diff --git a/metricbeat/docs/modules/googlecloud/pubsub.asciidoc b/metricbeat/docs/modules/googlecloud/pubsub.asciidoc new file mode 100644 index 00000000000..df751605e5a --- /dev/null +++ b/metricbeat/docs/modules/googlecloud/pubsub.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-googlecloud-pubsub]] +=== Google Cloud Platform pubsub metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json[] +---- diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc new file mode 100644 index 00000000000..e69de29bb2d diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml new file mode 100644 index 00000000000..8033a27f5ac --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml @@ -0,0 +1 @@ +- release: beta diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml new file mode 100644 index 00000000000..e7e3e032b71 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml @@ -0,0 +1,27 @@ +default: true +input: + module: googlecloud + metricset: stackdriver + defaults: + stackdriver: + service: pubsub + metrics: + - "pubsub.googleapis.com/subscription/ack_message_count" + - "pubsub.googleapis.com/subscription/backlog_bytes" + - "pubsub.googleapis.com/subscription/num_outstanding_messages" + - "pubsub.googleapis.com/subscription/num_undelivered_messages" + - "pubsub.googleapis.com/subscription/oldest_unacked_message_age" + - "pubsub.googleapis.com/subscription/pull_ack_message_operation_count" + - "pubsub.googleapis.com/subscription/pull_ack_request_count" + - "pubsub.googleapis.com/subscription/pull_message_operation_count" + - "pubsub.googleapis.com/subscription/pull_request_count" + - "pubsub.googleapis.com/subscription/push_request_count" + - "pubsub.googleapis.com/subscription/push_request_latencies" + - "pubsub.googleapis.com/subscription/sent_message_count" + - "pubsub.googleapis.com/subscription/streaming_pull_ack_message_operation_count" + - "pubsub.googleapis.com/subscription/streaming_pull_ack_request_count" + - "pubsub.googleapis.com/subscription/streaming_pull_message_operation_count" + - "pubsub.googleapis.com/subscription/streaming_pull_response_count" + - "pubsub.googleapis.com/topic/message_sizes" + - "pubsub.googleapis.com/topic/send_message_operation_count" + - "pubsub.googleapis.com/topic/send_request_count" From d7854717ad650882032b76fc87a54abf19f89e64 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 13 Jan 2020 20:16:45 +0100 Subject: [PATCH 02/15] Update module.yml to add the pubsub metricset to lightweight modules --- x-pack/metricbeat/module/googlecloud/module.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/metricbeat/module/googlecloud/module.yml b/x-pack/metricbeat/module/googlecloud/module.yml index c7de504341e..29dd239eec6 100644 --- a/x-pack/metricbeat/module/googlecloud/module.yml +++ b/x-pack/metricbeat/module/googlecloud/module.yml @@ -1,3 +1,4 @@ name: googlecloud metricsets: - compute +- pubsub From 478fed8b906aaf1766320afdd520453106ab2f5d Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 13 Jan 2020 20:19:16 +0100 Subject: [PATCH 03/15] Add metricset to the config.yml file of the module --- x-pack/metricbeat/module/googlecloud/_meta/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index 6ff566f37b3..ee7cf1f2b1c 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -1,6 +1,7 @@ - module: googlecloud metricsets: - compute + - pubsub zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" From b52de5abc01bbc7a396939e4d9932446ccb58586 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 14 Jan 2020 13:35:23 +0100 Subject: [PATCH 04/15] Updated docs --- metricbeat/docs/modules/googlecloud.asciidoc | 7 +++++- .../docs/modules/googlecloud/pubsub.asciidoc | 6 ----- metricbeat/docs/modules_list.asciidoc | 3 ++- .../module/googlecloud/_meta/docs.asciidoc | 2 +- .../googlecloud/pubsub/_meta/docs.asciidoc | 25 +++++++++++++++++++ 5 files changed, 34 insertions(+), 9 deletions(-) diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index d987df58a67..9d7563a72fb 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -43,7 +43,7 @@ Authentication and authorization in Google Cloud Platform can be achieved in man } ---- -Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role: +Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role (applies to all metricsets): * `Compute Viewer` (1 permission in use of 138): - `monitoring.metricDescriptors.list` @@ -82,6 +82,7 @@ metricbeat.modules: - module: googlecloud metricsets: - compute + - pubsub zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" @@ -96,5 +97,9 @@ The following metricsets are available: * <> +* <> + include::googlecloud/compute.asciidoc[] +include::googlecloud/pubsub.asciidoc[] + diff --git a/metricbeat/docs/modules/googlecloud/pubsub.asciidoc b/metricbeat/docs/modules/googlecloud/pubsub.asciidoc index df751605e5a..66798829a79 100644 --- a/metricbeat/docs/modules/googlecloud/pubsub.asciidoc +++ b/metricbeat/docs/modules/googlecloud/pubsub.asciidoc @@ -16,9 +16,3 @@ This is a default metricset. If the host module is unconfigured, this metricset For a description of each field in the metricset, see the <> section. -Here is an example document generated by this metricset: - -[source,json] ----- -include::../../../../x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json[] ----- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 6a627a66353..a01861ca8a0 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -89,7 +89,8 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.1+| .1+| |<> beta[] +.2+| .2+| |<> beta[] +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | diff --git a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc index 865d8160053..ff29a7dabdf 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc @@ -33,7 +33,7 @@ Authentication and authorization in Google Cloud Platform can be achieved in man } ---- -Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role: +Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role (applies to all metricsets): * `Compute Viewer` (1 permission in use of 138): - `monitoring.metricDescriptors.list` diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc index e69de29bb2d..9fa3ea4f50c 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc @@ -0,0 +1,25 @@ +PubSub Metricset to fetch metrics from https://cloud.google.com/pubsub/[Pub/Sub] topics and subscriptions in Google Cloud Platform. + +The `pubsub` Metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub[Stackdriver API]. The field names have been left untouched for people already familiar with them. + +No special permissions are needed apart from the ones detailed in the module section of the docs. + +- `pubsub.subscription.ack_message_count`: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. +- `pubsub.subscription.backlog_bytes`: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. +- `pubsub.subscription.num_outstanding_messages`: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. +- `pubsub.subscription.num_undelivered_messages`: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. +- `pubsub.subscription.oldest_unacked_message_age`: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. +- `pubsub.subscription.pull_ack_message_operation_count`: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.subscription.pull_ack_request_count`: Cumulative count of acknowledge requests, grouped by result. +- `pubsub.subscription.pull_message_operation_count`: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.subscription.pull_request_count`: Cumulative count of pull requests, grouped by result. +- `pubsub.subscription.push_request_count`: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. +- `pubsub.subscription.push_request_latencies`: Distribution of push request latencies (in microseconds), grouped by result. +- `pubsub.subscription.sent_message_count`: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. +- `pubsub.subscription.streaming_pull_ack_message_operation_count`: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.subscription.streaming_pull_ack_request_count`: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. +- `pubsub.subscription.streaming_pull_message_operation_count`: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count +- `pubsub.subscription.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. +- `pubsub.topic.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. +- `pubsub.topic.send_message_operation_count`: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.topic.send_request_count`: Cumulative count of publish requests, grouped by result. From 4b53e7b06f141b4539b9f40e793913e9fc794738 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 14 Jan 2020 13:35:50 +0100 Subject: [PATCH 05/15] Update yamls and fields --- x-pack/metricbeat/metricbeat.reference.yml | 1 + .../googlecloud/pubsub/_meta/fields.yml | 71 ++++++++++++++++++- .../modules.d/googlecloud.yml.disabled | 1 + 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 1669ac25985..49e611ac165 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -427,6 +427,7 @@ metricbeat.modules: - module: googlecloud metricsets: - compute + - pubsub zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml index 8033a27f5ac..64063c44497 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml @@ -1 +1,70 @@ -- release: beta +- name: pubsub + release: beta + type: group + description: Google Cloud PubSub metrics + fields: + - name: subscription + type: group + description: Suscription related metrics + fields: + - name: ack_message_count + type: long + description: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. + - name: backlog_bytes + type: long + description: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. + - name: num_outstanding_messages + type: long + description: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. + - name: num_undelivered_messages + type: long + description: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. + - name: oldest_unacked_message_age + type: long + description: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. + - name: pull_ack_message_operation_count + type: long + description: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + - name: pull_ack_request_count + type: long + description: Cumulative count of acknowledge requests, grouped by result. + - name: pull_message_operation_count + type: long + description: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + - name: pull_request_count + type: long + description: Cumulative count of pull requests, grouped by result. + - name: push_request_count + type: long + description: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. + - name: push_request_latencies + type: long + description: Distribution of push request latencies (in microseconds), grouped by result. + - name: sent_message_count + type: long + description: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. + - name: streaming_pull_ack_message_operation_count + type: long + description: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + - name: streaming_pull_ack_request_count + type: long + description: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. + - name: streaming_pull_message_operation_count + type: long + description: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count + - name: streaming_pull_response_count + type: long + description: Cumulative count of streaming pull responses, grouped by result. + - name: topic + type: group + description: Topic related metrics + fields: + - name: streaming_pull_response_count + type: long + description: Cumulative count of streaming pull responses, grouped by result. + - name: send_message_operation_count + type: long + description: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + - name: send_request_count + type: long + description: Cumulative count of publish requests, grouped by result. diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index 9f5408ca352..0dc8ec2d1af 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -4,6 +4,7 @@ - module: googlecloud metricsets: - compute + - pubsub zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" From c8fc8169e0cc21851586020c2a2763c0de328355 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 14 Jan 2020 13:36:37 +0100 Subject: [PATCH 06/15] Add PubSub code to Stackdriver metricset --- .../metricbeat/module/googlecloud/fields.go | 2 +- .../stackdriver/metadata_services.go | 2 ++ .../stackdriver/metrics_requester.go | 24 ++++++++++++++++++- .../stackdriver/metrics_requester_test.go | 21 ++++++++++++++++ .../timeseries_metadata_collector.go | 5 +++- 5 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index 8d22aa84558..c99ff155f07 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJy8l7Fy2zwMx3c/BbbcffclD6Chi4c0Q9sMTVcfRUIya4rQgaB96tP3KEWqZFuK04u62ST4xw9/ApZ8DwdsMiiJSofaUTQbALHiMIO7x3YVtmkZnp2Sgri62wAwOlQBM8hR1AbAYNBsa7HkM/i0AQB43D5DRSY63AAUFp0JWbtxD15VOEn54FSOLrTbANLUmAHlP1HL69L4/FgjBuSH/4blq2f/RFcoyihR7zrBVofbD4QmCFa3x2uq6ig4Cj+3dipTMsV6tDoxfnJd2065L2F05tzMMY/1QZTXONmcSz4nNhYsLONJOXcRsCS6JDwWN0x1jWaXN4Jhpyl6uRrf53Lky5mAiZFPXlNlfQmtcJ8G8gZkj0slXaLVSh9QVoF7lb4Zb+i5Oq5yG4wB+Yhmp4kxLNZqKObuvMmuVvs1VjkyUAGt6pAEyLfV7ilI2k2fZ5p3ShnFOvtLJfUPQvyeTGel07ceRTlHWgka2D6/gOyVgA2gIzN6cQ1Yn367+iJuAw+qxJ3Yag7qvdwvSRAK4kT7aq/1EFCTN5f3N3S2DYeV+kd9/Chvk1K6lW6UUw4omKq5Ms5wqF4FJiXvWJ6+AdXIbT9en5me58RWcG1/UhJBD0JvG9QBrelQm+ENi4bxqGdGYxFgkvwzndq4diZ/fIG9CpAjeuDovfXl/7eMh0c5Ea81IRrtcYUH3sWUdHm6SUluzFd1he7jn3kDX//M+zvCgF7W9i7lADoiv4/qH3i2THY/++bZ/kEIovTBsD0ir/pS+jsAAP//l2hICA==" + return "eJzcWU9v47YTvedTzG13f0i8d+OHBRYuut1D2wDZ9GpQ5FhmTXFUcmjD++kLUpYs2ZYjx5GB9BZb5Mx7b/5aeYAVbqeQE+UGpaGg7gBYs8EpfPiWvoVZ/BoejeAFueLDHYBDg8LjFDJkcQeg0EunS9Zkp/DlDgDg2+wRClLB4B3AQqNRfpoePIAVBXZcTozI0Pj0GIC3JU6Bsr9R8u6r9v22jeDRTf7XfH3y7v50gSyUYHHRDaelH37Bbz1jMfy8pKIMjK3jh9J2zeSOQtn6tiN8J1yzynJNoXXnUMw2Hm09Cyux87DPeZ+xtsGFdrgRxhwdOGf0nOG2ceWoLFHNsy2jn0sKlk+er30ZsnnPgY6Q362kQtsckuHaDWRb4CWeo3QMrRRyhTwKuJ3pwfCanCvDKNFw6NGtUc0lOfRnuSoKmTlMspNs/whFhg5oAclq4wTIJrZL8hyfxr97kreLMrA2+qeI1t8I4o8ouhMyfqqhCGNICkYFs8dn4KVg0B5kcA4tmy1oG3tXTWIYcC9ynLMu+kBdivs5GoQFuYh2J6+24FGSVcfxazJb+9VI+SPevpRn0VKMSlXK0QcsHBV9NA7gUDkKmOi8wvL9T6ASXcrH0zVT49k4zTi2PtEJowWmlwWqAI2pUPLwgkRNeZQ9pXEWQMf5b7RJ51JN/vU7LIWHDNGCC9Zqm98PKQ+LvCE3VoVI1OsRBt5RlVR+qkqJavSzOoHu7Wdeg6+eea9D6NHy2NpFH0BrdJehuoFm55HVaMqQ+ZCNsJA+huwpZBfuoz5kjb3BO2kHw1NoPkQqaSYfY+jD0cYi5GpeoE9juC9Mw5vNLBTBCNZrBFlHamfdR1eWNgZVXu11X/efweE/AT37+4p7dUCh0Wt02wRg0kshE3JlKK9q4Cr4P4iFSSkPXv/Eeu8JtgO9IfRRTFYTMakBNA8+xZYqOmHuR29DMafAcVdS2uZ1MK4jsl8wG7A7MVHFMdgF98FDGfwS0KqStOV7yAKDJYYtcids52kE2zh5axqjhoCMQs/z5GMPfS7y6ybv1xzh4364fqrTqXLXQ6mP0UWEymDMvF3ZzaoxUo23mDRE9utNp6gd+mB4Ar/GFR0ULrTV9Y+MU1c9tnrt532z7UjxuSCVCCsUymiLfcwHaLZrRTdQ6mTT2+lzHuet4hqdveuAjhvMJM8ro+iXo4PzSxDMWJSnwcGzNXqFiYW/T60p3UnvIhzoojRYoOWkNihCn2ZCJlgu00vKpttO4IkAhVzWYgBZswVJloW2Hshi58IkvV5oO3MxBTCeBHSOXMyTOKZyvUbbuQtSWBBlicJBEQzr0iDEX0l+oNpxW7JSXzmaftGenc5CnemJTM2+cZH6f6Glo3oIXJQlaYm+0XaWFulse1CaTHVZxkEsjUb7uiXNs0NRxAXn5sPpqXb9GMv1PzCqTmg5bitpHHY7Hmw0L8GSfYg9ZtuRVqvLGuIBp1vlxgGzWyXE/yUp/PKqtBiqoENfkvVj9Y2jlKi8vRz0Gi5TqeXrfv3+iFev/N37TtSC7jRQN9z8MqP98t22yKjV2PtVpdCg/e+h941T+k+1ZyFXysVBesXLqJf/O/pvAAAA//9iGg+a" } diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go index d5956bb565c..f5f4cd2ef72 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go @@ -17,6 +17,8 @@ func NewMetadataServiceForConfig(c config) (googlecloud.MetadataService, error) switch c.ServiceName { case googlecloud.ServiceCompute: return compute.NewMetadataService(c.ProjectID, c.Zone, c.opt...) + case googlecloud.ServicePubsub: + return nil, nil default: return nil, errors.Errorf("service '%s' not supported", c.ServiceName) } diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index bb59d6fffc3..958ea479f50 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -7,6 +7,7 @@ package stackdriver import ( "context" "fmt" + "regexp" "sync" "time" @@ -51,11 +52,13 @@ type stackdriverMetricsRequester struct { func (r *stackdriverMetricsRequester) Metric(ctx context.Context, m string) (out []*monitoringpb.TimeSeries) { out = make([]*monitoringpb.TimeSeries, 0) + filter := r.getFilterForMetric(m) + req := &monitoringpb.ListTimeSeriesRequest{ Name: "projects/" + r.config.ProjectID, Interval: r.interval, View: monitoringpb.ListTimeSeriesRequest_FULL, - Filter: fmt.Sprintf(`metric.type="%s" AND resource.labels.zone = "%s"`, m, r.config.Zone), + Filter: filter, } it := r.client.ListTimeSeries(ctx, req) @@ -104,6 +107,25 @@ func (r *stackdriverMetricsRequester) Metrics(ctx context.Context, ms []string) return results, nil } +var serviceRegexp = regexp.MustCompile(`^(?P[a-z]+)\.googleapis.com.*`) + +// getFilterForMetric returns the filter associated with the corresponding filter. Some services like Pub/Sub fails +// if they have a region specified. +func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) { + f = fmt.Sprintf(`metric.type="%s"`, m) + + service := serviceRegexp.ReplaceAllString(m, "${service}") + + switch service { + case googlecloud.ServicePubsub: + return + default: + f = fmt.Sprintf(`%s AND resource.labels.zone = "%s"`, f, r.config.Zone) + } + + return +} + // Returns a GCP TimeInterval based on the provided config func getTimeInterval(windowTime time.Duration) (*monitoringpb.TimeInterval, error) { var startTime, endTime time.Time diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go new file mode 100644 index 00000000000..53c2ab78bf0 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stackdriver + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetFilterForMetric(t *testing.T) { + r := stackdriverMetricsRequester{config: config{Zone: "us-central1-a"}} + + s := r.getFilterForMetric("compute.googleapis.com/firewall/dropped_bytes_count") + assert.Equal(t, `metric.type="compute.googleapis.com/firewall/dropped_bytes_count" AND resource.labels.zone = "us-central1-a"`, s) + + s = r.getFilterForMetric("pubsub.googleapis.com/subscription/ack_message_count") + assert.Equal(t, `metric.type="pubsub.googleapis.com/subscription/ack_message_count"`, s) +} diff --git a/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go index bc1d39fba2a..90e217afd6d 100644 --- a/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go +++ b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go @@ -52,7 +52,6 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i ecs := common.MapStr{ ECSCloud: common.MapStr{ - ECSCloudAvailabilityZone: availabilityZone, ECSCloudAccount: common.MapStr{ ECSCloudAccountID: accountID, }, @@ -60,6 +59,10 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i }, } + if availabilityZone != "" { + ecs[ECSCloud+"."+ECSCloudAvailabilityZone] = availabilityZone + } + //Remove keys from resource that refers to ECS fields if s.timeSeries == nil { From 3ea4006df24450e53a1cd4b5b670b8298e9cedc8 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 14 Jan 2020 13:36:47 +0100 Subject: [PATCH 07/15] Add data json files for pubsub --- .../module/googlecloud/pubsub/_meta/data.json | 0 .../pubsub/_meta/data_subscription.json | 53 ++++++++++++++++ .../googlecloud/pubsub/_meta/data_topic.json | 61 +++++++++++++++++++ 3 files changed, 114 insertions(+) delete mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json new file mode 100644 index 00000000000..1503d6aa80f --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json @@ -0,0 +1,53 @@ +{ + "@timestamp": "2020-01-14T11:46:00.000Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "googlecloud": { + "labels": { + "resource": { + "subscription_id": "test-subscription" + } + }, + "pubsub": { + "subscription": { + "num_undelivered_messages": 0, + "oldest_unacked_message_age": 0, + "backlog_bytes": 0 + } + } + }, + "service": { + "type": "googlecloud" + }, + "cloud": { + "account": { + "id": "elastic-metricbeat" + }, + "provider": "googlecloud" + }, + "host": { + "name": "mcastro" + }, + "agent": { + "version": "8.0.0", + "type": "metricbeat", + "ephemeral_id": "b3b8d9da-0094-4d54-b1dc-4d5e103d38aa", + "hostname": "mcastro", + "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228" + }, + "ecs": { + "version": "1.4.0" + }, + "event": { + "dataset": "googlecloud.pubsub", + "module": "googlecloud", + "duration": 933264720 + }, + "metricset": { + "name": "pubsub", + "period": 300000 + } +} diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json new file mode 100644 index 00000000000..f85d8148782 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json @@ -0,0 +1,61 @@ +{ + "@timestamp": "2020-01-14T11:42:00.000Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "ecs": { + "version": "1.4.0" + }, + "cloud": { + "provider": "googlecloud", + "account": { + "id": "elastic-metricbeat" + } + }, + "metricset": { + "name": "pubsub", + "period": 300000 + }, + "event": { + "duration": 932060852, + "dataset": "googlecloud.pubsub", + "module": "googlecloud" + }, + "googlecloud": { + "labels": { + "resource": { + "subscription_id": "test-subscription" + } + }, + "pubsub": { + "topic": { + "message_sizes": { + "bucket_options": { + "Options": { + "ExponentialBuckets": { + "growth_factor": 4, + "scale": 1, + "num_finite_buckets": 16 + } + } + } + } + } + } + }, + "service": { + "type": "googlecloud" + }, + "host": { + "name": "mcastro" + }, + "agent": { + "ephemeral_id": "b3b8d9da-0094-4d54-b1dc-4d5e103d38aa", + "hostname": "mcastro", + "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", + "version": "8.0.0", + "type": "metricbeat" + } +} From 62a16f62b3caa92085634964537a3e985623d83f Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 14 Jan 2020 13:41:47 +0100 Subject: [PATCH 08/15] Add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9786d3152d7..bef589c8683 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -591,6 +591,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add a `system/service` metricset for systemd data. {pull}14206[14206] - Add cost warnings for the azure module. {pull}15356[15356] - Release elb module as GA. {pull}15485[15485] +- Add PubSub metricset to Google Cloud Platform module {pull}15536[15536] *Packetbeat* From 9e6408324f568be78f196e7063ccbd4e9ca7943e Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 14 Jan 2020 13:50:20 +0100 Subject: [PATCH 09/15] Reorder changelog --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 18bfb3e53fe..1f9c03ba9e6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -598,8 +598,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add cost warnings for the azure module. {pull}15356[15356] - Add DynamoDB AWS Metricbeat light module {pull}15097[15097] - Release elb module as GA. {pull}15485[15485] -- Add PubSub metricset to Google Cloud Platform module {pull}15536[15536] - Add a `system/network_summary` metricset {pull}15196[15196] +- Add PubSub metricset to Google Cloud Platform module {pull}15536[15536] *Packetbeat* From 02fc27025ad3312d063fe55650c2e733e5decb28 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 3 Mar 2020 12:40:25 +0100 Subject: [PATCH 10/15] Atomic rebase --- CHANGELOG.next.asciidoc | 2 +- metricbeat/docs/modules/googlecloud.asciidoc | 5 +++++ x-pack/metricbeat/metricbeat.reference.yml | 2 ++ x-pack/metricbeat/module/googlecloud/_meta/config.yml | 2 ++ x-pack/metricbeat/modules.d/googlecloud.yml.disabled | 2 ++ 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d38090335a3..b704cce6830 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -180,7 +180,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add DynamoDB AWS Metricbeat light module {pull}15097[15097] - Release elb module as GA. {pull}15485[15485] - Add a `system/network_summary` metricset {pull}15196[15196] -- Add PubSub metricset to Google Cloud Platform module {pull}15536[15536] - Add mesh metricset for Istio Metricbeat module {pull}15535[15535] - Add mixer metricset for Istio Metricbeat module {pull}15696[15696] - Add pilot metricset for Istio Metricbeat module {pull}15761[15761] @@ -210,6 +209,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `cloudfoundry` module to send events from Cloud Foundry. {pull}16671[16671] - Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269] - Align fields to ECS and add more tests for the azure module. {issue}16024[16024] {pull}16754[16754] +- Add PubSub metricset to Google Cloud Platform module {pull}15536[15536] *Packetbeat* diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index f7ff546b72e..b5e50678659 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -86,6 +86,7 @@ metricbeat.modules: - pubsub - loadbalancing zone: "us-central1-a" + region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false @@ -99,6 +100,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -107,6 +110,8 @@ The following metricsets are available: include::googlecloud/compute.asciidoc[] +include::googlecloud/loadbalancing.asciidoc[] + include::googlecloud/pubsub.asciidoc[] include::googlecloud/loadbalancing.asciidoc[] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 06b21802740..bf144e2e36f 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -474,7 +474,9 @@ metricbeat.modules: metricsets: - compute - pubsub + - loadbalancing zone: "us-central1-a" + region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index ee7cf1f2b1c..b33dfaa03d6 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -2,7 +2,9 @@ metricsets: - compute - pubsub + - loadbalancing zone: "us-central1-a" + region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index 0dc8ec2d1af..15304fdba00 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -5,7 +5,9 @@ metricsets: - compute - pubsub + - loadbalancing zone: "us-central1-a" + region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false From 3fb61bf700c7114b246650bfa019690326ceba88 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 3 Mar 2020 12:52:32 +0100 Subject: [PATCH 11/15] updated asciidoc --- metricbeat/docs/modules/googlecloud.asciidoc | 6 ------ metricbeat/docs/modules_list.asciidoc | 3 ++- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index b5e50678659..7928316598b 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -104,15 +104,9 @@ The following metricsets are available: * <> -* <> - -* <> - include::googlecloud/compute.asciidoc[] include::googlecloud/loadbalancing.asciidoc[] include::googlecloud/pubsub.asciidoc[] -include::googlecloud/loadbalancing.asciidoc[] - diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 3ac1af27bce..30efcd5e63a 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -107,8 +107,9 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.2+| .2+| |<> beta[] +.3+| .3+| |<> beta[] |<> beta[] +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | From a4974584185a0bab8c9b10b5479f8c34bc932b8b Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 3 Mar 2020 12:52:43 +0100 Subject: [PATCH 12/15] go modules changes --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index de1bb8b07f9..5b050f213fa 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 + github.com/elastic/beats v7.6.0+incompatible github.com/elastic/ecs v1.4.0 github.com/elastic/go-libaudit v0.4.0 github.com/elastic/go-licenser v0.2.1 diff --git a/go.sum b/go.sum index d95286d6c56..c4f38904244 100644 --- a/go.sum +++ b/go.sum @@ -209,6 +209,8 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/elastic/beats v7.6.0+incompatible h1:vUcDbZ/qsvox6pC/t/tSdSItlt1/pKbsKaHEVS4GWss= +github.com/elastic/beats v7.6.0+incompatible/go.mod h1:7cX7zGsOwJ01FLkZs9Tg5nBdnQi6XB3hYAyWekpKgeY= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/ecs v1.4.0 h1:BGIUwWJhThRO2IQxzm7ekV9TMUGwZoYyevT5/1xmMf0= From b915c60fcebd304c7ac1b94c5f378362daf859c8 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 3 Mar 2020 13:22:09 +0100 Subject: [PATCH 13/15] go mod tidy --- go.mod | 1 + go.sum | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/go.mod b/go.mod index 5b050f213fa..692fb21e323 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/bsm/sarama-cluster v2.1.14-0.20180625083203-7e67d87a6b3f+incompatible github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e // indirect github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e + github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 github.com/cloudfoundry-community/go-cfclient v0.0.0-20190808214049-35bcce23fc5f github.com/cloudfoundry/sonde-go v0.0.0-20171206171820-b33733203bb4 diff --git a/go.sum b/go.sum index c4f38904244..1e2e41fa0fa 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tT github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= @@ -131,6 +133,8 @@ github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e/go.mod h1:V284 github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e h1:Gbx+iVCXG/1m5WSnidDGuHgN+vbIwl+6fR092ANU+Y8= github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e/go.mod h1:AZIh1CCnMrcVm6afFf96PBvE2MRpWFco91z8ObJtgDY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -582,6 +586,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= From 72d70cafe59bbd3e8b8239eed6507a424e13ee77 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 3 Mar 2020 13:51:28 +0100 Subject: [PATCH 14/15] Another go mod tidy --- go.mod | 2 -- go.sum | 8 -------- 2 files changed, 10 deletions(-) diff --git a/go.mod b/go.mod index 692fb21e323..de1bb8b07f9 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,6 @@ require ( github.com/bsm/sarama-cluster v2.1.14-0.20180625083203-7e67d87a6b3f+incompatible github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e // indirect github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e - github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 github.com/cloudfoundry-community/go-cfclient v0.0.0-20190808214049-35bcce23fc5f github.com/cloudfoundry/sonde-go v0.0.0-20171206171820-b33733203bb4 @@ -54,7 +53,6 @@ require ( github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 - github.com/elastic/beats v7.6.0+incompatible github.com/elastic/ecs v1.4.0 github.com/elastic/go-libaudit v0.4.0 github.com/elastic/go-licenser v0.2.1 diff --git a/go.sum b/go.sum index 1e2e41fa0fa..d95286d6c56 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,6 @@ github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tT github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= @@ -133,8 +131,6 @@ github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e/go.mod h1:V284 github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e h1:Gbx+iVCXG/1m5WSnidDGuHgN+vbIwl+6fR092ANU+Y8= github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e/go.mod h1:AZIh1CCnMrcVm6afFf96PBvE2MRpWFco91z8ObJtgDY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -213,8 +209,6 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= -github.com/elastic/beats v7.6.0+incompatible h1:vUcDbZ/qsvox6pC/t/tSdSItlt1/pKbsKaHEVS4GWss= -github.com/elastic/beats v7.6.0+incompatible/go.mod h1:7cX7zGsOwJ01FLkZs9Tg5nBdnQi6XB3hYAyWekpKgeY= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/ecs v1.4.0 h1:BGIUwWJhThRO2IQxzm7ekV9TMUGwZoYyevT5/1xmMf0= @@ -586,8 +580,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= From b9527cbeea8d8e3e0fb6877d50dde2a722d52544 Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 4 Mar 2020 15:28:02 +0100 Subject: [PATCH 15/15] Update fields --- x-pack/metricbeat/module/googlecloud/fields.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index e726b128b29..61e4e93847a 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJzsW0tz2zgSvvtX9M3JVuIc9ubDViWqrSS1ycY1cXJVQWBLwhhEc4CmFc2vnwL4ECmJJGiLSmZKN5uP7q+b6O9rPPQaHnB7CyuilUapKU+uAFixxlu4fh+uwsxfhjsteEk2vb4CsKhROLyFBbK4AkjQSasyVmRu4T9XAADvZ3eQUpJrvAJYKtSJuw03XoMRKbZc3mixQO3CbQDeZngLtPgdJZeXmu83beQO7c2/6stH3909nSKLRLAY9YZV0sW/4LaOMY1/XlKa5YyNx/dT2zazspRnjautxLc+16ywXIXQeGc/mU08yjgWRmLrZpfzLmNNg0tlcSO0Pnigz2if4abxxFKWYTJfbBndXFJu+OjzlS9NZtXxQCuRH42kVJkVBMOVG1hsgdfYF9IhtEzIB+RJwJWmo+HVYy7LJ/kaFh3aR0zmkiy63lgTyhd6f5Adjfb/ebpAC7SEYLV2AmRCtGty7O/6vzsGbxtlzkqrP4W3fiKI9z7pVkj/XwVFaE1SMCYwu/sGvBYMyoHMrUXDegvKeO6qgogD7sQK56zSLlBjcX/zBmFJ1qMt06sMOJRkksPvV49s5R4mGj/i9KU885b8VylK2fuApaW0K4w9OJRNAsY7L7B8/AKUoQ3j8XjNVHg2VjFOnR/vhNEA03CCCkBTZih4GEhRXR5ZR2n0Amg5/0Cb8Fyoye+fYS0cLBAN2NwYZVavYsrDIG/ITlUhEtXjBIJ3UCWFn6JSfDa6ozqC7vSaV+OrNO9pCB0anjp33gfQI9pxqM6Qs35kFRpNIlkILYxULT+n6ks/kUjgXeVgZHu6Zs72S6+7rJ4BogtIE8zC59Ukcy0YjVRHW54WhLee1diqRd5sFYrXtyCFlrkOPUMY1Js1Ft2BxT9ydAwbUX7DstHLLP3Yeqb2/5RgIDesdOP2YaFUTxa+HYdx68FYdBkZhzeTcNczBvZuNib6e7ZBC4vcl0JRZP0daqwpyrqVeyhtMJC6prsvvX7ifMX4a/r874+MDBpWQr8L0fa5j4cQC6MJZWVpw+v5UkgmO/hWVC986MRJEfF0BA8fmjZ5Ol8qEzq4qFQO+NnnoJIjBsQtvhXycxpTT70ayiZcxUeuoJQP9/d3b74G2YBCN7zAUIXMHbJJF/ZTo65xljPGxbYG5W8fAx4DtiDJM2W6cFamukb/gixIIdf40md6XCBLS4Z9JCyzueVj4CNE67f7e0/HLrdeWsgCCrkGScZgMQ9eIG986yy1CsGYpBCki7hcxOUiLoPGfwlxUYbRGnHKtdsxvfM/p3seHrz9QzaG5iJGTgTVjbASQ3fjzfVTXhwLDNd/PPHF804c6zyZ/sYQYDwFPp0ER9LgeCIcRYXPJMMBX7sVrpguexD4EzvtopVy8S3fIfApIB9ps8fDi2qqT5nYVmPdNYcpM94HnYmFPrOM1arUkrJ97VoRw9vZ/5pjB8gUolXFH9JyUaxnmbso1rOBxIOBi2JF+BqnV1OsCo3WqlilOuFi0FhgP2/hZ7Q+xavT31CbLstFHaYuy0WjIcTCgMty0VE/9a7xv286VovOvD1bAcKVRed6ybondXFEHdjr46d3u0WhipLhxZIs3M/uYKlp40DxtStYZ3eqkAyILNNKhoMt4NiiSIGM3r7cZ729oPrPCjwtrNYhgZ7ASk3wYXWhVOZcuS8xMbXATpP7KqrzJP9oZF3QLHOPxg8ofGsnJ5wO8anb7eS4sL3jUYR0HgpyH9EOLwQ/ca4/KMNRbw9L8BgzffJ7mtn0kPCedqo5UnKnmXWOE9sRUjtOaKNl9hkiOyCxLLO5c3oeOuifK7KN8tXkwonvmi6eNSdqnvbeEVA4Pr1Bi8BoU2XCFKSiqjdfv37q2loe0Qw8DeW+IH3/3JDK3IVT+v3QTrMd71IiXmMS2PyFMpC6lztWb07Frl3gd8dCPrwqdu1TZXLGliBqsUVbRpcJV05D6y2zIsDL3OwyN7vMzQaN/yJb+eeiwF3n+v1zHAUa3JxNQKTF0epBGZqTAZwVPwFqNP+Us2NhEp+pNmhL+WodqLsD6evOU9nhR52B5BOrHrFZCqf/IeFfAQAA///d9YAm" + return "eJzsXE1z47gRvftX9M3jlEdzyM2V2qpZJ9mdykzWFXvmqgLBloQYBBigYa3216cAfoiURBKURO3Olm6WRKBfN9DvNT7o9/CKmwdYar2UyKV26Q0ACZL4ALc/hW/h0X8NT5LRQpvs9gbAoERm8QESJHYDkKLlRuQktHqAH24AAH56fIJMp07iDcBCoEztQ/jhPSiWYcvkTLIEpQ0/A9AmxwfQyX+RU/lVs32zD2fRzP5Sf32w7fbpDImljNioFkZwG9/AbixhFv8811nuCBuP74a23c3SaJc3vm0FvjVcj0XPlQuNNrvBbOIRyhJTHFs/dhnv6qzZ4UIYXDMp9x7o67Sv42bnqdF5juk82RDaOddO0cHnK1tSq2XHA61AflJcZ0ItIXRcmYFkA7TCPpf2oeWMvyJNAq7sOhpePedyN8loGLRo3jCdc23Q9vqaapfI3Ul20Nt/uyxBA3oBodfaCGgVvF1pS/5X/3fH5G2jdCSk+I353s8E8cUH3TDuP1VQmJSaM8IUHp++Aq0YgbDAnTGoSG5AKM9dlRNxwC1b4pxE1gVqLO6vvkNYaOPRluEVCixyrdL98atntrCvE80fdv5UfvQ9+VEpUtnbgIXRWZcbO3B0PgkYb7zA8ukX0DmaMB8P50yFZ20E4dTx8UYIFZAeDlABaMoIBQsDIarTI+9IjV4ALeM/63V4LuTkty+wYhYSRAXGKSXU8j4mPRTSWpupMoSjeJtA8PaypLBTZIqPRrdXB9CdX/NqfJXmHYfQoqKpY+dtgH5DMw7VBWLWj6xCIzVLEyaZ4qJl51x16WfNUvixMjCyPF0R5bup151WJ4DoAtIEk/i4qnQuGaHi4mDJ04Lw0bMaGZG4ZqlQNN8AZ5I7GWqGMKnXKyyqA4P/c2gJ1qwcw7LQy43+deOZ2n8owYBTJGTj5/1EqZ4sbFsK89aDMWhzrSzOJuGuEyb2djXG+mu2wR4S51OhSLL+CjW2K513K/dQ2GAgdE1zv/TaibMVY69p8x+/5lqhIsHkj8HbPvPxEGJhNKEsjV7Tar5gnLQZbBVVC+8bsZxFPB3Bw/tdK5fNF0KFCi4qlAN2djmo5IgBcYsvhfyaRtVLr4ayMVvxkS0o5eeXl6cPz0E2oNANLzC6Qmb32aQL+7lR1zjLFWOyqUH5nw8BjwFbkOSFIl0YK0Ndo3+nDXDGV3jnIz3OkYXRirwnxPO5oUPgI0TrPy8vno6tM15atAFkfAVcK4XFOjhBWvvSmUsRnFFpIUhXcbmKy1VcBjv/Q4iLUIRGsXPu3Y6pnf881fPw5O2fsjE0FzFzIqhuRC8xdDe+u37Ki2OB4fyPJ7543oljnaPpbwwBxlPg8SQ4kgbHE+EoKjyRDAdsbXe4YqrsQeBHVtpFKWXjS7594FNAPlBmj4cXVVSfM7CtwrprDVNGvA86aWLywjJWq1JLyna1a6kJPj7+qzl3QKtCtCr/Q1iuinVSd1fFOhlIPBi4KlaErXF6NcWu0GitilWqM24GjQX2+238jNaneHX6DrXpul3U0dV1u2g0hFgYcN0uOminPjX+66xjt+jCx7MVIFwatLaXrHtCF0fUgb0+ff5xuylUUTK8W2gDL49PsJB6bUHQrS1YZ3urUCtgeS4FDxdbwJJBloFWcnO3y3o7TvXfFTjOrdYlgR7HSk3wbnWhFOpSsS8xkW6BnSb2lVeXCf5Bz7qgGaIejR9Q+NZJTrgd4kO3Pcmx4XjHowjh3BfkPqId3gg+cq0/KMNRrYcleEw3ffJ7ntX0kPCed6k5UnKnWXWOE9sRUjtOaKNl9gSRHZBY4vncWjkPFfTvK7KN9JXahhvfNV2ctCZq3vbeElC4Pr1Gg0BoMqHCEqSiqg/Pz5+7jpZHFAPHodwVpG9fGlLpbLil3w/tPMfxNtOaVpgGNn8nFGT2bsvqzaXYrQ38bonx1/vi1D4TyhG2BFGyDZrSu5zZchlaH5kVDl7XZte12XVtNtj5H+Qo/1IUuK1cv32Jo0CF64sJCDc4Wj10jupsAB+LV4Aaxb92ZImp1EeqDdpot1wF6u5AWiHMXWJd0jB9rsvaTy55dsnIS9rWJXV/x9Upz67+4F0JI3ZcfcL46zxDG16dOn3WP7rMSUbiDQuZ8sNX9m69KaXXEtNlIbkft5/r3ef7wvfigRSleEOzCQD6bx5KvSxS97QdZ01MFhdWrPgNq+LBqRb02qF3bPY6Y7MKQP3DHQgFrDXMPcntsnljhleDca4sr8GWwcSwV9EGd2shd3YFqNJcC0X3kDgCpQk2SK1h63fDqdrIud2YdAi0TNHSPNjYQp+z5WlvS31cYig1yxei7qrpVJjrcKnLo1EO5U7KeTOz69fDJsrxhie1I9tX0lpJbdA6STP4pzbAIMUg5mWlfqipxQbXftiSbSsUHzKdBodTZKkUCrs8j4jZ+Q7YhiJ1kPTK+PTjvNS4emPf9YBOO5ghPEeOol1NDs6ugBFhlh8GB1+VFK8YvLD3xQrYtwkHwAZElkvMUFGx4k012qAJCSO+Cv9YombbGTzrYqlcnXRqJTe+WCMmlAWtsNVgFrZ3m8aMnwLFvjMao42fJ16mluINVastcBb2ppEZyJwkkUsEEhn2nPK2ot132Bsf8b/vbDIEZyrvaxPFVoPgRlciMGqWhBcfL1SdVVd/26lJukpLL8TlcfoxRVpxguALnIuL03Nl+smn659Aqg7EcloqqQ22GQ/WglagtHrvOWbTCq1IxxHijk+Xmhs7nl1qQvyN6xR/OGpaxEawvo5zoSlR3scZHPTtzZtc8ONWvy++6Ynr3u8kWtBWg/SClV8ihV19txR53vcm+yIUVf+979xxCv9dLJw2pMYL6QmbUcP/0er/AQAA//8JtEe4" }