diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b55ad88642f..2f1e4043572 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -345,6 +345,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Reference kubernetes manifests mount data directory from the host when running metricbeat as daemonset, so data persist between executions in the same node. {pull}17429[17429] - Add more detailed error messages, system tests and small refactoring to the service metricset in windows. {pull}17725[17725] - Stack Monitoring modules now auto-configure required metricsets when `xpack.enabled: true` is set. {issue}16471[[16471] {pull}17609[17609] +- Add aggregation aligner as a config parameter for googlecloud stackdriver metricset. {issue}17141[[17141] {pull}17719[17719] - Move the perfmon metricset to GA. {issue}16608[16608] {pull}17879[17879] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 81e2d066557..04c431efd59 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -17174,6 +17174,7 @@ GCP module + *`googlecloud.labels`*:: + -- @@ -17181,6 +17182,1244 @@ type: object -- +*`googlecloud.stackdriver.*.*.*.*`*:: ++ +-- +Metrics that returned from StackDriver API query. + + +type: object + +-- + +[float] +=== compute + +Google Cloud Compute metrics + + + + +*`googlecloud.compute.instance.firewall.dropped_bytes_count.value`*:: ++ +-- +Incoming bytes dropped by the firewall + +type: long + +-- + +*`googlecloud.compute.instance.firewall.dropped_packets_count.value`*:: ++ +-- +Incoming packets dropped by the firewall + +type: long + +-- + + +*`googlecloud.compute.instance.cpu.reserved_cores.value`*:: ++ +-- +Number of cores reserved on the host of the instance + +type: double + +-- + +*`googlecloud.compute.instance.cpu.utilization.value`*:: ++ +-- +The fraction of the allocated CPU that is currently in use on the instance + +type: double + +-- + +*`googlecloud.compute.instance.cpu.usage_time.value`*:: ++ +-- +Usage for all cores in seconds + +type: double + +-- + + +*`googlecloud.compute.instance.disk.read_bytes_count.value`*:: ++ +-- +Count of bytes read from disk + +type: long + +-- + +*`googlecloud.compute.instance.disk.read_ops_count.value`*:: ++ +-- +Count of disk read IO operations + +type: long + +-- + +*`googlecloud.compute.instance.disk.write_bytes_count.value`*:: ++ +-- +Count of bytes written to disk + +type: long + +-- + +*`googlecloud.compute.instance.disk.write_ops_count.value`*:: ++ +-- +Count of disk write IO operations + +type: long + +-- + +*`googlecloud.compute.instance.uptime.value`*:: ++ +-- +How long the VM has been running, in seconds + +type: long + +-- + + +*`googlecloud.compute.instance.network.received_bytes_count.value`*:: ++ +-- +Count of bytes received from the network + +type: long + +-- + +*`googlecloud.compute.instance.network.received_packets_count.value`*:: ++ +-- +Count of packets received from the network + +type: long + +-- + +*`googlecloud.compute.instance.network.sent_bytes_count.value`*:: ++ +-- +Count of bytes sent over the network + +type: long + +-- + +*`googlecloud.compute.instance.network.sent_packets_count.value`*:: ++ +-- +Count of packets sent over the network + +type: long + +-- + +[float] +=== loadbalancing + +Google Cloud Load Balancing metrics + + +[float] +=== https + +Google Cloud Load Balancing metrics + + +[float] +=== backend_latencies + +A distribution of the latency calculated from when the request was sent by the proxy to the backend until the proxy received from the backend the last byte of response. + + +*`googlecloud.loadbalancing.https.backend_latencies.count.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.backend_latencies.mean.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.backend_latencies.bucket_counts.value`*:: ++ +-- +type: long + +-- + + + + +*`googlecloud.loadbalancing.https.backend_latencies.bucket_options.Options.ExponentialBuckets.growth_factor.value`*:: ++ +-- +type: double + +-- + +*`googlecloud.loadbalancing.https.backend_latencies.bucket_options.Options.ExponentialBuckets.scale.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.backend_latencies.bucket_options.Options.ExponentialBuckets.num_finite_buckets.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.backend_request_bytes_count.value`*:: ++ +-- +The number of bytes sent as requests from HTTP/S load balancer to backends. + +type: long + +-- + +*`googlecloud.loadbalancing.https.backend_request_count.value`*:: ++ +-- +The number of requests served by backends of HTTP/S load balancer. + +type: long + +-- + +*`googlecloud.loadbalancing.https.backend_response_bytes_count.value`*:: ++ +-- +The number of bytes sent as responses from backends (or cache) to HTTP/S load balancer. + +type: long + +-- + +[float] +=== frontend_tcp_rtt + +A distribution of the RTT measured for each connection between client and proxy. + + +*`googlecloud.loadbalancing.https.frontend_tcp_rtt.count.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.frontend_tcp_rtt.mean.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.frontend_tcp_rtt.bucket_counts.value`*:: ++ +-- +type: long + +-- + + + + +*`googlecloud.loadbalancing.https.frontend_tcp_rtt.bucket_options.Options.ExponentialBuckets.growth_factor.value`*:: ++ +-- +type: double + +-- + +*`googlecloud.loadbalancing.https.frontend_tcp_rtt.bucket_options.Options.ExponentialBuckets.scale.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.frontend_tcp_rtt.bucket_options.Options.ExponentialBuckets.num_finite_buckets.value`*:: ++ +-- +type: long + +-- + + +[float] +=== backend_latencies + +A distribution of the latency calculated from when the request was sent by the proxy to the backend until the proxy received from the backend the last byte of response. + + +*`googlecloud.loadbalancing.https.internal.backend_latencies.count.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.internal.backend_latencies.mean.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.internal.backend_latencies.bucket_counts.value`*:: ++ +-- +type: long + +-- + + + + +*`googlecloud.loadbalancing.https.internal.backend_latencies.bucket_options.Options.ExponentialBuckets.growth_factor.value`*:: ++ +-- +type: double + +-- + +*`googlecloud.loadbalancing.https.internal.backend_latencies.bucket_options.Options.ExponentialBuckets.scale.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.internal.backend_latencies.bucket_options.Options.ExponentialBuckets.num_finite_buckets.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.internal.request_bytes_count.value`*:: ++ +-- +The number of bytes sent as requests from clients to HTTP/S load balancer. + +type: long + +-- + +*`googlecloud.loadbalancing.https.internal.request_count.value`*:: ++ +-- +The number of requests served by HTTP/S load balancer. + +type: long + +-- + +*`googlecloud.loadbalancing.https.internal.response_bytes_count.value`*:: ++ +-- +The number of bytes sent as responses from HTTP/S load balancer to clients. + +type: long + +-- + +[float] +=== total_latencies + +A distribution of the latency calculated from when the request was received by the proxy until the proxy got ACK from client on last response byte. + + +*`googlecloud.loadbalancing.https.internal.total_latencies.count.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.internal.total_latencies.mean.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.internal.total_latencies.bucket_counts.value`*:: ++ +-- +type: long + +-- + + + + +*`googlecloud.loadbalancing.https.internal.total_latencies.bucket_options.Options.ExponentialBuckets.growth_factor.value`*:: ++ +-- +type: double + +-- + +*`googlecloud.loadbalancing.https.internal.total_latencies.bucket_options.Options.ExponentialBuckets.scale.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.internal.total_latencies.bucket_options.Options.ExponentialBuckets.num_finite_buckets.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.request_bytes_count.value`*:: ++ +-- +The number of bytes sent as requests from clients to HTTP/S load balancer. + +type: long + +-- + +*`googlecloud.loadbalancing.https.request_count.value`*:: ++ +-- +The number of requests served by HTTP/S load balancer. + +type: long + +-- + +*`googlecloud.loadbalancing.https.response_bytes_count.value`*:: ++ +-- +The number of bytes sent as responses from HTTP/S load balancer to clients. + +type: long + +-- + +[float] +=== total_latencies + +A distribution of the latency calculated from when the request was received by the proxy until the proxy got ACK from client on last response byte. + + +*`googlecloud.loadbalancing.https.total_latencies.count.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.total_latencies.mean.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.total_latencies.bucket_counts.value`*:: ++ +-- +type: long + +-- + + + + +*`googlecloud.loadbalancing.https.total_latencies.bucket_options.Options.ExponentialBuckets.growth_factor.value`*:: ++ +-- +type: double + +-- + +*`googlecloud.loadbalancing.https.total_latencies.bucket_options.Options.ExponentialBuckets.scale.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.https.total_latencies.bucket_options.Options.ExponentialBuckets.num_finite_buckets.value`*:: ++ +-- +type: long + +-- + +[float] +=== l3.internal + +Google Cloud Load Balancing metrics + + +*`googlecloud.loadbalancing.l3.internal.egress_bytes_count.value`*:: ++ +-- +The number of bytes sent from ILB backend to client (for TCP flows it's counting bytes on application stream only). + +type: long + +-- + +*`googlecloud.loadbalancing.l3.internal.egress_packets_count.value`*:: ++ +-- +The number of packets sent from ILB backend to client of the flow. + +type: long + +-- + +*`googlecloud.loadbalancing.l3.internal.ingress_bytes_count.value`*:: ++ +-- +The number of bytes sent from client to ILB backend (for TCP flows it's counting bytes on application stream only). + +type: long + +-- + +*`googlecloud.loadbalancing.l3.internal.ingress_packets_count.value`*:: ++ +-- +The number of packets sent from client to ILB backend. + +type: long + +-- + +[float] +=== rtt_latencies + +A distribution of RTT measured over TCP connections for ILB flows. + + +*`googlecloud.loadbalancing.l3.internal.rtt_latencies.count.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.l3.internal.rtt_latencies.mean.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.l3.internal.rtt_latencies.bucket_counts.value`*:: ++ +-- +type: long + +-- + + + + +*`googlecloud.loadbalancing.l3.internal.rtt_latencies.bucket_options.Options.ExponentialBuckets.growth_factor.value`*:: ++ +-- +type: double + +-- + +*`googlecloud.loadbalancing.l3.internal.rtt_latencies.bucket_options.Options.ExponentialBuckets.scale.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.l3.internal.rtt_latencies.bucket_options.Options.ExponentialBuckets.num_finite_buckets.value`*:: ++ +-- +type: long + +-- + +[float] +=== tcp_ssl_proxy + +Google Cloud Load Balancing metrics + + +*`googlecloud.loadbalancing.tcp_ssl_proxy.closed_connections.value`*:: ++ +-- +Number of connections that were terminated over TCP/SSL proxy. + +type: long + +-- + +*`googlecloud.loadbalancing.tcp_ssl_proxy.egress_bytes_count.value`*:: ++ +-- +Number of bytes sent from VM to client using proxy. + +type: long + +-- + +[float] +=== frontend_tcp_rtt + +A distribution of the smoothed RTT (in ms) measured by the proxy's TCP stack, each minute application layer bytes pass from proxy to client. + + +*`googlecloud.loadbalancing.tcp_ssl_proxy.frontend_tcp_rtt.count.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.tcp_ssl_proxy.frontend_tcp_rtt.mean.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.tcp_ssl_proxy.frontend_tcp_rtt.bucket_counts.value`*:: ++ +-- +type: long + +-- + + + + +*`googlecloud.loadbalancing.tcp_ssl_proxy.frontend_tcp_rtt.bucket_options.Options.ExponentialBuckets.growth_factor.value`*:: ++ +-- +type: double + +-- + +*`googlecloud.loadbalancing.tcp_ssl_proxy.frontend_tcp_rtt.bucket_options.Options.ExponentialBuckets.scale.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.tcp_ssl_proxy.frontend_tcp_rtt.bucket_options.Options.ExponentialBuckets.num_finite_buckets.value`*:: ++ +-- +type: long + +-- + +*`googlecloud.loadbalancing.tcp_ssl_proxy.ingress_bytes_count.value`*:: ++ +-- +Number of bytes sent from client to VM using proxy. + +type: long + +-- + +*`googlecloud.loadbalancing.tcp_ssl_proxy.new_connections.value`*:: ++ +-- +Number of connections that were created over TCP/SSL proxy. + +type: long + +-- + +*`googlecloud.loadbalancing.tcp_ssl_proxy.open_connections.value`*:: ++ +-- +Current number of outstanding connections through the TCP/SSL proxy. + +type: long + +-- + +[float] +=== pubsub + +Google Cloud PubSub metrics + + +[float] +=== subscription + +Suscription related metrics + + +*`googlecloud.pubsub.subscription.ack_message_count.value`*:: ++ +-- +Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. + +type: long + +-- + +*`googlecloud.pubsub.subscription.backlog_bytes.value`*:: ++ +-- +Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. + +type: long + +-- + +*`googlecloud.pubsub.subscription.num_outstanding_messages.value`*:: ++ +-- +Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. + +type: long + +-- + +*`googlecloud.pubsub.subscription.num_undelivered_messages.value`*:: ++ +-- +Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. + +type: long + +-- + +*`googlecloud.pubsub.subscription.oldest_unacked_message_age.value`*:: ++ +-- +Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. + +type: long + +-- + +*`googlecloud.pubsub.subscription.pull_ack_message_operation_count.value`*:: ++ +-- +Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + +type: long + +-- + +*`googlecloud.pubsub.subscription.pull_ack_request_count.value`*:: ++ +-- +Cumulative count of acknowledge requests, grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.subscription.pull_message_operation_count.value`*:: ++ +-- +Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + +type: long + +-- + +*`googlecloud.pubsub.subscription.pull_request_count.value`*:: ++ +-- +Cumulative count of pull requests, grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.subscription.push_request_count.value`*:: ++ +-- +Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. + +type: long + +-- + +*`googlecloud.pubsub.subscription.push_request_latencies.value`*:: ++ +-- +Distribution of push request latencies (in microseconds), grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.subscription.sent_message_count.value`*:: ++ +-- +Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. + +type: long + +-- + +*`googlecloud.pubsub.subscription.streaming_pull_ack_message_operation_count.value`*:: ++ +-- +Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + +type: long + +-- + +*`googlecloud.pubsub.subscription.streaming_pull_ack_request_count.value`*:: ++ +-- +Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.subscription.streaming_pull_message_operation_count.value`*:: ++ +-- +Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count + +type: long + +-- + +*`googlecloud.pubsub.subscription.streaming_pull_response_count.value`*:: ++ +-- +Cumulative count of streaming pull responses, grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.subscription.dead_letter_message_count.value`*:: ++ +-- +Cumulative count of messages published to dead letter topic, grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.subscription.mod_ack_deadline_message_count.value`*:: ++ +-- +Cumulative count of messages whose deadline was updated by ModifyAckDeadline requests, grouped by delivery type. + +type: long + +-- + +*`googlecloud.pubsub.subscription.mod_ack_deadline_message_operation_count.value`*:: ++ +-- +Cumulative count of ModifyAckDeadline message operations, grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.subscription.mod_ack_deadline_request_count.value`*:: ++ +-- +Cumulative count of ModifyAckDeadline requests, grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.subscription.oldest_retained_acked_message_age.value`*:: ++ +-- +Age (in seconds) of the oldest acknowledged message retained in a subscription. + +type: long + +-- + +*`googlecloud.pubsub.subscription.oldest_retained_acked_message_age_by_region.value`*:: ++ +-- +Age (in seconds) of the oldest acknowledged message retained in a subscription, broken down by Cloud region. + +type: long + +-- + +*`googlecloud.pubsub.subscription.oldest_unacked_message_age_by_region.value`*:: ++ +-- +Age (in seconds) of the oldest unacknowledged message in a subscription, broken down by Cloud region. + +type: long + +-- + +*`googlecloud.pubsub.subscription.retained_acked_bytes.value`*:: ++ +-- +Total byte size of the acknowledged messages retained in a subscription. + +type: long + +-- + +*`googlecloud.pubsub.subscription.retained_acked_bytes_by_region.value`*:: ++ +-- +Total byte size of the acknowledged messages retained in a subscription, broken down by Cloud region. + +type: long + +-- + +*`googlecloud.pubsub.subscription.seek_request_count.value`*:: ++ +-- +Cumulative count of seek attempts, grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.subscription.streaming_pull_mod_ack_deadline_message_operation_count.value`*:: ++ +-- +Cumulative count of StreamingPull ModifyAckDeadline operations, grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.subscription.streaming_pull_mod_ack_deadline_request_count.value`*:: ++ +-- +Cumulative count of streaming pull requests with non-empty ModifyAckDeadline fields, grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.subscription.byte_cost.value`*:: ++ +-- +Cumulative cost of operations, measured in bytes. This is used to measure quota utilization. + +type: long + +-- + +*`googlecloud.pubsub.subscription.config_updates_count.value`*:: ++ +-- +Cumulative count of configuration changes for each subscription, grouped by operation type and result. + +type: long + +-- + +*`googlecloud.pubsub.subscription.unacked_bytes_by_region.value`*:: ++ +-- +Total byte size of the unacknowledged messages in a subscription, broken down by Cloud region. + +type: long + +-- + +[float] +=== topic + +Topic related metrics + + +*`googlecloud.pubsub.topic.streaming_pull_response_count.value`*:: ++ +-- +Cumulative count of streaming pull responses, grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.topic.send_message_operation_count.value`*:: ++ +-- +Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + +type: long + +-- + +*`googlecloud.pubsub.topic.send_request_count.value`*:: ++ +-- +Cumulative count of publish requests, grouped by result. + +type: long + +-- + +*`googlecloud.pubsub.topic.oldest_retained_acked_message_age_by_region.value`*:: ++ +-- +Age (in seconds) of the oldest acknowledged message retained in a topic, broken down by Cloud region. + +type: long + +-- + +*`googlecloud.pubsub.topic.oldest_unacked_message_age_by_region.value`*:: ++ +-- +Age (in seconds) of the oldest unacknowledged message in a topic, broken down by Cloud region. + +type: long + +-- + +*`googlecloud.pubsub.topic.retained_acked_bytes_by_region.value`*:: ++ +-- +Total byte size of the acknowledged messages retained in a topic, broken down by Cloud region. + +type: long + +-- + +*`googlecloud.pubsub.topic.byte_cost.value`*:: ++ +-- +Cost of operations, measured in bytes. This is used to measure utilization for quotas. + +type: long + +-- + +*`googlecloud.pubsub.topic.config_updates_count.value`*:: ++ +-- +Cumulative count of configuration changes, grouped by operation type and result. + +type: long + +-- + +*`googlecloud.pubsub.topic.message_sizes.value`*:: ++ +-- +Distribution of publish message sizes (in bytes) + +type: long + +-- + +*`googlecloud.pubsub.topic.unacked_bytes_by_region.value`*:: ++ +-- +Total byte size of the unacknowledged messages in a topic, broken down by Cloud region. + +type: long + +-- + +[float] +=== snapshot + +Snapshot related metrics + + +*`googlecloud.pubsub.snapshot.oldest_message_age.value`*:: ++ +-- +Age (in seconds) of the oldest message retained in a snapshot. + +type: long + +-- + +*`googlecloud.pubsub.snapshot.oldest_message_age_by_region.value`*:: ++ +-- +Age (in seconds) of the oldest message retained in a snapshot, broken down by Cloud region. + +type: long + +-- + +*`googlecloud.pubsub.snapshot.backlog_bytes.value`*:: ++ +-- +Total byte size of the messages retained in a snapshot. + +type: long + +-- + +*`googlecloud.pubsub.snapshot.backlog_bytes_by_region.value`*:: ++ +-- +Total byte size of the messages retained in a snapshot, broken down by Cloud region. + +type: long + +-- + +*`googlecloud.pubsub.snapshot.num_messages.value`*:: ++ +-- +Number of messages retained in a snapshot. + +type: long + +-- + +*`googlecloud.pubsub.snapshot.num_messages_by_region.value`*:: ++ +-- +Number of messages retained in a snapshot, broken down by Cloud region. + +type: long + +-- + +*`googlecloud.pubsub.snapshot.config_updates_count.value`*:: ++ +-- +Cumulative count of configuration changes, grouped by operation type and result. + +type: long + +-- + +[float] +=== storage + +Google Cloud Storage metrics + + + +*`googlecloud.storage.api.request_count.value`*:: ++ +-- +Delta count of API calls, grouped by the API method name and response code. + +type: long + +-- + + +*`googlecloud.storage.authz.acl_based_object_access_count.value`*:: ++ +-- +Delta count of requests that result in an object being granted access solely due to object ACLs. + +type: long + +-- + +*`googlecloud.storage.authz.acl_operations_count.value`*:: ++ +-- +Usage of ACL operations broken down by type. + +type: long + +-- + +*`googlecloud.storage.authz.object_specific_acl_mutation_count.value`*:: ++ +-- +Delta count of changes made to object specific ACLs. + +type: long + +-- + + +*`googlecloud.storage.network.received_bytes_count.value`*:: ++ +-- +Delta count of bytes received over the network, grouped by the API method name and response code. + +type: long + +-- + +*`googlecloud.storage.network.sent_bytes_count.value`*:: ++ +-- +Delta count of bytes sent over the network, grouped by the API method name and response code. + +type: long + +-- + + +*`googlecloud.storage.storage.object_count.value`*:: ++ +-- +Total number of objects per bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. + +type: long + +-- + +*`googlecloud.storage.storage.total_byte_seconds.value`*:: ++ +-- +Delta count of bytes received over the network, grouped by the API method name and response code. + +type: long + +-- + +*`googlecloud.storage.storage.total_bytes.value`*:: ++ +-- +Total size of all objects in the bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. + +type: long + +-- + [[exported-fields-graphite]] == Graphite fields diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index 19a741eb5d0..6e3d7e881d5 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -17,17 +17,23 @@ Note: extra GCP charges on Stackdriver Monitoring API requests will be generated This is a list of the possible module parameters you can tune: * *zone*: A single string with the zone you want to monitor like "us-central1-a". If you need to fetch from multiple regions, you have to setup a different configuration for each (but you don't need a new instance of Metricbeat running) + * *region*: A single string with the region you want to monitor like "us-central1". This will enable monitoring for all zones under this region. + * *project_id*: A single string with your GCP Project ID + * *credentials_file_path*: A single string pointing to the JSON file path reachable by Metricbeat that you have created using IAM. + * *exclude_labels*: (`true`/`false` default `false`) Do not extract extra labels and metadata information from Metricsets and fetch metrics onlly. At the moment, *labels and metadata extraction is only supported* in Compute Metricset. +* *period*: A single time duration specified for this module collection frequency. + [float] == Authentication, authorization and permissions. Authentication and authorization in Google Cloud Platform can be achieved in many ways. For the current version of the Google Cloud Platform module for Metricbeat, the only supported method is using Service Account JSON files. A typical JSON with a private key looks like this: [float] -==== Example Credentials +=== Example Credentials [source,json] ---- { @@ -62,7 +68,9 @@ Google Cloud Platform offers the https://cloud.google.com/monitoring/api/metrics If you also want to *extract service labels* (by setting `exclude_labels` to false, which is the default state). You also make a new API check on the corresponding service. Service labels requires a new API call to extract those metrics. In the worst case the number of API calls will be doubled. In the best case, all metrics come from the same GCP entity and 100% of the required information is included in the first API call (which is cached for subsequent calls). -A recommended `period` value between fetches is between 5 and 10 minutes, depending on how granular you want your metrics. GCP restricts information for less than 5 minutes. +If `period` value is set to 5-minute and sample period of the metric type is 60-second, then this module will collect data from this metric type once every 5 minutes with aggregation. +GCP monitoring data has a up to 240 seconds latency, which means latest monitoring data will be up to 4 minutes old. Please see https://cloud.google.com/monitoring/api/v3/latency-n-retention[Latency of GCP Monitoring Metric Data] for more details. +In googlecloud module, metrics are collected based on this ingest delay, which is also obtained from ListMetricDescriptors API. [float] === Rough estimation of the number of API Calls @@ -101,13 +109,21 @@ metricbeat.modules: - module: googlecloud metricsets: - compute + region: "us-central1" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + +- module: googlecloud + metricsets: - pubsub - loadbalancing zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false - period: 300s + period: 60s - module: googlecloud metricsets: @@ -130,6 +146,8 @@ The following metricsets are available: * <> +* <> + * <> include::googlecloud/compute.asciidoc[] @@ -138,5 +156,7 @@ include::googlecloud/loadbalancing.asciidoc[] include::googlecloud/pubsub.asciidoc[] +include::googlecloud/stackdriver.asciidoc[] + include::googlecloud/storage.asciidoc[] diff --git a/metricbeat/docs/modules/googlecloud/stackdriver.asciidoc b/metricbeat/docs/modules/googlecloud/stackdriver.asciidoc new file mode 100644 index 00000000000..16609f7b01e --- /dev/null +++ b/metricbeat/docs/modules/googlecloud/stackdriver.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-googlecloud-stackdriver]] +=== Google Cloud Platform stackdriver metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/googlecloud/stackdriver/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/googlecloud/stackdriver/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 6bdb22404f5..fe03abd62a3 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -109,9 +109,10 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.4+| .4+| |<> beta[] +.5+| .5+| |<> beta[] |<> beta[] |<> beta[] +|<> beta[] |<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index c843bc364d7..3e751957bd3 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -498,13 +498,21 @@ metricbeat.modules: - module: googlecloud metricsets: - compute + region: "us-central1" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + +- module: googlecloud + metricsets: - pubsub - loadbalancing zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false - period: 300s + period: 60s - module: googlecloud metricsets: diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index 5df057bba18..e717a98ee6d 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -1,13 +1,21 @@ - module: googlecloud metricsets: - compute + region: "us-central1" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + +- module: googlecloud + metricsets: - pubsub - loadbalancing zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false - period: 300s + period: 60s - module: googlecloud metricsets: diff --git a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc index ff4b03cb023..a02a1b80979 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc @@ -7,17 +7,23 @@ Note: extra GCP charges on Stackdriver Monitoring API requests will be generated This is a list of the possible module parameters you can tune: * *zone*: A single string with the zone you want to monitor like "us-central1-a". If you need to fetch from multiple regions, you have to setup a different configuration for each (but you don't need a new instance of Metricbeat running) + * *region*: A single string with the region you want to monitor like "us-central1". This will enable monitoring for all zones under this region. + * *project_id*: A single string with your GCP Project ID + * *credentials_file_path*: A single string pointing to the JSON file path reachable by Metricbeat that you have created using IAM. + * *exclude_labels*: (`true`/`false` default `false`) Do not extract extra labels and metadata information from Metricsets and fetch metrics onlly. At the moment, *labels and metadata extraction is only supported* in Compute Metricset. +* *period*: A single time duration specified for this module collection frequency. + [float] == Authentication, authorization and permissions. Authentication and authorization in Google Cloud Platform can be achieved in many ways. For the current version of the Google Cloud Platform module for Metricbeat, the only supported method is using Service Account JSON files. A typical JSON with a private key looks like this: [float] -==== Example Credentials +=== Example Credentials [source,json] ---- { @@ -52,7 +58,9 @@ Google Cloud Platform offers the https://cloud.google.com/monitoring/api/metrics If you also want to *extract service labels* (by setting `exclude_labels` to false, which is the default state). You also make a new API check on the corresponding service. Service labels requires a new API call to extract those metrics. In the worst case the number of API calls will be doubled. In the best case, all metrics come from the same GCP entity and 100% of the required information is included in the first API call (which is cached for subsequent calls). -A recommended `period` value between fetches is between 5 and 10 minutes, depending on how granular you want your metrics. GCP restricts information for less than 5 minutes. +If `period` value is set to 5-minute and sample period of the metric type is 60-second, then this module will collect data from this metric type once every 5 minutes with aggregation. +GCP monitoring data has a up to 240 seconds latency, which means latest monitoring data will be up to 4 minutes old. Please see https://cloud.google.com/monitoring/api/v3/latency-n-retention[Latency of GCP Monitoring Metric Data] for more details. +In googlecloud module, metrics are collected based on this ingest delay, which is also obtained from ListMetricDescriptors API. [float] === Rough estimation of the number of API Calls diff --git a/x-pack/metricbeat/module/googlecloud/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/_meta/fields.yml index 8bbae3bf4fe..41c1c09fad9 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/fields.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/fields.yml @@ -4,14 +4,23 @@ description: > GCP module fields: - - name: googlecloud.labels - type: object + - name: googlecloud + type: group fields: - - name: user.* + - name: labels type: object - - name: metadata.* - type: object - - name: metrics.* - type: object - - name: system.* + fields: + - name: user.* + type: object + - name: metadata.* + type: object + - name: metrics.* + type: object + - name: system.* + type: object + - name: "stackdriver.*.*.*.*" type: object + object_type: double + object_type_mapping_type: "*" + description: > + Metrics that returned from StackDriver API query. diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json index cd37490aa90..f1f9f8f386b 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json @@ -5,8 +5,8 @@ "id": "elastic-observability" }, "instance": { - "id": "9077240380975650630", - "name": "gke-observability-8--observability-8--bc1afd95-cwh3" + "id": "6889336735612324102", + "name": "gke-dev-oblt-dev-oblt-pool-83a8831b-kd53" }, "machine": { "type": "n1-standard-4" @@ -23,16 +23,24 @@ "compute": { "instance": { "disk": { - "read_bytes_count": 0, - "read_ops_count": 0, - "write_bytes_count": 0, - "write_ops_count": 0 + "read_bytes_count": { + "value": 0 + }, + "read_ops_count": { + "value": 0 + }, + "write_bytes_count": { + "value": 0 + }, + "write_ops_count": { + "value": 0 + } } } }, "labels": { "metrics": { - "device_name": "gke-observability-8-0--pvc-ad47fe58-7bcf-11e9-a839-42010a8401a4", + "device_name": "disk-4", "device_type": "permanent", "storage_type": "pd-standard" }, diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network_01.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network_01.json deleted file mode 100644 index 26c21d62295..00000000000 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network_01.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "@timestamp": "2020-01-08T16:04:00.000Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "cloud": { - "machine": { - "type": "f1-micro" - }, - "availability_zone": "us-central1-a", - "account": { - "id": "elastic-metricbeat" - }, - "provider": "googlecloud", - "instance": { - "id": "4503798379141677974", - "name": "instance-1" - } - }, - "ecs": { - "version": "1.2.0" - }, - "host": { - "name": "mcastro", - "hostname": "mcastro", - "architecture": "x86_64", - "os": { - "kernel": "5.4.3-arch1-1", - "platform": "antergos", - "version": "", - "family": "", - "name": "Antergos Linux" - }, - "id": "54f70115bae545cbac2b150f254472a0", - "containerized": false - }, - "agent": { - "hostname": "mcastro", - "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", - "version": "8.0.0", - "type": "metricbeat", - "ephemeral_id": "8b802033-b611-414b-bcaf-1aa19e5f5901" - }, - "event": { - "duration": 1397728251, - "dataset": "googlecloud.compute", - "module": "googlecloud" - }, - "metricset": { - "name": "compute", - "period": 300000 - }, - "googlecloud": { - "labels": { - "metrics": { - "loadbalanced": "false" - } - }, - "compute": { - "instance": { - "network": { - "received_bytes_count": 3846, - "sent_bytes_count": 1750, - "received_packets_count": 17 - } - } - } - }, - "service": { - "type": "googlecloud" - } -} diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network_02.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network_02.json deleted file mode 100644 index cc71eda5683..00000000000 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network_02.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "@timestamp": "2020-01-08T16:05:00.000Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "event": { - "dataset": "googlecloud.compute", - "module": "googlecloud", - "duration": 1398297740 - }, - "metricset": { - "name": "compute", - "period": 300000 - }, - "googlecloud": { - "labels": { - "metrics": { - "loadbalanced": "false" - } - }, - "compute": { - "instance": { - "network": { - "sent_bytes_count": 3977 - } - } - } - }, - "service": { - "type": "googlecloud" - }, - "ecs": { - "version": "1.2.0" - }, - "host": { - "containerized": false, - "name": "mcastro", - "hostname": "mcastro", - "architecture": "x86_64", - "os": { - "family": "", - "name": "Antergos Linux", - "kernel": "5.4.3-arch1-1", - "platform": "antergos", - "version": "" - }, - "id": "54f70115bae545cbac2b150f254472a0" - }, - "agent": { - "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", - "version": "8.0.0", - "type": "metricbeat", - "ephemeral_id": "8b802033-b611-414b-bcaf-1aa19e5f5901", - "hostname": "mcastro" - }, - "cloud": { - "account": { - "id": "elastic-metricbeat" - }, - "provider": "googlecloud", - "instance": { - "id": "4503798379141677974", - "name": "instance-1" - }, - "machine": { - "type": "f1-micro" - }, - "availability_zone": "us-central1-a" - } -} diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/compute/_meta/fields.yml index 01be5ebf386..5cbdfb3ea56 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/fields.yml +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/fields.yml @@ -9,54 +9,54 @@ - name: firewall type: group fields: - - name: dropped_bytes_count + - name: dropped_bytes_count.value type: long description: Incoming bytes dropped by the firewall - - name: dropped_packets_count + - name: dropped_packets_count.value type: long description: Incoming packets dropped by the firewall - name: cpu type: group fields: - - name: reserved_cores + - name: reserved_cores.value type: double description: Number of cores reserved on the host of the instance - - name: utilization + - name: utilization.value type: double description: The fraction of the allocated CPU that is currently in use on the instance - - name: usage_time + - name: usage_time.value type: double description: Usage for all cores in seconds - name: disk type: group fields: - - name: read_bytes_count + - name: read_bytes_count.value type: long description: Count of bytes read from disk - - name: read_ops_count + - name: read_ops_count.value type: long description: Count of disk read IO operations - - name: write_bytes_count + - name: write_bytes_count.value type: long description: Count of bytes written to disk - - name: write_ops_count + - name: write_ops_count.value type: long description: Count of disk write IO operations - - name: uptime + - name: uptime.value type: long description: How long the VM has been running, in seconds - name: network type: group fields: - - name: received_bytes_count + - name: received_bytes_count.value type: long description: Count of bytes received from the network - - name: received_packets_count + - name: received_packets_count.value type: long description: Count of packets received from the network - - name: sent_bytes_count + - name: sent_bytes_count.value type: long description: Count of bytes sent over the network - - name: sent_packets_count + - name: sent_packets_count.value type: long description: Count of packets sent over the network diff --git a/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go b/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go index ae9400fdeb3..a4d47d95dd6 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go @@ -11,11 +11,11 @@ import ( "testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/stackdriver" ) func TestData(t *testing.T) { - config := googlecloud.GetConfigForTest(t, "compute") + config := stackdriver.GetConfigForTest(t, "compute") metricSet := mbtest.NewFetcher(t, config) metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/module/googlecloud/compute/manifest.yml b/x-pack/metricbeat/module/googlecloud/compute/manifest.yml index 03b16dcd440..34210db8c0e 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/manifest.yml +++ b/x-pack/metricbeat/module/googlecloud/compute/manifest.yml @@ -6,16 +6,17 @@ input: stackdriver: service: compute metrics: - - "compute.googleapis.com/firewall/dropped_bytes_count" - - "compute.googleapis.com/firewall/dropped_packets_count" - - "compute.googleapis.com/instance/cpu/reserved_cores" - - "compute.googleapis.com/instance/cpu/usage_time" - - "compute.googleapis.com/instance/cpu/utilization" - - "compute.googleapis.com/instance/disk/read_bytes_count" - - "compute.googleapis.com/instance/disk/read_ops_count" - - "compute.googleapis.com/instance/disk/write_bytes_count" - - "compute.googleapis.com/instance/disk/write_ops_count" - - "compute.googleapis.com/instance/network/received_bytes_count" - - "compute.googleapis.com/instance/network/received_packets_count" - - "compute.googleapis.com/instance/network/sent_bytes_count" - - "compute.googleapis.com/instance/uptime" + - metric_types: + - "compute.googleapis.com/firewall/dropped_bytes_count" + - "compute.googleapis.com/firewall/dropped_packets_count" + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/disk/read_bytes_count" + - "compute.googleapis.com/instance/disk/read_ops_count" + - "compute.googleapis.com/instance/disk/write_bytes_count" + - "compute.googleapis.com/instance/disk/write_ops_count" + - "compute.googleapis.com/instance/network/received_bytes_count" + - "compute.googleapis.com/instance/network/received_packets_count" + - "compute.googleapis.com/instance/network/sent_bytes_count" + - "compute.googleapis.com/instance/uptime" diff --git a/x-pack/metricbeat/module/googlecloud/constants.go b/x-pack/metricbeat/module/googlecloud/constants.go index b5ded8bf9bc..19b7e27c53d 100644 --- a/x-pack/metricbeat/module/googlecloud/constants.go +++ b/x-pack/metricbeat/module/googlecloud/constants.go @@ -4,16 +4,14 @@ package googlecloud +import monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3" + const ( // ModuleName in Metricbeat ModuleName = "googlecloud" - // MinTimeIntervalDataWindowMinutes is the minimum time in minutes that we allow the user to specify when requesting past metrics. Less than 5 minutes - // usually return no results. - MinTimeIntervalDataWindowMinutes = 5 - - // MaxTimeIntervalDataWindowMinutes is the max time in minutes that we allow the user to specify when requesting past metrics. - MaxTimeIntervalDataWindowMinutes = 60 + // MonitoringMetricsSamplingRate (in second) refers to how frequent monitoring collects measurement in GCP. + MonitoringMetricsSamplingRate = 60 ) // Metricsets / GCP services names @@ -73,3 +71,53 @@ const ( LabelUser = "user" LabelMetadata = "metadata" ) + +// Available perSeriesAligner map +// https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.alertPolicies#Aligner +var AlignersMapToGCP = map[string]monitoringpb.Aggregation_Aligner{ + "ALIGN_NONE": monitoringpb.Aggregation_ALIGN_NONE, + "ALIGN_DELTA": monitoringpb.Aggregation_ALIGN_DELTA, + "ALIGN_RATE": monitoringpb.Aggregation_ALIGN_RATE, + "ALIGN_INTERPOLATE": monitoringpb.Aggregation_ALIGN_INTERPOLATE, + "ALIGN_NEXT_OLDER": monitoringpb.Aggregation_ALIGN_NEXT_OLDER, + "ALIGN_MIN": monitoringpb.Aggregation_ALIGN_MIN, + "ALIGN_MAX": monitoringpb.Aggregation_ALIGN_MAX, + "ALIGN_MEAN": monitoringpb.Aggregation_ALIGN_MEAN, + "ALIGN_COUNT": monitoringpb.Aggregation_ALIGN_COUNT, + "ALIGN_SUM": monitoringpb.Aggregation_ALIGN_SUM, + "ALIGN_STDDEV": monitoringpb.Aggregation_ALIGN_STDDEV, + "ALIGN_COUNT_TRUE": monitoringpb.Aggregation_ALIGN_COUNT_TRUE, + "ALIGN_COUNT_FALSE": monitoringpb.Aggregation_ALIGN_COUNT_FALSE, + "ALIGN_FRACTION_TRUE": monitoringpb.Aggregation_ALIGN_FRACTION_TRUE, + "ALIGN_PERCENTILE_99": monitoringpb.Aggregation_ALIGN_PERCENTILE_99, + "ALIGN_PERCENTILE_95": monitoringpb.Aggregation_ALIGN_PERCENTILE_95, + "ALIGN_PERCENTILE_50": monitoringpb.Aggregation_ALIGN_PERCENTILE_50, + "ALIGN_PERCENTILE_05": monitoringpb.Aggregation_ALIGN_PERCENTILE_05, + "ALIGN_PERCENT_CHANGE": monitoringpb.Aggregation_ALIGN_PERCENT_CHANGE, +} + +const ( + DefaultAligner = "ALIGN_NONE" +) + +var AlignersMapToSuffix = map[string]string{ + "ALIGN_NONE": ".value", + "ALIGN_DELTA": ".delta", + "ALIGN_RATE": ".rate", + "ALIGN_INTERPOLATE": ".interpolate", + "ALIGN_NEXT_OLDER": ".next_older", + "ALIGN_MIN": ".min", + "ALIGN_MAX": ".max", + "ALIGN_MEAN": ".avg", + "ALIGN_COUNT": ".count", + "ALIGN_SUM": ".sum", + "ALIGN_STDDEV": ".stddev", + "ALIGN_COUNT_TRUE": ".count_true", + "ALIGN_COUNT_FALSE": ".count_false", + "ALIGN_FRACTION_TRUE": ".fraction_true", + "ALIGN_PERCENTILE_99": ".percentile_99", + "ALIGN_PERCENTILE_95": ".percentile_95", + "ALIGN_PERCENTILE_50": ".percentile_50", + "ALIGN_PERCENTILE_05": ".percentile_05", + "ALIGN_PERCENT_CHANGE": ".percent_change", +} diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index 8fac4947787..6d629ce05c2 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJzsXUuT2zYSvvtX9M321nh82JtrK1WTyW7iWnsztTPJVQWCLQk7IMDgMYry67cAkBQp8QFSpPwo3WYkEv11A/i+bgCk3sEz7j/ARsoNR8qlTV8BGGY4foDXP/tP4d59DA+cmLVU2etXAAo5Eo0fIEFDXgGkqKliuWFSfIAfXgEA/Hz/AJlMLcdXAGuGPNUf/BfvQJAMGyZvOUmQa/81gNnn+AFk8j+kpviofn+9DatR3f6t+rj13sPVGRqSEkNG3aEY1fE36L02mMVfT2WWW4O1y49D22xmo6TNa582At/orvvQculC7Z7jYNbxMKENERQbX3YZ72qs3uCaKdwRzk8u6Gu0r+F646mSeY7pKtkb1CsqrTCt15e2uBSbjgsagfwoqMyY2IBvuDQDyR7MFvtcOoWWE/qMZhFwRdPR8Koxl9tFekOhRvWC6YpKhbrX11TahB8PslZv/2OzBBXINfhWKyMghfd2K7Vx37q/OwZvE6U1jLO/iGt9JohPLuiKUPdfCYVwLikxmML9w29gtsQA00CtUigM3wMTjrtKJ+KAa7LBlWFZF6ixuH9zDcJaKoe2CC8ToJFKkZ72XzWymX5eaPyQ+afyvWvJ9UqYys4GrJXMutw4giPzRcA44wHLx19B5qj8eGyfMyWenWIGl46PM2JQgJHDAQqAloyQtzAQomp65B1ToxdAw/gvcuev83Py98+wJRoSRAHKCsHE5iZmegg0O6mWmiEU2csCgncyS4KdMFNcNLq9akE3v+ZV+ErNm4ZQozBLx87ZAPmCahyqC8SsH1mJhkuSJoQTQVnDzlx56SdJUvixNDAyPd0akx9Pve5pdQaILiB1MImLq0hXnBgUlLWmPA0Id47VjGKJracK4fY9UMKp5T5n8IN6t8WQHSj8w6I2sCNFHxaJXq7kn3vH1O6fAgxYYRivfX06Ucorg21t/Lh1YBTqXAqNt4tw1xkD+1CNkf6cbbCFxLqpECZZf4Ya25TMu5V7KGwwELq6uV977cTZirFXt/nPP3MpUBhG+I/e2z7z8RBiYdShbJTcme1qTaiRavCuqFz41IimJOLqCB4+bVrYbLVmwmdwUaEcsHPMQQVHDIhbfCrkahpRlV41ZSO65CMdKOWXp6eH949eNiDohhMYWSLTp2zShX1u1BXOomJM9hUo93Ub8BiwgSQvFOlgrAh1hf6NVEAJ3eJbF+lxjqyVFMZ5Ymi+UqYNfIRo/ffpydGxtspJi1SAhG6BSiEw1MEJmp1LnSln3hmRBkG6istVXK7iMtj4VyEuTBhUgsy5djsmd/5+sufhwds/ZGNoLmLkRFDdiFZi6G58c/2UF8cCw/M/nvjieSeOdSbT3xgCjKfA6SQ4kgbHE+EoKjyTDAdsHVa4YrLsQeATM+2QSun4lO8U+BKQW9Ls8fCikuo5A9tIrLtqmCLifdCNNIRfWMYqVWpI2bF2baSBu/t/18cOSBFEq/Tfh+WqWGc1d1Wss4HEg4GrYkXYGqdXS6wKjdaqWKWacTFoLLAvt/AzWp/i1ekb1KbrclFHU9flotEQYmHAdbmo1U61a/z3247Vogtvz5aAcKNQ616y7gldHFF79vr46cfDolBJyfBmLRU83T/AmsudBmZe68A6h1OFUgDJc86oP9gC2igkGUjB92+PWe/Iqf6zAtPcahwS6HGs0ATnVhdKJi4V+wKTkQ2wy8S+9OoywW/1rAuaMqZH4wcUvrGT40+HuNAddnK0395xKHw4TwW5j2iHF4In1vqDMhx197AEj2mmT37nqaaHhHfeUnOk5C5TdY4T2xFSO05oo2X2DJEdkFhD85XWfOUz6C8rsrXpy6X2J74rujirJqqf9j4QkD8+vUOFYFBlTPgSpKSq94+Pn7q2lkckA9NQHgvS759rUmm1P6XfD22e7XidSWm2mHo2f8MEZPrtgdXrpdhr7fldG0Kfb8KufcaENdgQRE72qArvcqKLMrTaMgsOXmuza212rc0GG/9KtvIvRYGHzPX3z3EUKHB3MQGhCkerh8xRzAbwPjwCVEv+pTXaEJG6SDVBK2k3W0/dHUhLhLlNtE1qpuc6rP1gk0ebjDykrW1StTctT3m01T/OFd9j0/ITQp9XGWr/6NT5o/7eZpYTw14wyJTrvqJ17UwJueOYboLk3h3+r1afb4Lv4YIUOXtBtfcA+k8ecrkJU/e8FWdpCA8HVjT7C8vkwYoG9MqhN+T2+ZbclgCqL94CE0Aa3dwzuW22qo3wsjPmmuUV2CKY6NcqmuBea8it3gKKNJdMmBtIrAEhDezRNLqt3w0rKiNzu7FoF0ieojYrb+MAfUU25z0tdbdBn2oWD0S9LYdTMNfhUpdHoxzKLeer+syuHg9baI7XPKkcOTyS1pjUCrXl5hb+JRUQSNGLeZGpt92qsca17w9k2wjF+0ym3uEUScqZwC7PI2I23wbbUKRaSa+ITz/OS/WrM/ZNd+iynenDM7EX9XZxcHoLxBjM8nZw8Jvg7Bm9F/omVMDuHr8BrIBlOccMhQkVbypRe01IiKFb/2KJim1v4VGGUrnc6ZSC712yZggTGqTAxg23fnm3bky5IRDWnVEpqdw4cTK1YS8oGvcCJX5tGomCzHLDco5gWIY9u7yNaPdt9sZH/KejRQbvTOl9ZSIsNTCqZCkCo0aJf/DxQtlZefS3OTWNLKelE+JiO31KkhZ2EFyCc3FxeixNP7jp+h1IVUssl6WSymCT8WDHzBaEFO8cx+wboWXpOEI88ulSY+PIs0sNiH9QmeIPk4ZFbASr4zgXGhLFeZxRne58XXE0BtWlaC63CWd6G8ogZx+CfTAyZ3QU+s4eW9iF3VZqhNKqP1Rk89SvASR7+CxTtt7f0eefygvOqa0nj8pzvT11I25uxnuyLGVGdsMQ6qI0VegSKfQeXK5Cba1PSygTCuxuL1bJfqVw0/7KoS/lzw0kSj6jS3134pAaBZxTVhMu42XHusJc7h114mKLbu3rPVNGXxvimbpiJuwTu0IjLp75IT73FpHRCd2XEpJm9n9KyxPlZMi9ryIhP/U2bAeM8tSN7hWVekYvwhvp6pGvtuKZCBtmt/C0ZRqYdsW/T9aKS+APKw2pv6WuGzqVYs02q5AdzbGr19YXwYgNrgDdEuEme/XIf3Oe1+Jeue/t+xcADHVFqSkXoLCuBfdzuOtw6D5ndNrG15O79cwtr2+wUNIo0gsu+voS6ZtdHZn3lSl9EVompf9qkuGiJP7usuBz/Po6k8lzPJpJ388T9Zqce+n0It+zov/llP170fCxQ6aiV0FyvZXH0Y49uFLcfaaEF+xyoXWQjqWCwpVBDrw49/XjnUoTS5+y6aqTB8PcQLbsjBrAODG0wmaLHfsZG8w6lpliGY1qYvi+HS1413nu0P/GhD9znir2gvWTuWOPJMb/rsGhIJKqSaFzHY98DC2PPB9JchatLv0POfaXAbHPov2E3JDDULh7+AiU+DMTte539OC+ydBsZertl2MgPK1NZYrtkkqs2f51psf1E518lRCN6Sr8pMaKUIp6jnlxFIdq1cmfHg5j3U9oUfyYByTo6t6NIsJJfcABWnLke0gtugSwuPLu/lNPtudcOiSWM3gS3uzv+vL+Uy1lPeaf/r2xIrw6R8rWjK4cysyauUrzo2iXC0sZSeuBK623RPBwfLztFdvTB1jkS9Yne3r0ivXjF3KfO+vg+IDNom60vlN8LuI4Je7zerYY0TO8usOnT7Uj/L5hDTmq4nmfRggKP4Byossi8YVwi65UPDx7Kyj6FlKyv/EhcqGrrlOYh0cXiClWXkmWc0dA/sH7F8LLJwakNf7WlPQ81BDeDOKL465fNPg2xvjBkTnS9jIbJpxXvcrCW06+so79fwAAAP//qGqEJg==" + return "eJzsXV9z2zYSf8+n2OlLkpvEebi3zE1nXOeuzVxy9ZzdvnJAcCXhDAIs/lhVP/0NAJIiJZECKZKOM05faonE/nax+O0fgNR7eMDdR1hLueZIubTZKwDDDMeP8Ppn/yncuI/hlhOzkip//QpAIUei8SOkaMgrgAw1VawwTIqP8OMrAICfb24hl5nl+ApgxZBn+qP/4j0IkuOhSPfP7Ar3uZK2KD9p3te8l5MUua4/rm6V6f+QmsbHh/c3x7Aa1dXfWl91jrO/K0dDMmLIqDsVo3r4jXqnDeZx91X3/KANoQ+ZYo9OyfDfD+ftFT5IwreZtKmfvRPfJjkpCibW5aU/tAY/4Q3Vv6/BCGA2xIBCY5XADFZK5nDnEH/yiOH69jP8YVHtro40ozIvrGnCOnTGto5NdzoC13LwmzByNVGRbsSENkRQPDk7h8K7BmsOuGIKt4Tzowv6Bu0buDl4pmRRYJakO4M6odIKc/VIuD2E35bIpVh3XNAy52dBZc7EGvzwlTBId2A22KfYMcCC0Ac0M0IsBUSDrP2vsLPMjEKN6hGzhEqFOkLjo+XZqfN/bJ6iArkCP3YtCqTwOm+kNu5b9/8d7tzGag3j7C/iRp8U6L2bAEWo+6sCRDiXlBjM4Ob2t8AbTAO1SqEwfAdMOCavVImDr8kaE8NynBT9b25YWEnlMJemZgI0Uiky3elQGdMPM3kUmWuh37jx3AyFhe4kBRLvUOYAlCxmhOQgBESffwVZoPJ+emz/JqqtYgaXsZUTZVCAkeeNFWDNby0v54y56sVT9C6cXhgtCL/Irb/Or9vfv8KGaEgRBSgrBBPrdzGLR6DZSjXX+qHIHmcLlkdrKEgL68jZpFu3Exjnipc1yipejsOpUZhl7OgkgXRJ5CBsi9mvH19d4EiSpYQTQVlLzlS57hdJMvipEjAw5d0YUxwuye7ldgGILiBNMKmzq8gSTgwKyvAU0bcgXDvOM4qltplmhNt3QAmnlvt8wzv4doMhs1D4h0VtYEvKOSwTxkLJP3eOzd0fJRiwwjDe+Pp40VRXBtnaeO91YBTqQgqNV7Nw2sXuva9mSUzud3ac1LplEZZdTNYbO6AsuqP+OUPCGWM2xf3aKydOVoy8psx//llIgcIwwn/y2vaJj4cQC6MJZa3k1mySFaFGqt7pO8bTm1kfi9KU8P6E/VhED0sfCxA2T1ZM+EwwmHUKaYc8VfJIVDCMT6Zc5STqMq8RCYmumEsH8vnl/v72w50PMBAijAtFssKnj3mnS4N5sNdoyxo13dXQ3Nen4MdADqS6qNWDyNLstQ5vpAJK6AbfOqsPU2elpDBOH0OLRBkzLtT99/7e0be2ygUkqQAJ3QCVQmCovFM0W5eIU868MiILYewlJF004EtIGgwhFga8hKQB0vata4NKkCk7zUOy8u8nLz/vzv1OHE+FEV4UTYcDxoqnxOGD9tNiHFOc54h4coznpjhmGk2RQ0gyniYvJcqBVDmWLAfR5SSEeUbivuMWn7+fVWJkDh8SMx2fQB7Dnw/4iQR+OMgB6fqURm6l7F2VUmn9PgWMNIQvHPzqWNYKgIcRby0NXN/8u+lHIEUIdZX+3iwvce4lzkVLhZc4933EuTFRbo4u1eAINyy+TdicGgrvqRtRg6NafEx7hhHtpX310r56aV89n/ZVvT/+96uO7tXCG9EVIFwr1DqC1nvMGEfpnuc+f/lp36qqyBverKSC+5tbWHG51cDMax34aX8mUwogRcEZ9cd8QBuFJAcp+O7tIT8eqBZzQmKccq2jET3qlTHEKdeFlYll56FEZmQL8jzzUOm25ESc1K8LoDKmJz84kx20dqX8+RhnwP2ulPZbVQ6FN+pxMO+j5PMN64t6DJEhPGqM2PA9ZLC+0D1NRX8uaE9b3A4M1/PUuWMC9YAwPSZIR4foiwP0mfBsaJFozROfoT9tgG4scS61P19fU8oElVfzbP2eqvwx9S0qBIMqZ8IXOhWpfbi7+9K1oT44nRiH9TCM/f61EWat9s9H9AOc5iiCzqU0G8w8+79hAnL9dh8FmmXfa+3jgX+s6V04sZAzYQ22wignO1SldgXRZclbbwUGBV/qwIsGfKkDB0OIhQEvdeAAacNz/mnIcp8T//41jiwFbhcOO1Th4JgjCxQTw7wJj2k1SgxpjTZEZM5qbehK2vXGE34H3gpnYVNt04boqQ7F39r0zqYDD8Nrm9bjjct07mz9h1PFz9u4DIfQhyRH7R9vm2od3NjccmLYI4YQ5yaxlKGdQCG3HLN1CNfX+7/rXvm7YIFwQYacPaLaeQD95za5XIclPUV/XBrCw1Eezf7CKv2woqVArdYbcvVwRa4qGPUXb4EJIK0p71n0Nk8a3l5NzLSrv4ZcGhZ9v6QN8bWGwuoNoMgKyYR5B6k1IKSBHZrWFPYrY0UtZB5lZp0OyTPUJvEy9gokZD3Fs2zXa/Tpa/m42tvKwYLQDsW69BqkVmE5T5rrvn6Eb1YGaOhTq7N/eLC15BVqy80V/EsqIJChTwTKGuDUrRobfPxhT8gtg3zIZebVzpBknAns1P+85abeMjxnr5PEWFqpH+2yc+xEPuvJXWJivZFGzqjeLARRb4AYg3lxGiL8Jjh7QK+LfhdqbneP3+RWwPKCY47ChBo7k6h99EiJoRv/GpWaka/gTobivNrHlYLvXKJnCBMapMDWDVe+Ad0Uppw7hP44KiWV8xkX0NbsEUXrXqDE99CRKMgtN6zgCIbl2LOH3bJ53aqewO6fDpobXqXKBrWg0OJgVMkqUAzyGP+g6qL5XXWgur1kjayWqwvc5cGBMWle2PVwydEThbG7CsCtW8bfQVA7YdElKKYW2+ZD2DKzASHFe8c9u5aBWTaMLg80W9ZPDvRbyjn+QWWGP45ykVg71geSFnWP8kTSIAdwGiccjUG1LAkWNuVMb0Jp5VBAQAFGFowO0qFz9hZRZLuRGqGS7Y9Y2SLz/YZ0B19lxla7a/rwqbrgkgo+msrm0flYmbg1G6/PErQaOSXnsJelr0KXhKHXY+kK+GT9WwEaUcZ365Kku0Thuu/VVE+l1TtIlXxAl0JvxT6tKtGO6FwsqWtHD2MqJQ8mdObG3+k+0xh/PIV70mmZSIOR06IRF8ogER96i9ToxPBpA0+7ojgm8JHh55yS31CSf6xz2LYYpK/z94RKPbku4U2HzVmojxswEbb6ruB+wzQwDVaHpK+8BP6w0pDW2w87FaBSrNg6CfnVdLuSp+YliLJBIaAbIhwV1C92aLNAYw5qI3j5/jUP56alikGL0VzXlsAl/LZ/oKFgdNyW3b279cLNumdbiGkU2eKtaF+CPduuzByvzemz0zxlwjeWWpfl93eaU1+i3beclF6i16RZwWWpQCMJ8KHWpwY9Ow9PnQ9cGvmrFeKme54NkzbDezl+7fiJePtMUpKh3l1HCEEKvZGHbdvYE0Tl3RdmJCUpLtoY6uialAqdJfAnIu5+1GPZbZlDT13NgrMmb+FbYqWdQTrSzMLmM5/FGmrYJqJJ7RqNbaQpn1tQe995VNT/CE7jN1NeHeKe42dE9pWgVGTdtNlUJ1rvwsgDj7SSgkXHof6nYmNqnthHFT8hN2TvENe3n4ESf2Cl4QSONtw3OZqNzDyKyhPCiwCozPB0CCbWbP66UO/mUVyepERjlpS/mkMoRT3dGjmwRt2MK39Yx/m9X+Ki/NUeSNGV/WtFhEsQAhrQkiPfQWbR5bblldc3X3oSWafYPmeeTJ/w4xluXm++NHLyQ17q33QsTa0LpGzFaOKw5tZM2584sHzVb8tJ1jRiheGENfdPBZx6N/14lxv0ewWj9T34tYLDN9lfuhrh8LzTAsqcfCX/VLRyTO6XzXLp45O9OcanXo3nM/zwGgpU5QNfLUOU2gDlRFfVsQfhauT949uCoh8hI7t33lDOgPV1CovwdAoxZYua5AV39OTf6fBIePU4iLTG35qRnudWwotpfG+gTNeftdfv1ZmuCKiyasJ5PcMsvHDnG5vk/wcAAP//BgUovg==" } diff --git a/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json index 2106cb1e277..f9335292924 100644 --- a/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json @@ -14,120 +14,30 @@ "googlecloud": { "labels": { "metrics": { - "cache_result": "DISABLED", - "proxy_continent": "Europe", - "response_code": "502", - "response_code_class": "500" + "client_network": "ocp-be-c5kjr-network", + "client_subnetwork": "ocp-be-c5kjr-worker-subnet", + "client_zone": "us-central1-a" }, "resource": { - "backend_name": "INVALID_BACKEND", - "backend_scope": "INVALID_BACKEND", - "backend_scope_type": "INVALID_BACKEND", - "backend_target_name": "test1-backend-ks", + "backend_name": "ocp-be-c5kjr-master-us-central1-a", + "backend_scope": "us-central1-a", + "backend_scope_type": "ZONE", + "backend_subnetwork_name": "ocp-be-c5kjr-master-subnet", + "backend_target_name": "ocp-be-c5kjr-api-internal", "backend_target_type": "BACKEND_SERVICE", - "backend_type": "INVALID_BACKEND", - "forwarding_rule_name": "test-lb-ks-forwarding-rule", - "matched_url_path_rule": "UNMATCHED", - "region": "global", - "target_proxy_name": "test-lb-ks-target-proxy", - "url_map_name": "test-lb-ks" + "backend_type": "INSTANCE_GROUP", + "forwarding_rule_name": "ocp-be-c5kjr-api-internal", + "load_balancer_name": "ocp-be-c5kjr-api-internal", + "network_name": "ocp-be-c5kjr-network", + "region": "us-central1" } }, "loadbalancing": { - "https": { - "backend_latencies": { - "count": 4, - "mean": 97.927, - "bucket_options": { - "Options": { - "ExponentialBuckets": { - "num_finite_buckets": 66, - "growth_factor": 1.4, - "scale": 1 - } - } - }, - "bucket_counts": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 4 - ] - }, - "backend_request_bytes_count": 736, - "backend_request_count": 4, - "backend_response_bytes_count": 1952, - "frontend_tcp_rtt": { - "count": 4, - "mean": 50, - "bucket_options": { - "Options": { - "ExponentialBuckets": { - "num_finite_buckets": 66, - "growth_factor": 1.4, - "scale": 1 - } - } - }, - "bucket_counts": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 4 - ] - }, - "request_bytes_count": 736, - "request_count": 4, - "response_bytes_count": 1952, - "total_latencies": { - "count": 4, - "mean": 98.423, - "bucket_options": { - "Options": { - "ExponentialBuckets": { - "num_finite_buckets": 66, - "growth_factor": 1.4, - "scale": 1 - } - } - }, - "bucket_counts": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 4 - ] + "l3": { + "internal": { + "egress_packets_count": { + "value": 0 + } } } } diff --git a/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/fields.yml index 57761c8ea1b..93855f7ee9c 100644 --- a/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/fields.yml +++ b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/fields.yml @@ -11,11 +11,11 @@ description: A distribution of the latency calculated from when the request was sent by the proxy to the backend until the proxy received from the backend the last byte of response. type: group fields: - - name: count + - name: count.value type: long - - name: mean + - name: mean.value type: long - - name: bucket_counts + - name: bucket_counts.value type: long - name: bucket_options type: group @@ -26,30 +26,30 @@ - name: ExponentialBuckets type: group fields: - - name: growth_factor + - name: growth_factor.value type: double - - name: scale + - name: scale.value type: long - - name: num_finite_buckets + - name: num_finite_buckets.value type: long - - name: backend_request_bytes_count + - name: backend_request_bytes_count.value type: long description: The number of bytes sent as requests from HTTP/S load balancer to backends. - - name: backend_request_count + - name: backend_request_count.value type: long description: The number of requests served by backends of HTTP/S load balancer. - - name: backend_response_bytes_count + - name: backend_response_bytes_count.value type: long description: The number of bytes sent as responses from backends (or cache) to HTTP/S load balancer. - name: frontend_tcp_rtt description: A distribution of the RTT measured for each connection between client and proxy. type: group fields: - - name: count + - name: count.value type: long - - name: mean + - name: mean.value type: long - - name: bucket_counts + - name: bucket_counts.value type: long - name: bucket_options type: group @@ -60,11 +60,11 @@ - name: ExponentialBuckets type: group fields: - - name: growth_factor + - name: growth_factor.value type: double - - name: scale + - name: scale.value type: long - - name: num_finite_buckets + - name: num_finite_buckets.value type: long - name: internal type: group @@ -73,11 +73,11 @@ description: A distribution of the latency calculated from when the request was sent by the proxy to the backend until the proxy received from the backend the last byte of response. type: group fields: - - name: count + - name: count.value type: long - - name: mean + - name: mean.value type: long - - name: bucket_counts + - name: bucket_counts.value type: long - name: bucket_options type: group @@ -88,30 +88,30 @@ - name: ExponentialBuckets type: group fields: - - name: growth_factor + - name: growth_factor.value type: double - - name: scale + - name: scale.value type: long - - name: num_finite_buckets + - name: num_finite_buckets.value type: long - - name: request_bytes_count + - name: request_bytes_count.value type: long description: The number of bytes sent as requests from clients to HTTP/S load balancer. - - name: request_count + - name: request_count.value type: long description: The number of requests served by HTTP/S load balancer. - - name: response_bytes_count + - name: response_bytes_count.value type: long description: The number of bytes sent as responses from HTTP/S load balancer to clients. - name: total_latencies description: A distribution of the latency calculated from when the request was received by the proxy until the proxy got ACK from client on last response byte. type: group fields: - - name: count + - name: count.value type: long - - name: mean + - name: mean.value type: long - - name: bucket_counts + - name: bucket_counts.value type: long - name: bucket_options type: group @@ -122,30 +122,30 @@ - name: ExponentialBuckets type: group fields: - - name: growth_factor + - name: growth_factor.value type: double - - name: scale + - name: scale.value type: long - - name: num_finite_buckets + - name: num_finite_buckets.value type: long - - name: request_bytes_count + - name: request_bytes_count.value type: long description: The number of bytes sent as requests from clients to HTTP/S load balancer. - - name: request_count + - name: request_count.value type: long description: The number of requests served by HTTP/S load balancer. - - name: response_bytes_count + - name: response_bytes_count.value type: long description: The number of bytes sent as responses from HTTP/S load balancer to clients. - name: total_latencies description: A distribution of the latency calculated from when the request was received by the proxy until the proxy got ACK from client on last response byte. type: group fields: - - name: count + - name: count.value type: long - - name: mean + - name: mean.value type: long - - name: bucket_counts + - name: bucket_counts.value type: long - name: bucket_options type: group @@ -156,37 +156,37 @@ - name: ExponentialBuckets type: group fields: - - name: growth_factor + - name: growth_factor.value type: double - - name: scale + - name: scale.value type: long - - name: num_finite_buckets + - name: num_finite_buckets.value type: long - name: l3.internal type: group description: Google Cloud Load Balancing metrics fields: - - name: egress_bytes_count + - name: egress_bytes_count.value type: long description: The number of bytes sent from ILB backend to client (for TCP flows it's counting bytes on application stream only). - - name: egress_packets_count + - name: egress_packets_count.value type: long description: The number of packets sent from ILB backend to client of the flow. - - name: ingress_bytes_count + - name: ingress_bytes_count.value type: long description: The number of bytes sent from client to ILB backend (for TCP flows it's counting bytes on application stream only). - - name: ingress_packets_count + - name: ingress_packets_count.value type: long description: The number of packets sent from client to ILB backend. - name: rtt_latencies description: A distribution of RTT measured over TCP connections for ILB flows. type: group fields: - - name: count + - name: count.value type: long - - name: mean + - name: mean.value type: long - - name: bucket_counts + - name: bucket_counts.value type: long - name: bucket_options type: group @@ -197,31 +197,31 @@ - name: ExponentialBuckets type: group fields: - - name: growth_factor + - name: growth_factor.value type: double - - name: scale + - name: scale.value type: long - - name: num_finite_buckets + - name: num_finite_buckets.value type: long - name: tcp_ssl_proxy type: group description: Google Cloud Load Balancing metrics fields: - - name: closed_connections + - name: closed_connections.value type: long description: Number of connections that were terminated over TCP/SSL proxy. - - name: egress_bytes_count + - name: egress_bytes_count.value type: long description: Number of bytes sent from VM to client using proxy. - name: frontend_tcp_rtt description: A distribution of the smoothed RTT (in ms) measured by the proxy's TCP stack, each minute application layer bytes pass from proxy to client. type: group fields: - - name: count + - name: count.value type: long - - name: mean + - name: mean.value type: long - - name: bucket_counts + - name: bucket_counts.value type: long - name: bucket_options type: group @@ -232,18 +232,18 @@ - name: ExponentialBuckets type: group fields: - - name: growth_factor + - name: growth_factor.value type: double - - name: scale + - name: scale.value type: long - - name: num_finite_buckets + - name: num_finite_buckets.value type: long - - name: ingress_bytes_count + - name: ingress_bytes_count.value type: long description: Number of bytes sent from client to VM using proxy. - - name: new_connections + - name: new_connections.value type: long description: Number of connections that were created over TCP/SSL proxy. - - name: open_connections + - name: open_connections.value type: long description: Current number of outstanding connections through the TCP/SSL proxy. diff --git a/x-pack/metricbeat/module/googlecloud/loadbalancing/manifest.yml b/x-pack/metricbeat/module/googlecloud/loadbalancing/manifest.yml index 479f92c94d4..173b8fbd265 100644 --- a/x-pack/metricbeat/module/googlecloud/loadbalancing/manifest.yml +++ b/x-pack/metricbeat/module/googlecloud/loadbalancing/manifest.yml @@ -6,28 +6,29 @@ input: stackdriver: service: loadbalancing metrics: - - "loadbalancing.googleapis.com/https/backend_latencies" - - "loadbalancing.googleapis.com/https/backend_latencies" - - "loadbalancing.googleapis.com/https/backend_request_bytes_count" - - "loadbalancing.googleapis.com/https/backend_request_count" - - "loadbalancing.googleapis.com/https/backend_response_bytes_count" - - "loadbalancing.googleapis.com/https/frontend_tcp_rtt" - - "loadbalancing.googleapis.com/https/request_bytes_count" - - "loadbalancing.googleapis.com/https/request_bytes_count" - - "loadbalancing.googleapis.com/https/request_count" - - "loadbalancing.googleapis.com/https/request_count" - - "loadbalancing.googleapis.com/https/response_bytes_count" - - "loadbalancing.googleapis.com/https/response_bytes_count" - - "loadbalancing.googleapis.com/https/total_latencies" - - "loadbalancing.googleapis.com/https/total_latencies" - - "loadbalancing.googleapis.com/l3/internal/egress_bytes_count" - - "loadbalancing.googleapis.com/l3/internal/egress_packets_count" - - "loadbalancing.googleapis.com/l3/internal/ingress_bytes_count" - - "loadbalancing.googleapis.com/l3/internal/ingress_packets_count" - - "loadbalancing.googleapis.com/l3/internal/rtt_latencies" - - "loadbalancing.googleapis.com/tcp_ssl_proxy/closed_connections" - - "loadbalancing.googleapis.com/tcp_ssl_proxy/egress_bytes_count" - - "loadbalancing.googleapis.com/tcp_ssl_proxy/frontend_tcp_rtt" - - "loadbalancing.googleapis.com/tcp_ssl_proxy/ingress_bytes_count" - - "loadbalancing.googleapis.com/tcp_ssl_proxy/new_connections" - - "loadbalancing.googleapis.com/tcp_ssl_proxy/open_connections" + - metric_types: + - "loadbalancing.googleapis.com/https/backend_latencies" + - "loadbalancing.googleapis.com/https/backend_latencies" + - "loadbalancing.googleapis.com/https/backend_request_bytes_count" + - "loadbalancing.googleapis.com/https/backend_request_count" + - "loadbalancing.googleapis.com/https/backend_response_bytes_count" + - "loadbalancing.googleapis.com/https/frontend_tcp_rtt" + - "loadbalancing.googleapis.com/https/request_bytes_count" + - "loadbalancing.googleapis.com/https/request_bytes_count" + - "loadbalancing.googleapis.com/https/request_count" + - "loadbalancing.googleapis.com/https/request_count" + - "loadbalancing.googleapis.com/https/response_bytes_count" + - "loadbalancing.googleapis.com/https/response_bytes_count" + - "loadbalancing.googleapis.com/https/total_latencies" + - "loadbalancing.googleapis.com/https/total_latencies" + - "loadbalancing.googleapis.com/l3/internal/egress_bytes_count" + - "loadbalancing.googleapis.com/l3/internal/egress_packets_count" + - "loadbalancing.googleapis.com/l3/internal/ingress_bytes_count" + - "loadbalancing.googleapis.com/l3/internal/ingress_packets_count" + - "loadbalancing.googleapis.com/l3/internal/rtt_latencies" + - "loadbalancing.googleapis.com/tcp_ssl_proxy/closed_connections" + - "loadbalancing.googleapis.com/tcp_ssl_proxy/egress_bytes_count" + - "loadbalancing.googleapis.com/tcp_ssl_proxy/frontend_tcp_rtt" + - "loadbalancing.googleapis.com/tcp_ssl_proxy/ingress_bytes_count" + - "loadbalancing.googleapis.com/tcp_ssl_proxy/new_connections" + - "loadbalancing.googleapis.com/tcp_ssl_proxy/open_connections" diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json index 86da8d924e5..fd0dd71838a 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json @@ -18,30 +18,10 @@ } }, "pubsub": { - "snapshot": { - "backlog_bytes": 19, - "backlog_bytes_by_region": 19, - "num_messages": 4, - "num_messages_by_region": 4, - "oldest_message_age": 69319, - "oldest_message_age_by_region": 69319 - }, "subscription": { - "backlog_bytes": 0, - "num_undelivered_messages": 0, - "oldest_retained_acked_message_age": 0, - "oldest_retained_acked_message_age_by_region": 0, - "oldest_unacked_message_age": 0, - "oldest_unacked_message_age_by_region": 69277, - "retained_acked_bytes": 0, - "retained_acked_bytes_by_region": 0, - "unacked_bytes_by_region": 19 - }, - "topic": { - "oldest_retained_acked_message_age_by_region": 0, - "oldest_unacked_message_age_by_region": 69319, - "retained_acked_bytes_by_region": 0, - "unacked_bytes_by_region": 76 + "oldest_unacked_message_age": { + "value": 0 + } } } }, diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml index 32e4acbb521..ae6443e219f 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml @@ -7,152 +7,155 @@ type: group description: Suscription related metrics fields: - - name: ack_message_count + - name: ack_message_count.value type: long description: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. - - name: backlog_bytes + - name: backlog_bytes.value type: long description: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. - - name: num_outstanding_messages + - name: num_outstanding_messages.value type: long description: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. - - name: num_undelivered_messages + - name: num_undelivered_messages.value type: long description: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. - - name: oldest_unacked_message_age + - name: oldest_unacked_message_age.value type: long description: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. - - name: pull_ack_message_operation_count + - name: pull_ack_message_operation_count.value type: long description: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. - - name: pull_ack_request_count + - name: pull_ack_request_count.value type: long description: Cumulative count of acknowledge requests, grouped by result. - - name: pull_message_operation_count + - name: pull_message_operation_count.value type: long description: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. - - name: pull_request_count + - name: pull_request_count.value type: long description: Cumulative count of pull requests, grouped by result. - - name: push_request_count + - name: push_request_count.value type: long description: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. - - name: push_request_latencies + - name: push_request_latencies.value type: long description: Distribution of push request latencies (in microseconds), grouped by result. - - name: sent_message_count + - name: sent_message_count.value type: long description: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. - - name: streaming_pull_ack_message_operation_count + - name: streaming_pull_ack_message_operation_count.value type: long description: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. - - name: streaming_pull_ack_request_count + - name: streaming_pull_ack_request_count.value type: long description: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. - - name: streaming_pull_message_operation_count + - name: streaming_pull_message_operation_count.value type: long description: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count - - name: streaming_pull_response_count + - name: streaming_pull_response_count.value type: long description: Cumulative count of streaming pull responses, grouped by result. - - name: dead_letter_message_count + - name: dead_letter_message_count.value type: long description: Cumulative count of messages published to dead letter topic, grouped by result. - - name: mod_ack_deadline_message_count + - name: mod_ack_deadline_message_count.value type: long description: Cumulative count of messages whose deadline was updated by ModifyAckDeadline requests, grouped by delivery type. - - name: mod_ack_deadline_message_operation_count + - name: mod_ack_deadline_message_operation_count.value type: long description: Cumulative count of ModifyAckDeadline message operations, grouped by result. - - name: mod_ack_deadline_request_count + - name: mod_ack_deadline_request_count.value type: long description: Cumulative count of ModifyAckDeadline requests, grouped by result. - - name: oldest_retained_acked_message_age + - name: oldest_retained_acked_message_age.value type: long description: Age (in seconds) of the oldest acknowledged message retained in a subscription. - - name: oldest_retained_acked_message_age_by_region + - name: oldest_retained_acked_message_age_by_region.value type: long description: Age (in seconds) of the oldest acknowledged message retained in a subscription, broken down by Cloud region. - - name: oldest_unacked_message_age_by_region + - name: oldest_unacked_message_age_by_region.value type: long description: Age (in seconds) of the oldest unacknowledged message in a subscription, broken down by Cloud region. - - name: retained_acked_bytes + - name: retained_acked_bytes.value type: long description: Total byte size of the acknowledged messages retained in a subscription. - - name: retained_acked_bytes_by_region + - name: retained_acked_bytes_by_region.value type: long description: Total byte size of the acknowledged messages retained in a subscription, broken down by Cloud region. - - name: seek_request_count + - name: seek_request_count.value type: long description: Cumulative count of seek attempts, grouped by result. - - name: streaming_pull_mod_ack_deadline_message_operation_count + - name: streaming_pull_mod_ack_deadline_message_operation_count.value type: long description: Cumulative count of StreamingPull ModifyAckDeadline operations, grouped by result. - - name: streaming_pull_mod_ack_deadline_request_count + - name: streaming_pull_mod_ack_deadline_request_count.value type: long description: Cumulative count of streaming pull requests with non-empty ModifyAckDeadline fields, grouped by result. - - name: byte_cost + - name: byte_cost.value type: long description: Cumulative cost of operations, measured in bytes. This is used to measure quota utilization. - - name: config_updates_count + - name: config_updates_count.value type: long description: Cumulative count of configuration changes for each subscription, grouped by operation type and result. - - name: unacked_bytes_by_region + - name: unacked_bytes_by_region.value type: long description: Total byte size of the unacknowledged messages in a subscription, broken down by Cloud region. - name: topic type: group description: Topic related metrics fields: - - name: streaming_pull_response_count + - name: streaming_pull_response_count.value type: long description: Cumulative count of streaming pull responses, grouped by result. - - name: send_message_operation_count + - name: send_message_operation_count.value type: long description: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. - - name: send_request_count + - name: send_request_count.value type: long description: Cumulative count of publish requests, grouped by result. - - name: oldest_retained_acked_message_age_by_region + - name: oldest_retained_acked_message_age_by_region.value type: long description: Age (in seconds) of the oldest acknowledged message retained in a topic, broken down by Cloud region. - - name: oldest_unacked_message_age_by_region + - name: oldest_unacked_message_age_by_region.value type: long description: Age (in seconds) of the oldest unacknowledged message in a topic, broken down by Cloud region. - - name: retained_acked_bytes_by_region + - name: retained_acked_bytes_by_region.value type: long description: Total byte size of the acknowledged messages retained in a topic, broken down by Cloud region. - - name: byte_cost + - name: byte_cost.value type: long description: Cost of operations, measured in bytes. This is used to measure utilization for quotas. - - name: config_updates_count + - name: config_updates_count.value type: long description: Cumulative count of configuration changes, grouped by operation type and result. - - name: unacked_bytes_by_region + - name: message_sizes.value + type: long + description: Distribution of publish message sizes (in bytes) + - name: unacked_bytes_by_region.value type: long description: Total byte size of the unacknowledged messages in a topic, broken down by Cloud region. - name: snapshot type: group description: Snapshot related metrics fields: - - name: oldest_message_age + - name: oldest_message_age.value type: long description: Age (in seconds) of the oldest message retained in a snapshot. - - name: oldest_message_age_by_region + - name: oldest_message_age_by_region.value type: long description: Age (in seconds) of the oldest message retained in a snapshot, broken down by Cloud region. - - name: backlog_bytes + - name: backlog_bytes.value type: long description: Total byte size of the messages retained in a snapshot. - - name: backlog_bytes_by_region + - name: backlog_bytes_by_region.value type: long description: Total byte size of the messages retained in a snapshot, broken down by Cloud region. - - name: num_messages + - name: num_messages.value type: long description: Number of messages retained in a snapshot. - - name: num_messages_by_region + - name: num_messages_by_region.value type: long description: Number of messages retained in a snapshot, broken down by Cloud region. - - name: config_updates_count + - name: config_updates_count.value type: long description: Cumulative count of configuration changes, grouped by operation type and result. diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml index 3d8cdb0949c..285136f3cf8 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml +++ b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml @@ -6,50 +6,51 @@ input: stackdriver: service: pubsub metrics: - - "pubsub.googleapis.com/snapshot/backlog_bytes" - - "pubsub.googleapis.com/snapshot/backlog_bytes_by_region" - - "pubsub.googleapis.com/snapshot/config_updates_count" - - "pubsub.googleapis.com/snapshot/num_messages" - - "pubsub.googleapis.com/snapshot/num_messages_by_region" - - "pubsub.googleapis.com/snapshot/oldest_message_age" - - "pubsub.googleapis.com/snapshot/oldest_message_age_by_region" - - "pubsub.googleapis.com/subscription/ack_message_count" - - "pubsub.googleapis.com/subscription/backlog_bytes" - - "pubsub.googleapis.com/subscription/byte_cost" - - "pubsub.googleapis.com/subscription/config_updates_count" - - "pubsub.googleapis.com/subscription/dead_letter_message_count" - - "pubsub.googleapis.com/subscription/mod_ack_deadline_message_count" - - "pubsub.googleapis.com/subscription/mod_ack_deadline_message_operation_count" - - "pubsub.googleapis.com/subscription/mod_ack_deadline_request_count" - - "pubsub.googleapis.com/subscription/num_outstanding_messages" - - "pubsub.googleapis.com/subscription/num_undelivered_messages" - - "pubsub.googleapis.com/subscription/oldest_retained_acked_message_age" - - "pubsub.googleapis.com/subscription/oldest_retained_acked_message_age_by_region" - - "pubsub.googleapis.com/subscription/oldest_unacked_message_age" - - "pubsub.googleapis.com/subscription/oldest_unacked_message_age_by_region" - - "pubsub.googleapis.com/subscription/pull_ack_message_operation_count" - - "pubsub.googleapis.com/subscription/pull_ack_request_count" - - "pubsub.googleapis.com/subscription/pull_message_operation_count" - - "pubsub.googleapis.com/subscription/pull_request_count" - - "pubsub.googleapis.com/subscription/push_request_count" - - "pubsub.googleapis.com/subscription/push_request_latencies" - - "pubsub.googleapis.com/subscription/retained_acked_bytes" - - "pubsub.googleapis.com/subscription/retained_acked_bytes_by_region" - - "pubsub.googleapis.com/subscription/seek_request_count" - - "pubsub.googleapis.com/subscription/sent_message_count" - - "pubsub.googleapis.com/subscription/streaming_pull_ack_message_operation_count" - - "pubsub.googleapis.com/subscription/streaming_pull_ack_request_count" - - "pubsub.googleapis.com/subscription/streaming_pull_message_operation_count" - - "pubsub.googleapis.com/subscription/streaming_pull_mod_ack_deadline_message_operation_count" - - "pubsub.googleapis.com/subscription/streaming_pull_mod_ack_deadline_request_count" - - "pubsub.googleapis.com/subscription/streaming_pull_response_count" - - "pubsub.googleapis.com/subscription/unacked_bytes_by_region" - - "pubsub.googleapis.com/topic/byte_cost" - - "pubsub.googleapis.com/topic/config_updates_count" - - "pubsub.googleapis.com/topic/message_sizes" - - "pubsub.googleapis.com/topic/oldest_retained_acked_message_age_by_region" - - "pubsub.googleapis.com/topic/oldest_unacked_message_age_by_region" - - "pubsub.googleapis.com/topic/retained_acked_bytes_by_region" - - "pubsub.googleapis.com/topic/send_message_operation_count" - - "pubsub.googleapis.com/topic/send_request_count" - - "pubsub.googleapis.com/topic/unacked_bytes_by_region" + - metric_types: + - "pubsub.googleapis.com/snapshot/backlog_bytes" + - "pubsub.googleapis.com/snapshot/backlog_bytes_by_region" + - "pubsub.googleapis.com/snapshot/config_updates_count" + - "pubsub.googleapis.com/snapshot/num_messages" + - "pubsub.googleapis.com/snapshot/num_messages_by_region" + - "pubsub.googleapis.com/snapshot/oldest_message_age" + - "pubsub.googleapis.com/snapshot/oldest_message_age_by_region" + - "pubsub.googleapis.com/subscription/ack_message_count" + - "pubsub.googleapis.com/subscription/backlog_bytes" + - "pubsub.googleapis.com/subscription/byte_cost" + - "pubsub.googleapis.com/subscription/config_updates_count" + - "pubsub.googleapis.com/subscription/dead_letter_message_count" + - "pubsub.googleapis.com/subscription/mod_ack_deadline_message_count" + - "pubsub.googleapis.com/subscription/mod_ack_deadline_message_operation_count" + - "pubsub.googleapis.com/subscription/mod_ack_deadline_request_count" + - "pubsub.googleapis.com/subscription/num_outstanding_messages" + - "pubsub.googleapis.com/subscription/num_undelivered_messages" + - "pubsub.googleapis.com/subscription/oldest_retained_acked_message_age" + - "pubsub.googleapis.com/subscription/oldest_retained_acked_message_age_by_region" + - "pubsub.googleapis.com/subscription/oldest_unacked_message_age" + - "pubsub.googleapis.com/subscription/oldest_unacked_message_age_by_region" + - "pubsub.googleapis.com/subscription/pull_ack_message_operation_count" + - "pubsub.googleapis.com/subscription/pull_ack_request_count" + - "pubsub.googleapis.com/subscription/pull_message_operation_count" + - "pubsub.googleapis.com/subscription/pull_request_count" + - "pubsub.googleapis.com/subscription/push_request_count" + - "pubsub.googleapis.com/subscription/push_request_latencies" + - "pubsub.googleapis.com/subscription/retained_acked_bytes" + - "pubsub.googleapis.com/subscription/retained_acked_bytes_by_region" + - "pubsub.googleapis.com/subscription/seek_request_count" + - "pubsub.googleapis.com/subscription/sent_message_count" + - "pubsub.googleapis.com/subscription/streaming_pull_ack_message_operation_count" + - "pubsub.googleapis.com/subscription/streaming_pull_ack_request_count" + - "pubsub.googleapis.com/subscription/streaming_pull_message_operation_count" + - "pubsub.googleapis.com/subscription/streaming_pull_mod_ack_deadline_message_operation_count" + - "pubsub.googleapis.com/subscription/streaming_pull_mod_ack_deadline_request_count" + - "pubsub.googleapis.com/subscription/streaming_pull_response_count" + - "pubsub.googleapis.com/subscription/unacked_bytes_by_region" + - "pubsub.googleapis.com/topic/byte_cost" + - "pubsub.googleapis.com/topic/config_updates_count" + - "pubsub.googleapis.com/topic/message_sizes" + - "pubsub.googleapis.com/topic/oldest_retained_acked_message_age_by_region" + - "pubsub.googleapis.com/topic/oldest_unacked_message_age_by_region" + - "pubsub.googleapis.com/topic/retained_acked_bytes_by_region" + - "pubsub.googleapis.com/topic/send_message_operation_count" + - "pubsub.googleapis.com/topic/send_request_count" + - "pubsub.googleapis.com/topic/unacked_bytes_by_region" diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/stackdriver/_meta/docs.asciidoc new file mode 100644 index 00000000000..00cce58cac8 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/_meta/docs.asciidoc @@ -0,0 +1,82 @@ +Stackdriver provides visibility into the performance, uptime, and overall health +of cloud-powered applications. It collects metrics, events, and metadata from +different services from Google Cloud. This metricset is to collect monitoring +metrics from Google Cloud using `ListTimeSeries` API. + +[float] +== Metricset config and parameters + +* *metric_types*: Required, a list of metric type strings. Each call of the +`ListTimeSeries` API can return any number of time series from a single metric +type. Metric type is to used for identifying a specific time series. + +* *aligner*: A single string with which aggregation operation need to be applied +onto time series data for ListTimeSeries API. If it's not given, default aligner +is set to be `ALIGN_NONE`. Sample period of each metric type is obtained from +making https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.metricDescriptors/list [ListMetricDescriptors API] call. + +[float] +=== Example Configuration +* `stackdriver` metricset is enabled to collect metrics from all zones under +`europe-west1-c` region in `elastic-observability` project. Two sets of metrics +are specified: first one is to collect CPU usage time and utilization with +aggregation aligner ALIGN_MEAN; second one is to collect uptime with aggregation +aligner ALIGN_SUM. These metric types all have 240 seconds ingest delay time and +60 seconds sample period. With `period` specified as `300s` in the config below, +Metricbeat will collect compute metrics from googlecloud every 5-minute with +given aggregation aligner applied for each metric type. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - stackdriver + zone: "europe-west1-c" + project_id: elastic-observability + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + stackdriver: + service: compute + metrics: + - aligner: ALIGN_MEAN + metric_types: + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - aligner: ALIGN_SUM + metric_types: + - "compute.googleapis.com/instance/uptime" + +---- + +* `stackdriver` metricset is enabled to collect metrics from all zones under +`europe-west1-c` region in `elastic-observability` project. Two sets of metrics +are specified: first one is to collect CPU usage time and utilization with +aggregation aligner ALIGN_MEAN; second one is to collect uptime with aggregation +aligner ALIGN_SUM. These metric types all have 240 seconds ingest delay time and +60 seconds sample period. With `period` specified as `60s` in the config below, +Metricbeat will collect compute metrics from googlecloud every minute with no +aggregation. This case, the aligners specified in the configuration will be +ignored. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - stackdriver + zone: "europe-west1-c" + project_id: elastic-observability + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s + stackdriver: + service: compute + metrics: + - aligner: ALIGN_MEAN + metric_types: + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - aligner: ALIGN_SUM + metric_types: + - "compute.googleapis.com/instance/uptime" +---- diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/compute/identity.go b/x-pack/metricbeat/module/googlecloud/stackdriver/compute/identity.go index 9367862e6c7..19e434e8df7 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/compute/identity.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/compute/identity.go @@ -23,7 +23,7 @@ func (s *metadataCollector) ID(ctx context.Context, in *googlecloud.MetadataColl if in.Timestamp != nil { metadata.ECS.Put("timestamp", in.Timestamp) } else if in.Point != nil { - metadata.ECS.Put("timestamp", in.Point.Interval.StartTime) + metadata.ECS.Put("timestamp", in.Point.Interval.EndTime) } else { return "", errors.New("no timestamp information found") } diff --git a/x-pack/metricbeat/module/googlecloud/integration.go b/x-pack/metricbeat/module/googlecloud/stackdriver/integration.go similarity index 86% rename from x-pack/metricbeat/module/googlecloud/integration.go rename to x-pack/metricbeat/module/googlecloud/stackdriver/integration.go index 0e7ac055bc7..68e3750d5a5 100644 --- a/x-pack/metricbeat/module/googlecloud/integration.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/integration.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package googlecloud +package stackdriver import ( "os" @@ -36,7 +36,11 @@ func GetConfigForTest(t *testing.T, metricSetName string) map[string]interface{} if metricSetName == "stackdriver" { config["stackdriver.service"] = "compute" - config["stackdriver.metrics"] = []string{"compute.googleapis.com/instance/uptime"} + stackDriverConfig := stackDriverConfig{ + Aligner: "ALIGN_NONE", + MetricTypes: []string{"compute.googleapis.com/instance/uptime"}, + } + config["stackdriver.metrics"] = stackDriverConfig } } return config diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index 8d2147d285c..edfd52a64e3 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -11,9 +11,10 @@ import ( "sync" "time" + "github.com/golang/protobuf/ptypes/duration" + monitoring "cloud.google.com/go/monitoring/apiv3" "github.com/golang/protobuf/ptypes/timestamp" - "github.com/pkg/errors" "google.golang.org/api/iterator" monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3" @@ -21,42 +22,31 @@ import ( "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud" ) -func newStackdriverMetricsRequester(ctx context.Context, c config, window time.Duration, logger *logp.Logger) (*stackdriverMetricsRequester, error) { - interval, err := getTimeInterval(window) - if err != nil { - return nil, errors.Wrap(err, "error trying to get time window") - } - - client, err := monitoring.NewMetricClient(ctx, c.opt...) - if err != nil { - return nil, errors.Wrap(err, "error creating Stackdriver client") - } - - return &stackdriverMetricsRequester{ - config: c, - client: client, - logger: logger, - interval: interval, - }, nil -} - type stackdriverMetricsRequester struct { config config - client *monitoring.MetricClient - interval *monitoringpb.TimeInterval + client *monitoring.MetricClient logger *logp.Logger } -func (r *stackdriverMetricsRequester) Metric(ctx context.Context, m string) (out []*monitoringpb.TimeSeries) { - out = make([]*monitoringpb.TimeSeries, 0) +type timeSeriesWithAligner struct { + timeSeries []*monitoringpb.TimeSeries + aligner string +} + +func (r *stackdriverMetricsRequester) Metric(ctx context.Context, metricType string, timeInterval *monitoringpb.TimeInterval, aligner string) (out timeSeriesWithAligner) { + timeSeries := make([]*monitoringpb.TimeSeries, 0) req := &monitoringpb.ListTimeSeriesRequest{ Name: "projects/" + r.config.ProjectID, - Interval: r.interval, + Interval: timeInterval, View: monitoringpb.ListTimeSeriesRequest_FULL, - Filter: r.getFilterForMetric(m), + Filter: r.getFilterForMetric(metricType), + Aggregation: &monitoringpb.Aggregation{ + PerSeriesAligner: googlecloud.AlignersMapToGCP[aligner], + AlignmentPeriod: &r.config.period, + }, } it := r.client.ListTimeSeries(ctx, req) @@ -67,13 +57,15 @@ func (r *stackdriverMetricsRequester) Metric(ctx context.Context, m string) (out } if err != nil { - r.logger.Errorf("Could not read time series value: %s: %v", m, err) + r.logger.Errorf("Could not read time series value: %s: %v", metricType, err) break } - out = append(out, resp) + timeSeries = append(timeSeries, resp) } + out.aligner = aligner + out.timeSeries = timeSeries return } @@ -89,23 +81,29 @@ func constructFilter(m string, region string, zone string) string { return filter } -func (r *stackdriverMetricsRequester) Metrics(ctx context.Context, ms []string) ([]*monitoringpb.TimeSeries, error) { +func (r *stackdriverMetricsRequester) Metrics(ctx context.Context, stackDriverConfigs []stackDriverConfig, metricsMeta map[string]metricMeta) ([]timeSeriesWithAligner, error) { var lock sync.Mutex var wg sync.WaitGroup - results := make([]*monitoringpb.TimeSeries, 0) + results := make([]timeSeriesWithAligner, 0) - for _, metric := range ms { - wg.Add(1) + for _, sdc := range stackDriverConfigs { + aligner := sdc.Aligner + for _, mt := range sdc.MetricTypes { + metricType := mt + wg.Add(1) - go func(m string) { - defer wg.Done() + go func(metricType string) { + defer wg.Done() - ts := r.Metric(ctx, m) + metricMeta := metricsMeta[metricType] + interval, aligner := getTimeIntervalAligner(metricMeta.ingestDelay, metricMeta.samplePeriod, r.config.period, aligner) + ts := r.Metric(ctx, metricType, interval, aligner) - lock.Lock() - defer lock.Unlock() - results = append(results, ts...) - }(metric) + lock.Lock() + defer lock.Unlock() + results = append(results, ts) + }(metricType) + } } wg.Wait() @@ -144,21 +142,28 @@ func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) { return } -// Returns a GCP TimeInterval based on the provided config -func getTimeInterval(windowTime time.Duration) (*monitoringpb.TimeInterval, error) { - var startTime, endTime time.Time - - if windowTime > 0 { - endTime = time.Now().UTC() - startTime = time.Now().UTC().Add(-windowTime) +// Returns a GCP TimeInterval based on the ingestDelay and samplePeriod from ListMetricDescriptor +func getTimeIntervalAligner(ingestDelay time.Duration, samplePeriod time.Duration, collectionPeriod duration.Duration, inputAligner string) (*monitoringpb.TimeInterval, string) { + var startTime, endTime, currentTime time.Time + var needsAggregation bool + currentTime = time.Now().UTC() + + // When samplePeriod < collectionPeriod, aggregation will be done in ListTimeSeriesRequest. + // For example, samplePeriod = 60s, collectionPeriod = 300s, if perSeriesAligner is not given, + // ALIGN_MEAN will be used by default. + if int64(samplePeriod.Seconds()) < collectionPeriod.Seconds { + endTime = currentTime.Add(-ingestDelay) + startTime = endTime.Add(-time.Duration(collectionPeriod.Seconds) * time.Second) + needsAggregation = true } - if windowTime.Minutes() < googlecloud.MinTimeIntervalDataWindowMinutes { - return nil, errors.Errorf("the provided window time is too small. No less than %d minutes can be fetched", googlecloud.MinTimeIntervalDataWindowMinutes) - } - - if windowTime.Minutes() >= googlecloud.MaxTimeIntervalDataWindowMinutes { - return nil, errors.Errorf("the provided window time is too big. No more than %d minutes can be fetched", googlecloud.MaxTimeIntervalDataWindowMinutes) + // When samplePeriod == collectionPeriod, aggregation is not needed + // When samplePeriod > collectionPeriod, aggregation is not needed, use sample period + // to determine startTime and endTime to make sure there will be data point in this time range. + if int64(samplePeriod.Seconds()) >= collectionPeriod.Seconds { + endTime = time.Now().UTC().Add(-ingestDelay) + startTime = endTime.Add(-samplePeriod) + needsAggregation = false } interval := &monitoringpb.TimeInterval{ @@ -170,5 +175,11 @@ func getTimeInterval(windowTime time.Duration) (*monitoringpb.TimeInterval, erro }, } - return interval, nil + // Default aligner for aggregation is ALIGN_NONE if it's not given + updatedAligner := googlecloud.DefaultAligner + if needsAggregation && inputAligner != "" { + updatedAligner = inputAligner + } + + return interval, updatedAligner } diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go index be7b824d224..1ed10814b76 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go @@ -6,7 +6,9 @@ package stackdriver import ( "testing" + "time" + "github.com/golang/protobuf/ptypes/duration" "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/logp" @@ -103,3 +105,62 @@ func TestGetFilterForMetric(t *testing.T) { }) } } + +func TestGetTimeIntervalAligner(t *testing.T) { + cases := []struct { + title string + ingestDelay time.Duration + samplePeriod time.Duration + collectionPeriod duration.Duration + inputAligner string + expectedAligner string + }{ + { + "test collectionPeriod equals to samplePeriod", + time.Duration(240) * time.Second, + time.Duration(60) * time.Second, + duration.Duration{ + Seconds: int64(60), + }, + "", + "ALIGN_NONE", + }, + { + "test collectionPeriod larger than samplePeriod", + time.Duration(240) * time.Second, + time.Duration(60) * time.Second, + duration.Duration{ + Seconds: int64(300), + }, + "ALIGN_MEAN", + "ALIGN_MEAN", + }, + { + "test collectionPeriod smaller than samplePeriod", + time.Duration(240) * time.Second, + time.Duration(60) * time.Second, + duration.Duration{ + Seconds: int64(30), + }, + "ALIGN_MAX", + "ALIGN_NONE", + }, + { + "test collectionPeriod equals to samplePeriod with given aligner", + time.Duration(240) * time.Second, + time.Duration(60) * time.Second, + duration.Duration{ + Seconds: int64(60), + }, + "ALIGN_MEAN", + "ALIGN_NONE", + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + _, aligner := getTimeIntervalAligner(c.ingestDelay, c.samplePeriod, c.collectionPeriod, c.inputAligner) + assert.Equal(t, c.expectedAligner, aligner) + }) + } +} diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go index 69ac38ca101..81fa98751aa 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go @@ -6,8 +6,13 @@ package stackdriver import ( "context" + "fmt" "time" + "github.com/golang/protobuf/ptypes/duration" + + monitoring "cloud.google.com/go/monitoring/apiv3" + "github.com/pkg/errors" "google.golang.org/api/option" @@ -38,19 +43,33 @@ func init() { // interface methods except for Fetch. type MetricSet struct { mb.BaseMetricSet - config config + config config + metricsMeta map[string]metricMeta + requester *stackdriverMetricsRequester + stackDriverConfig []stackDriverConfig `config:"metrics" validate:"nonzero,required"` +} + +//stackDriverConfig holds a configuration specific for stackdriver metricset. +type stackDriverConfig struct { + MetricTypes []string `config:"metric_types" validate:"required"` + Aligner string `config:"aligner"` +} + +type metricMeta struct { + samplePeriod time.Duration + ingestDelay time.Duration } type config struct { - Metrics []string `config:"stackdriver.metrics" validate:"required"` - Zone string `config:"zone"` - Region string `config:"region"` - ProjectID string `config:"project_id" validate:"required"` - ExcludeLabels bool `config:"exclude_labels"` - ServiceName string `config:"stackdriver.service" validate:"required"` - CredentialsFilePath string `config:"credentials_file_path"` - - opt []option.ClientOption + Zone string `config:"zone"` + Region string `config:"region"` + ProjectID string `config:"project_id" validate:"required"` + ExcludeLabels bool `config:"exclude_labels"` + ServiceName string `config:"stackdriver.service" validate:"required"` + CredentialsFilePath string `config:"credentials_file_path"` + + opt []option.ClientOption + period duration.Duration } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -64,12 +83,39 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } + stackDriverConfigs := struct { + StackDriverMetrics []stackDriverConfig `config:"stackdriver.metrics" validate:"nonzero,required"` + }{} + + if err := base.Module().UnpackConfig(&stackDriverConfigs); err != nil { + return nil, err + } + + m.stackDriverConfig = stackDriverConfigs.StackDriverMetrics m.config.opt = []option.ClientOption{option.WithCredentialsFile(m.config.CredentialsFilePath)} + m.config.period.Seconds = int64(m.Module().Config().Period.Seconds()) if err := validatePeriodForGCP(m.Module().Config().Period); err != nil { return nil, err } + // Get ingest delay and sample period for each metric type + ctx := context.Background() + client, err := monitoring.NewMetricClient(ctx, m.config.opt...) + if err != nil { + return nil, errors.Wrap(err, "error creating Stackdriver client") + } + + m.metricsMeta, err = metricDescriptor(ctx, client, m.config.ProjectID, m.stackDriverConfig) + if err != nil { + return nil, errors.Wrap(err, "error calling metricDescriptor function") + } + + m.requester = &stackdriverMetricsRequester{ + config: m.config, + client: client, + logger: m.Logger(), + } return m, nil } @@ -77,12 +123,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) (err error) { - reqs, err := newStackdriverMetricsRequester(ctx, m.config, m.Module().Config().Period, m.Logger()) - if err != nil { - return errors.Wrapf(err, "error trying to do create a request client to GCP project '%s' in zone '%s' or region '%s'", m.config.ProjectID, m.config.Zone, m.config.Region) - } - - responses, err := reqs.Metrics(ctx, m.config.Metrics) + responses, err := m.requester.Metrics(ctx, m.stackDriverConfig, m.metricsMeta) if err != nil { return errors.Wrapf(err, "error trying to get metrics for project '%s' and zone '%s' or region '%s'", m.config.ProjectID, m.config.Zone, m.config.Region) } @@ -99,7 +140,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) (err erro return nil } -func (m *MetricSet) eventMapping(ctx context.Context, tss []*monitoringpb.TimeSeries) ([]mb.Event, error) { +func (m *MetricSet) eventMapping(ctx context.Context, tss []timeSeriesWithAligner) ([]mb.Event, error) { e := newIncomingFieldExtractor(m.Logger()) var gcpService = googlecloud.NewStackdriverMetadataServiceForTimeSeries(nil) @@ -140,13 +181,14 @@ func (m *MetricSet) eventMapping(ctx context.Context, tss []*monitoringpb.TimeSe // validatePeriodForGCP returns nil if the Period in the module config is in the accepted threshold func validatePeriodForGCP(d time.Duration) (err error) { - if d.Seconds() < 300 { - return errors.New("period in Google Cloud config file cannot be set to less than 300 seconds") + if d.Seconds() < googlecloud.MonitoringMetricsSamplingRate { + return errors.Errorf("period in Google Cloud config file cannot be set to less than %d seconds", googlecloud.MonitoringMetricsSamplingRate) } return nil } +// Validate googlecloud module config func (c *config) Validate() error { // storage metricset does not require region or zone config parameter. if c.ServiceName == "storage" { @@ -158,3 +200,46 @@ func (c *config) Validate() error { } return nil } + +// Validate stackdriver related config +func (mc *stackDriverConfig) Validate() error { + gcpAlignerNames := make([]string, 0) + for k := range googlecloud.AlignersMapToGCP { + gcpAlignerNames = append(gcpAlignerNames, k) + } + + if mc.Aligner != "" { + if _, ok := googlecloud.AlignersMapToGCP[mc.Aligner]; !ok { + return errors.Errorf("the given aligner is not supported, please specify one of %s as aligner", gcpAlignerNames) + } + } + return nil +} + +// metricDescriptor calls ListMetricDescriptorsRequest API to get metric metadata +// (sample period and ingest delay) of each given metric type +func metricDescriptor(ctx context.Context, client *monitoring.MetricClient, projectID string, stackDriverConfigs []stackDriverConfig) (map[string]metricMeta, error) { + metricsWithMeta := make(map[string]metricMeta, 0) + + for _, sdc := range stackDriverConfigs { + for _, mt := range sdc.MetricTypes { + req := &monitoringpb.ListMetricDescriptorsRequest{ + Name: "projects/" + projectID, + Filter: fmt.Sprintf(`metric.type = "%s"`, mt), + } + + it := client.ListMetricDescriptors(ctx, req) + out, err := it.Next() + if err != nil { + return metricsWithMeta, errors.Errorf("Could not make ListMetricDescriptors request: %s: %v", mt, err) + } + + metricsWithMeta[mt] = metricMeta{ + samplePeriod: time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second, + ingestDelay: time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second, + } + } + } + + return metricsWithMeta, nil +} diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/response_parser.go b/x-pack/metricbeat/module/googlecloud/stackdriver/response_parser.go index 474f04a244b..b6e38f4d333 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/response_parser.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/response_parser.go @@ -9,9 +9,10 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud" "github.com/golang/protobuf/ptypes" + "github.com/pkg/errors" "google.golang.org/genproto/googleapis/monitoring/v3" "github.com/elastic/beats/v7/libbeat/common" @@ -36,7 +37,7 @@ type KeyValuePoint struct { } // extractTimeSeriesMetricValues valuable to send to Elasticsearch. This includes, for example, metric values, labels and timestamps -func (e *incomingFieldExtractor) extractTimeSeriesMetricValues(resp *monitoring.TimeSeries) (points []KeyValuePoint, err error) { +func (e *incomingFieldExtractor) extractTimeSeriesMetricValues(resp *monitoring.TimeSeries, aligner string) (points []KeyValuePoint, err error) { points = make([]KeyValuePoint, 0) for _, point := range resp.Points { @@ -48,7 +49,7 @@ func (e *incomingFieldExtractor) extractTimeSeriesMetricValues(resp *monitoring. } p := KeyValuePoint{ - Key: cleanMetricNameString(resp.Metric.Type), + Key: cleanMetricNameString(resp.Metric.Type, aligner), Value: getValueFromPoint(point), Timestamp: ts, } @@ -62,8 +63,8 @@ func (e *incomingFieldExtractor) extractTimeSeriesMetricValues(resp *monitoring. func (e *incomingFieldExtractor) getTimestamp(p *monitoring.Point) (ts time.Time, err error) { // Don't add point intervals that can't be "stated" at some timestamp. if p.Interval != nil { - if ts, err = ptypes.Timestamp(p.Interval.StartTime); err != nil { - return time.Time{}, errors.Errorf("error trying to parse timestamp '%#v' from metric\n", p.Interval.StartTime) + if ts, err = ptypes.Timestamp(p.Interval.EndTime); err != nil { + return time.Time{}, errors.Errorf("error trying to parse timestamp '%#v' from metric\n", p.Interval.EndTime) } return ts, nil } @@ -73,7 +74,7 @@ func (e *incomingFieldExtractor) getTimestamp(p *monitoring.Point) (ts time.Time var rx = regexp.MustCompile(`^[a-z_-]+\.googleapis.com\/`) -func cleanMetricNameString(s string) string { +func cleanMetricNameString(s string, aligner string) string { if s == "" { return "unknown" } @@ -83,7 +84,8 @@ func cleanMetricNameString(s string) string { removedPrefix := strings.TrimPrefix(s, prefix) replacedChars := strings.Replace(removedPrefix, "/", ".", -1) - return replacedChars + metricName := replacedChars + googlecloud.AlignersMapToSuffix[aligner] + return metricName } func getValueFromPoint(p *monitoring.Point) (out interface{}) { diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/response_parser_test.go b/x-pack/metricbeat/module/googlecloud/stackdriver/response_parser_test.go index 4f055ffc0cc..84f689d042b 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/response_parser_test.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/response_parser_test.go @@ -5,7 +5,10 @@ package stackdriver import ( + "testing" + "github.com/golang/protobuf/ptypes/timestamp" + "github.com/stretchr/testify/assert" "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/genproto/googleapis/api/monitoredres" "google.golang.org/genproto/googleapis/monitoring/v3" @@ -65,3 +68,32 @@ var metrics = []string{ "compute.googleapis.com/instance/disk/read_bytes_count", "compute.googleapis.com/http/server/response_latencies", } + +func TestCleanMetricNameString(t *testing.T) { + cases := []struct { + title string + metricType string + aligner string + expectedMetricName string + }{ + { + "test construct metric name with ALIGN_MEAN aligner", + "compute.googleapis.com/instance/cpu/usage_time", + "ALIGN_MEAN", + "instance.cpu.usage_time.avg", + }, + { + "test construct metric name with ALIGN_NONE aligner", + "compute.googleapis.com/instance/cpu/utilization", + "ALIGN_NONE", + "instance.cpu.utilization.value", + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + metricName := cleanMetricNameString(c.metricType, c.aligner) + assert.Equal(t, c.expectedMetricName, metricName) + }) + } +} diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/stackdriver_integration_test.go b/x-pack/metricbeat/module/googlecloud/stackdriver/stackdriver_integration_test.go index 8fe568ebe95..fd11a50c6e9 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/stackdriver_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/stackdriver_integration_test.go @@ -11,11 +11,10 @@ import ( "testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud" ) func TestData(t *testing.T) { - config := googlecloud.GetConfigForTest(t, "stackdriver") + config := GetConfigForTest(t, "stackdriver") metricSet := mbtest.NewFetcher(t, config) metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/timeseries.go b/x-pack/metricbeat/module/googlecloud/stackdriver/timeseries.go index fcf3184717c..c0b456f9954 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/timeseries.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/timeseries.go @@ -7,53 +7,54 @@ package stackdriver import ( "context" - monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud" ) //timeSeriesGrouped groups TimeSeries responses into common Elasticsearch friendly events. This is to avoid sending // events with a single metric that shares info (like timestamp) with another event with a single metric too -func (m *MetricSet) timeSeriesGrouped(ctx context.Context, gcpService googlecloud.MetadataService, tss []*monitoringpb.TimeSeries, e *incomingFieldExtractor) (map[string][]KeyValuePoint, error) { +func (m *MetricSet) timeSeriesGrouped(ctx context.Context, gcpService googlecloud.MetadataService, tsas []timeSeriesWithAligner, e *incomingFieldExtractor) (map[string][]KeyValuePoint, error) { eventGroups := make(map[string][]KeyValuePoint) metadataService := gcpService - for _, ts := range tss { - keyValues, err := e.extractTimeSeriesMetricValues(ts) - if err != nil { - return nil, err - } - - sdCollectorInputData := googlecloud.NewStackdriverCollectorInputData(ts, m.config.ProjectID, m.config.Zone, m.config.Region) - if gcpService == nil { - metadataService = googlecloud.NewStackdriverMetadataServiceForTimeSeries(ts) - } - - for i := range keyValues { - sdCollectorInputData.Timestamp = &keyValues[i].Timestamp - - id, err := metadataService.ID(ctx, sdCollectorInputData) + for _, tsa := range tsas { + aligner := tsa.aligner + for _, ts := range tsa.timeSeries { + keyValues, err := e.extractTimeSeriesMetricValues(ts, aligner) if err != nil { - m.Logger().Errorf("error trying to retrieve ID from metric event '%v'", err) - continue + return nil, err } - metadataCollectorData, err := metadataService.Metadata(ctx, sdCollectorInputData.TimeSeries) - if err != nil { - m.Logger().Error("error trying to retrieve labels from metric event") - continue + sdCollectorInputData := googlecloud.NewStackdriverCollectorInputData(ts, m.config.ProjectID, m.config.Zone, m.config.Region) + if gcpService == nil { + metadataService = googlecloud.NewStackdriverMetadataServiceForTimeSeries(ts) } - if _, ok := eventGroups[id]; !ok { - eventGroups[id] = make([]KeyValuePoint, 0) - } + for i := range keyValues { + sdCollectorInputData.Timestamp = &keyValues[i].Timestamp + + id, err := metadataService.ID(ctx, sdCollectorInputData) + if err != nil { + m.Logger().Errorf("error trying to retrieve ID from metric event '%v'", err) + continue + } - keyValues[i].ECS = metadataCollectorData.ECS - keyValues[i].Labels = metadataCollectorData.Labels + metadataCollectorData, err := metadataService.Metadata(ctx, sdCollectorInputData.TimeSeries) + if err != nil { + m.Logger().Error("error trying to retrieve labels from metric event") + continue + } - // Group the data into common events - eventGroups[id] = append(eventGroups[id], keyValues[i]) + if _, ok := eventGroups[id]; !ok { + eventGroups[id] = make([]KeyValuePoint, 0) + } + + keyValues[i].ECS = metadataCollectorData.ECS + keyValues[i].Labels = metadataCollectorData.Labels + + // Group the data into common events + eventGroups[id] = append(eventGroups[id], keyValues[i]) + } } } diff --git a/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json b/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json index f94a1375997..02541f8a1e7 100644 --- a/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json @@ -14,18 +14,19 @@ "googlecloud": { "labels": { "metrics": { - "storage_class": "REGIONAL" + "method": "GetBucketMetadata", + "response_code": "OK" }, "resource": { - "bucket_name": "elastic-vsphere-images", - "location": "us-east1" + "bucket_name": "ocp-be-c5kjr-image-registry-us-central1-dsoafnbgctvfimpavswkgn", + "location": "us-central1" } }, "storage": { - "storage": { - "object_count": 3, - "total_byte_seconds": 58816542441472, - "total_bytes": 680747019 + "network": { + "sent_bytes_count": { + "value": 2637 + } } } }, diff --git a/x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml index 71fcb2bdbeb..cdf2fde2fff 100644 --- a/x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml +++ b/x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml @@ -6,39 +6,39 @@ - name: api type: group fields: - - name: request_count + - name: request_count.value type: long description: Delta count of API calls, grouped by the API method name and response code. - name: authz type: group fields: - - name: acl_based_object_access_count + - name: acl_based_object_access_count.value type: long description: Delta count of requests that result in an object being granted access solely due to object ACLs. - - name: acl_operations_count + - name: acl_operations_count.value type: long description: Usage of ACL operations broken down by type. - - name: object_specific_acl_mutation_count + - name: object_specific_acl_mutation_count.value type: long description: Delta count of changes made to object specific ACLs. - name: network type: group fields: - - name: received_bytes_count + - name: received_bytes_count.value type: long description: Delta count of bytes received over the network, grouped by the API method name and response code. - - name: sent_bytes_count + - name: sent_bytes_count.value type: long description: Delta count of bytes sent over the network, grouped by the API method name and response code. - name: storage type: group fields: - - name: object_count + - name: object_count.value type: long description: Total number of objects per bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. - - name: total_byte_seconds + - name: total_byte_seconds.value type: long description: Delta count of bytes received over the network, grouped by the API method name and response code. - - name: total_bytes + - name: total_bytes.value type: long description: Total size of all objects in the bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. diff --git a/x-pack/metricbeat/module/googlecloud/storage/manifest.yml b/x-pack/metricbeat/module/googlecloud/storage/manifest.yml index f462867dcb5..2a9363cf78d 100644 --- a/x-pack/metricbeat/module/googlecloud/storage/manifest.yml +++ b/x-pack/metricbeat/module/googlecloud/storage/manifest.yml @@ -6,12 +6,13 @@ input: stackdriver: service: storage metrics: - - "storage.googleapis.com/api/request_count" - - "storage.googleapis.com/authz/acl_based_object_access_count" - - "storage.googleapis.com/authz/acl_operations_count" - - "storage.googleapis.com/authz/object_specific_acl_mutation_count" - - "storage.googleapis.com/network/received_bytes_count" - - "storage.googleapis.com/network/sent_bytes_count" - - "storage.googleapis.com/storage/object_count" - - "storage.googleapis.com/storage/total_byte_seconds" - - "storage.googleapis.com/storage/total_bytes" + - metric_types: + - "storage.googleapis.com/api/request_count" + - "storage.googleapis.com/authz/acl_based_object_access_count" + - "storage.googleapis.com/authz/acl_operations_count" + - "storage.googleapis.com/authz/object_specific_acl_mutation_count" + - "storage.googleapis.com/network/received_bytes_count" + - "storage.googleapis.com/network/sent_bytes_count" + - "storage.googleapis.com/storage/object_count" + - "storage.googleapis.com/storage/total_byte_seconds" + - "storage.googleapis.com/storage/total_bytes" diff --git a/x-pack/metricbeat/module/googlecloud/storage/storage_integration_test.go b/x-pack/metricbeat/module/googlecloud/storage/storage_integration_test.go index f669a7c46e9..0f2b010f5f2 100644 --- a/x-pack/metricbeat/module/googlecloud/storage/storage_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/storage/storage_integration_test.go @@ -11,11 +11,11 @@ import ( "testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/stackdriver" ) func TestData(t *testing.T) { - config := googlecloud.GetConfigForTest(t, "storage") + config := stackdriver.GetConfigForTest(t, "storage") metricSet := mbtest.NewFetcher(t, config) metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index fc7d792dadf..392b64718d8 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -4,13 +4,21 @@ - module: googlecloud metricsets: - compute + region: "us-central1" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + +- module: googlecloud + metricsets: - pubsub - loadbalancing zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false - period: 300s + period: 60s - module: googlecloud metricsets: