From 485eea1591a3d3a620af5eedc678ea92c41a1109 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 3 Feb 2022 03:43:10 -0800 Subject: [PATCH 1/2] Add dataproc metricset --- CHANGELOG-developer.next.asciidoc | 1 + x-pack/metricbeat/metricbeat.reference.yml | 1 + x-pack/metricbeat/module/gcp/_meta/config.yml | 1 + x-pack/metricbeat/module/gcp/constants.go | 1 + .../module/gcp/dataproc/_meta/data.json | 41 +++++++++++ .../gcp/dataproc/_meta/data_cluster.json | 41 +++++++++++ .../module/gcp/dataproc/_meta/docs.asciidoc | 32 +++++++++ .../module/gcp/dataproc/_meta/fields.yml | 71 +++++++++++++++++++ .../gcp/dataproc/dataproc_integration_test.go | 59 +++++++++++++++ .../module/gcp/dataproc/dataproc_test.go | 20 ++++++ .../module/gcp/dataproc/manifest.yml | 30 ++++++++ x-pack/metricbeat/module/gcp/fields.go | 2 +- .../module/gcp/metrics/metrics_requester.go | 2 +- .../module/gcp/metrics/response_parser.go | 23 ++++++ x-pack/metricbeat/module/gcp/module.yml | 1 + x-pack/metricbeat/modules.d/gcp.yml.disabled | 1 + 16 files changed, 325 insertions(+), 2 deletions(-) create mode 100644 x-pack/metricbeat/module/gcp/dataproc/_meta/data.json create mode 100644 x-pack/metricbeat/module/gcp/dataproc/_meta/data_cluster.json create mode 100644 x-pack/metricbeat/module/gcp/dataproc/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/gcp/dataproc/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/gcp/dataproc/dataproc_integration_test.go create mode 100644 x-pack/metricbeat/module/gcp/dataproc/dataproc_test.go create mode 100644 x-pack/metricbeat/module/gcp/dataproc/manifest.yml diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 77740f117045..770019d0c8c7 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -132,6 +132,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Add support for `credentials_json` in `gcp` module, all metricsets {pull}29584[29584] - Add gcp firestore metricset. {pull}29918[29918] - Added TESTING_FILEBEAT_FILEPATTERN option for filebeat module pytests {pull}30103[30103] +- Add gcp dataproc metricset. {pull}30008[30008] ==== Deprecated diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index e34bef34c383..19479fb56f59 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -554,6 +554,7 @@ metricbeat.modules: - pubsub - loadbalancing - firestore + - dataproc zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" diff --git a/x-pack/metricbeat/module/gcp/_meta/config.yml b/x-pack/metricbeat/module/gcp/_meta/config.yml index 632e9e235381..30fe25e95b69 100644 --- a/x-pack/metricbeat/module/gcp/_meta/config.yml +++ b/x-pack/metricbeat/module/gcp/_meta/config.yml @@ -12,6 +12,7 @@ - pubsub - loadbalancing - firestore + - dataproc zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" diff --git a/x-pack/metricbeat/module/gcp/constants.go b/x-pack/metricbeat/module/gcp/constants.go index c8bd1812ff83..03773e38ca21 100644 --- a/x-pack/metricbeat/module/gcp/constants.go +++ b/x-pack/metricbeat/module/gcp/constants.go @@ -23,6 +23,7 @@ const ( ServicePubsub = "pubsub" ServiceStorage = "storage" ServiceFirestore = "firestore" + ServiceDataproc = "dataproc" ) //Paths within the GCP monitoring.TimeSeries response, if converted to JSON, where you can find each ECS field required for the output event diff --git a/x-pack/metricbeat/module/gcp/dataproc/_meta/data.json b/x-pack/metricbeat/module/gcp/dataproc/_meta/data.json new file mode 100644 index 000000000000..9daed1f80b20 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/dataproc/_meta/data.json @@ -0,0 +1,41 @@ +{ + "@timestamp": "2016-05-23T08:05:34.853Z", + "cloud": { + "account": { + "id": "elastic-apm" + }, + "provider": "gcp" + }, + "event": { + "dataset": "gcp.dataproc", + "duration": 115000, + "module": "gcp" + }, + "gcp": { + "labels": { + "metrics": { + "storage_class": "MULTI_REGIONAL" + }, + "resource": { + "bucket_name": "artifacts.elastic-apm.appspot.com", + "location": "us" + } + }, + "dataproc": { + "cluster": { + "hdfs": { + "datanodes": { + "value": 15 + } + } + } + } + }, + "metricset": { + "name": "dataproc", + "period": 10000 + }, + "service": { + "type": "gcp" + } +} diff --git a/x-pack/metricbeat/module/gcp/dataproc/_meta/data_cluster.json b/x-pack/metricbeat/module/gcp/dataproc/_meta/data_cluster.json new file mode 100644 index 000000000000..9daed1f80b20 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/dataproc/_meta/data_cluster.json @@ -0,0 +1,41 @@ +{ + "@timestamp": "2016-05-23T08:05:34.853Z", + "cloud": { + "account": { + "id": "elastic-apm" + }, + "provider": "gcp" + }, + "event": { + "dataset": "gcp.dataproc", + "duration": 115000, + "module": "gcp" + }, + "gcp": { + "labels": { + "metrics": { + "storage_class": "MULTI_REGIONAL" + }, + "resource": { + "bucket_name": "artifacts.elastic-apm.appspot.com", + "location": "us" + } + }, + "dataproc": { + "cluster": { + "hdfs": { + "datanodes": { + "value": 15 + } + } + } + } + }, + "metricset": { + "name": "dataproc", + "period": 10000 + }, + "service": { + "type": "gcp" + } +} diff --git a/x-pack/metricbeat/module/gcp/dataproc/_meta/docs.asciidoc b/x-pack/metricbeat/module/gcp/dataproc/_meta/docs.asciidoc new file mode 100644 index 000000000000..ff94b21fc363 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/dataproc/_meta/docs.asciidoc @@ -0,0 +1,32 @@ +Dataproc metricset fetches metrics from https://cloud.google.com/dataproc/[Dataproc] in Google Cloud Platform. + +The `dataproc` metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-dataproc[GCP Dataproc Monitoring API]. The field names have been left untouched for people already familiar with them. + +You can specify a single region to fetch metrics like `us-central1`. Be aware that GCP Storage does not use zones so `us-central1-a` will return nothing. If no region is specified, it will return metrics from all buckets. + +[float] +=== Metrics +Here is a list of metrics collected by `dataproc` metricset: + +- `dataproc.batch.spark.executors`: Indicates the number of Batch Spark executors. +- `dataproc.cluster.hdfs.datanodes`: Indicates the number of HDFS DataNodes that are running inside a cluster. +- `dataproc.cluster.hdfs.storage_capacity`: Indicates capacity of HDFS system running on cluster in GB. +- `dataproc.cluster.hdfs.storage_utilization`: The percentage of HDFS storage currently used. +- `dataproc.cluster.hdfs.unhealthy_blocks`: Indicates the number of unhealthy blocks inside the cluster. +- `dataproc.cluster.job.completion_time`: The time jobs took to complete from the time the user submits a job to the time Dataproc reports it is completed. +- `dataproc.cluster.job.duration`: The time jobs have spent in a given state. +- `dataproc.cluster.job.failed_count`: Indicates the number of jobs that have failed on a cluster. +- `dataproc.cluster.job.running_count`: Indicates the number of jobs that are running on a cluster. +- `dataproc.cluster.job.submitted_count`: Indicates the number of jobs that have been submitted to a cluster. +- `dataproc.cluster.operation.completion_time`: The time operations took to complete from the time the user submits a operation to the time Dataproc reports it is completed. +- `dataproc.cluster.operation.duration`: The time operations have spent in a given state. +- `dataproc.cluster.operation.failed_count`: Indicates the number of operations that have failed on a cluster. +- `dataproc.cluster.operation.running_count`: Indicates the number of operations that are running on a cluster. +- `dataproc.cluster.operation.submitted_count`: Indicates the number of operations that have been submitted to a cluster. +- `dataproc.cluster.yarn.allocated_memory_percentage`: The percentage of YARN memory is allocated. +- `dataproc.cluster.yarn.apps`: Indicates the number of active YARN applications. +- `dataproc.cluster.yarn.containers`: Indicates the number of YARN containers. +- `dataproc.cluster.yarn.memory_size`: Indicates the YARN memory size in GB. Sampled every 60 seconds. +- `dataproc.cluster.yarn.nodemanagers`: Indicates the number of YARN NodeManagers running inside cluster. +- `dataproc.cluster.yarn.pending_memory_size`: The current memory request, in GB, that is pending to be fulfilled by the scheduler. +- `dataproc.cluster.yarn.virtual_cores`: Indicates the number of virtual cores in YARN. diff --git a/x-pack/metricbeat/module/gcp/dataproc/_meta/fields.yml b/x-pack/metricbeat/module/gcp/dataproc/_meta/fields.yml new file mode 100644 index 000000000000..9131d0ba45a7 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/dataproc/_meta/fields.yml @@ -0,0 +1,71 @@ +- name: dataproc + description: Google Cloud Dataproc metrics + release: beta + type: group + fields: + - name: batch.spark.executors.count + type: long + description: Indicates the number of Batch Spark executors. + - name: cluster.hdfs.datanodes.count + type: long + description: Indicates the number of HDFS DataNodes that are running inside a cluster. + - name: cluster.hdfs.storage_capacity.value + type: double + description: Indicates capacity of HDFS system running on cluster in GB. + - name: cluster.hdfs.storage_utilization.value + type: double + description: The percentage of HDFS storage currently used. + - name: cluster.hdfs.unhealthy_blocks.count + type: long + description: Indicates the number of unhealthy blocks inside the cluster. + - name: cluster.job.completion_time.value + type: long + description: The time jobs took to complete from the time the user submits a job to the time Dataproc reports it is completed. + - name: cluster.job.duration.value + type: long + description: The time jobs have spent in a given state. + - name: cluster.job.failed.count + type: long + description: Indicates the number of jobs that have failed on a cluster. + - name: cluster.job.running.count + type: long + description: Indicates the number of jobs that are running on a cluster. + - name: cluster.job.submitted.count + type: long + description: Indicates the number of jobs that have been submitted to a cluster. + - name: cluster.operation.completion_time.value + type: long + description: The time operations took to complete from the time the user submits a operation to the time Dataproc reports it is completed. + - name: cluster.operation.duration.value + type: long + description: The time operations have spent in a given state. + - name: cluster.operation.failed.count + type: long + description: Indicates the number of operations that have failed on a cluster. + - name: cluster.operation.running.count + type: long + description: Indicates the number of operations that are running on a cluster. + - name: cluster.operation.submitted.count + type: long + description: Indicates the number of operations that have been submitted to a cluster. + - name: cluster.yarn.allocated_memory_percentage.value + type: double + description: The percentage of YARN memory is allocated. + - name: cluster.yarn.apps.count + type: long + description: Indicates the number of active YARN applications. + - name: cluster.yarn.containers.count + type: long + description: Indicates the number of YARN containers. + - name: cluster.yarn.memory_size.value + type: double + description: Indicates the YARN memory size in GB. + - name: cluster.yarn.nodemanagers.count + type: long + description: Indicates the number of YARN NodeManagers running inside cluster. + - name: cluster.yarn.pending_memory_size.value + type: double + description: The current memory request, in GB, that is pending to be fulfilled by the scheduler. + - name: cluster.yarn.virtual_cores.count + type: long + description: Indicates the number of virtual cores in YARN. diff --git a/x-pack/metricbeat/module/gcp/dataproc/dataproc_integration_test.go b/x-pack/metricbeat/module/gcp/dataproc/dataproc_integration_test.go new file mode 100644 index 000000000000..ce1a58821f7e --- /dev/null +++ b/x-pack/metricbeat/module/gcp/dataproc/dataproc_integration_test.go @@ -0,0 +1,59 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration && gcp +// +build integration,gcp + +package dataproc + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/metrics" +) + +func TestFetch(t *testing.T) { + config := metrics.GetConfigForTest(t, "dataproc") + fmt.Printf("%+v\n", config) + + metricSet := mbtest.NewReportingMetricSetV2WithContext(t, config) + events, errs := mbtest.ReportingFetchV2WithContext(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} + +func TestData(t *testing.T) { + metricPrefixIs := func(metricPrefix string) func(e common.MapStr) bool { + return func(e common.MapStr) bool { + v, err := e.GetValue(metricPrefix) + return err == nil && v != nil + } + } + + dataFiles := []struct { + metricPrefix string + path string + }{ + {"gcp.dataproc", "./_meta/data.json"}, + {"gcp.dataproc.cluster", "./_meta/data_cluster.json"}, + } + + config := metrics.GetConfigForTest(t, "dataproc") + + for _, df := range dataFiles { + metricSet := mbtest.NewFetcher(t, config) + t.Run(fmt.Sprintf("metric prefix: %s", df.metricPrefix), func(t *testing.T) { + metricSet.WriteEventsCond(t, df.path, metricPrefixIs(df.metricPrefix)) + }) + } +} diff --git a/x-pack/metricbeat/module/gcp/dataproc/dataproc_test.go b/x-pack/metricbeat/module/gcp/dataproc/dataproc_test.go new file mode 100644 index 000000000000..8d8ceafe47e5 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/dataproc/dataproc_test.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. +package dataproc + +import ( + "os" + + "github.com/elastic/beats/v7/metricbeat/mb" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/metrics" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/gcp/dataproc/manifest.yml b/x-pack/metricbeat/module/gcp/dataproc/manifest.yml new file mode 100644 index 000000000000..6d457c0efae3 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/dataproc/manifest.yml @@ -0,0 +1,30 @@ +default: false +input: + module: gcp + metricset: metrics + defaults: + metrics: + - service: dataproc + metric_types: + - "batch/spark/executors" + - "cluster/hdfs/datanodes" + - "cluster/hdfs/storage_capacity" + - "cluster/hdfs/storage_utilization" + - "cluster/hdfs/unhealthy_blocks" + - "cluster/job/completion_time" + - "cluster/job/duration" + - "cluster/job/failed_count" + - "cluster/job/running_count" + - "cluster/job/submitted_count" + - "cluster/operation/completion_time" + - "cluster/operation/duration" + - "cluster/operation/failed_count" + - "cluster/operation/running_count" + - "cluster/operation/submitted_count" + - "cluster/yarn/allocated_memory_percentage" + - "cluster/yarn/apps" + - "cluster/yarn/containers" + - "cluster/yarn/memory_size" + - "cluster/yarn/nodemanagers" + - "cluster/yarn/pending_memory_size" + - "cluster/yarn/virtual_cores" diff --git a/x-pack/metricbeat/module/gcp/fields.go b/x-pack/metricbeat/module/gcp/fields.go index d9d6047873ff..f1fc860fc0c1 100644 --- a/x-pack/metricbeat/module/gcp/fields.go +++ b/x-pack/metricbeat/module/gcp/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGcp returns asset data. // This is the base64 encoded zlib format compressed contents of module/gcp. func AssetGcp() string { - return "eJzcXVtvG7mSfs+vKMxLkoWj7Jld7EOwOEDGOTMbbDxjwEleeyh2SeKITXZ4saP8+gPe+iK1rFZf5GSSJ1vqqq+urCqS7Vewxd0bWNPyGYBhhuMbeP6blGuOcM2lzeGWE7OSqnj+DEAhR6LxDSzRkGcAOWqqWGmYFG/gn88AAH67voVC5pbjM4AVQ57rN/6DVyBIgYmV+2d2pftZSZt+0/x+8xlOlsh19ev0qFz+hdQ0ft2BJ/0LuAQzUjGxhgKNYlQfUt6H0IRhNarFf7Q+OgrF/Qu/zMI3trh7kCrvJFygITkxZC7iTtRZaOudNljMQlqhllZRnIx4IvxTpZDw/6fTftWim0u79N7d8WlWkLJkYh2/+lOL+CPeeRPd0WyIAYXGKoE5rJQsoBWMb2/fwxeLarc4EGvJOGdifYxfi8wv4bvJNRrP7Ed4WzHNWIXOYEloqNRBI88OTddl9RbWa6mN/64GJii3OYLCteVEXYEhX6+A5H9ZbQoU5gqIyEFJK3KndlRKqkUHHibuJaOYFVKYzRBMSWUKS6kMeDpdjEolvTewfAiX2/A0vH8HcgVmg8msie8SuRRrDUZ2MTfSEN7Bd8UlMce5fnSPVZxIIa0whw5GZVFag70c7Dp8d0YHWzGFD4TzRa5kWWK+WO4M6g7Znb6Oi/5eUFk4qf3jEInBcueVn5j04J+VhG7R6Ix67d0Tbrt8vyeaSOwMPExoQwTFBS3tQqFGdY95RqVCfRTMQSbbg/O7LZaonCd6OpDIghQezsbFafTTxP8UNKvJGjPDClxopANAfXIEYCUVEM4jMCZAI5Ui173YL8q9laMf54/OAopQ91MSm3AuKTGYw/Xtp5C+mQZqlUJh+M4hsxqTwvooKWd6u1BIhnr0tfM/By94tKMUFhJHuBfjTJbj3LiC4EgGBO//AFmiIu4LjxrJo3hQzOA08jtSBgUY2U8BnvXEGvA0+6ugwEKq3WLpfEuKhSJFptk3HAjFea1fF2Jid6gCB+eczis/3yzg44bpmK2dA0vBd0DuCeNkyUO0fb6J5UlYgpxC3cP4M6xIwfiue9U9LpLVmA8U6SbAr6PM0Xo6afQDKTMmBvqrs8+BZXzMMBFRrS1qE4KYGQ3yQYDjCbokFJ9EWmnNlOKmIPUi1hIb+RTyCjQPUm0XTKwVaj1VGqbI7lNN78BENucgiVXBwmem4YhScTEKE06oHI3up3tUA0FMrJez4djykXrmceZ1iRULmJjDYEM0LBEFKCsEE+tHXTYAyHyaHwTjX5yULoc6MqCZoJhwPBAN2hBlML9q1FkLuCNFyTEHvEe1g//5z/qTtyuDCrT7nIn1FeTEEBeoQhq4Z5qlKLWlC8x//Fw/etBzuIJXG6n6dR2/pm/P0HfUmHJJrWs9FzlyNNjhdUd13pELRe0AllLUemV5xQICC714HIivFueE4RjokCa+WFQMNUgFXMqtLU+BC7XcnOg8hw7nWW+Pus3zP9db/BOoFIYwEYKuaM5gHhA0VaSspjDXt3BnCN3mirnM8Pb2fXraD1IOJ4vOw91Tv/3/v57PMmDxzFH5tsa1Qdno1uraFpYTw+7RdzOenmtd6kbLF1qxGa0Q9EsLJ2XgrGBmZMfqYAeknlrq0mpGo/FZwzj75ov4CdtIhzsg7ttCxjLIKwooES654leKmMM/gOhovPYH7nnPZbrs/fN/n6FBhV9cXTfZVKK2dqTc4ZwzL1SnRJ3HXWpxz5k67LsMLBHWConxlQ4Re47T9JrI8An8BssNFqgIz9zKTtYY4nBg4flBUsKhogmRZow9JkJBOjRLHGJNepsWbaQ6A17flE8LNvjTWKixCR1j+zg16F4YJgU4U9QXTQF+sHWid/KMeixdKbMilpuBXV29SpR+XuxI6StYKrlFAbl8EH6p2JV4BQX5Syq/kVQw0b1/dABwXGTfpDFPCOQh3jizAS61gEWn/luvYVGlI5Jr9JdpcqlrkokaH1iu09B7jrshfpsoDAyewHsfHcWc9FRWYKOZCU55KF9rLHOWDYTMwzZU3DYiS46TVcMNmo3KOMaP5zwe6zzZYB95/2RwoVCudHHRPrthusEtdoXczwZH+lo4JyA6+q9xXnZYhzY8YuJyNEycmx43NXbmfq+zlcKhW3a/KmzOvAJBh5N3CzUl7DBDHrjVuOcflwEeEA/cjfKIj3nJ1J4xQ3cVgDbHcgPRxhplfOA1klus7WbEeIEFaWSB2qff6mB32eq/qd0x4VQXIzGAqh3e6RLu+GJ6Rg9NG5NpazULdhy4MVnjFHuKrbZuG3APtixH4NcozFzYjSJCF8yY6eGXLM/C4GL4WZmCfIXbcP7yj7uRvurwjDjn0t76ih0HlEpS1DqddxkLMssJFlI82TaStz3He+TxNDkEQKNK3iTUxJ13hHyAc0A7Xsr8IFuMz2en8kQp85Fx1sTtssT0mDvywzSw7yW3xRT1Yo3YH++LLUV1AimeNnIsL7WAN8Qb4eu/d8nVjNXHRTqBa5ZKLRDvqM6W6HJlE/xjFdrjU8S90Xg8TVnZO2KIp9Muci6GS5IvCSeC9r3y8UGSHH5Jj8x4MH9jTKkXS0K3KPJs3Ji8vQA2DouRass3Hkv5v48fb1/feb1AUIwzpYSIo9M5u5EOq3f2luqELZ6XX+4qIO7jLrB9AOpSCj20L3tcl4F0VGaF9YVUQAnd4EunS/xqUAnCPf4Xdy/7CnApH6CcoTDaQT1PwzOb/lwwlzLzsZiJeuyCyP9rkbxg3AnQozA9ssrTPl7fvv707jat+XtYo5/WmNOqsOLyYQG/SuUI+J80MPNcg7dwfd3HFaVlyRn1yxJoo5AU/kBxP+GzcSdP20poHT+dTg0nBBl3yvlxM0Y0zdTRS5J5TRclntl2g2XvRs/E/FH3/sMvHb70YjXCFi/7iTOzLboFOx0kFcrLBEkT5rxav3QINCTrvLZKy0xrnpVKft0tKJfaX10UAn2tf3zDqm8706CVjvcqBIOqYMJf3vPNpYvPu7sPEGCcxDkqEvcnpbXWPt80fNTqOPHpA2iclx5HVNvx8815iAQ+XMCO1Hduw4woSxQTQLwOfWcjHKQ1rt3MnbbasJW0641PPUewvjrai/l3dRy2bt0t2pDb2cdewFHapbbLXsRv7fLOLmdsMLUgpd5I4xsjLtejRjnuWdDsG9aH7LRr+12x7I98+HuFpGbaA1AYnGfLXaZwzeTgi4GDAB4ccQtWiUgeQ0+lWLF1ZsucTDL3p+leVSBsw81XoBsi1qivgsVDh1Tdi/U8wisdUFv+uLqFLbKkjdGpZYzdm0AaZp8Z0nBLS56jNglyRtbD7q69XSO8qKf0L5OHBvIJ/hCFHgJsqPXiUM9XtF1WvBeEbpMgE8ZU5RuEboV84JivQyi9rX+uZhGtWMuRMz+adKxPop8lx1rRQl3J8oIstguygMi0+uBltEcT2EngO4MZlYPHTi2lhyO69e39KyiQaKuCp8TdHz9VZnFgbmT6Cnyx0hBojr9PYb9sJvYzZyR00wIxIj83RcmR5BlHY1DNGQWlXXKmN0HxjicEnmBkyWhLlp7AC5lnLnQdMc4Ezon+YSM1QuLkr9wG23vANzJnq91bun2XvjBBXB8TL6tMPaGghxKkhNsMqQlsNG6Q2w96p/J7AnaFQqMxmKF6iV4QAqGdMp+7MNEbQJGXkgm3rFnjN592aFrrSC85rKh4TSfHJVaGWFyktd55UC3CXMVQl1j75cakQoyuQ6cVaFwNFaX13jG/rbqd8JgPjnDBDnnmNtsR2Sa2V2k5zxqF7yyrSkOQnuuJ31YgkOOKCZZOEXQ9qrExzXhdjzNaUr4+vYbGd0j1Vtf06xc51Q70XLk8xDmt6Rj8Hcw4vQm9ZsbZTm9mwaU3QIzBouzGBZ8EZ1v0AuircIDIPeN3xhWwouRYoDChs8glhqMwS2Loxr9rt1rmF3AnQ4tS3ZgUfFe/vkMKbD2w8HsTTWbKGT5smvh3hDrvcAXSmt2jaD3rrw2SskSioLDcsJJjuOp2lqIzTgwKygYXRe+YNootbXJwL00SvyLuc33BqJIp4Q/xkHYZMW2z313QjSh99mqeWYet54Ift2xqxDnWAEf20UA9jUuYOftfv9VUKSvlaiMTCtccxEMpI1resEHq2q6LlCd3id2ty99/n2KlQ40zuGxi0l794IGZDQgpXjlf3rW0yvJhvt0WZ06P2BPqUm7wv1Tm+M9BznCu8i45VGrH1+GcZtxo6ZRkT+byh4KGndIJhKxOHs4oVDyAOARt6pZnXfCPDaDGr/R+Ej1+W2LcXkRjF8KP/f3eRGdhG+D+KHvCAW1KN86gQ4/BHJbdfmehSrmetq+6Pf2XR8H8CBO6uDsyyJO/zxnWGIm+t8ZijCwaRT7zvKYdFz9MMdvQzxwzkaCVM8c1EdP3vhwHmE+/Dp8TGFUpEV6Y8OwIjNYRtrv4coX5zrCRko28dfIOuSG1Zf37aImfszVM6TTpPinQbGTueaeF1BseXGvQZWlizebbglCeLYnGPIt/4oj4l+9miykQVwVu/JtHzum8eUX8g0rxCt9aEWEwh8AbtOTId5Bbf88yfvPt9YfOIqYWo84xA/Ud/uyI0/T1h0bG6nqh3HEkUY26RMpWjGYOWWHNmNy8p9V0pqMgeVNBieNRTU10FXkPzd4l5P0rvNP46gS3kTthd74GfhrI6YUwwTpDj9/v3UUOxDSUqGBp6RZNC2x6YwzlROvWHVj/1xRiCyEFRU8hJ7vwt7X8Ldv0PYVlOGtNTDwyFG+zgr9bcE94OuAsbXh1Wk46T2G3Xt7jl5IsXXb9cTxvujcQpaWPcF7ZMb7f4Tsz5b8DAAD//5UOSUI=" + return "eJzcXVuPG7eSfvevIPJiezHp7Mku9sFYHMAen+QY68kZ7NgB9qnDJksSLTbZ4UVj5dcveOuL1Bq1+qKJ47xEo27WVzeyqlikvkdb2L9Ba1K9QMgww+ENevmzlGsO6JZLS9E9x2YlVfnyBUIKOGANb1ABBr9AiIImilWGSfEG/f0FQgj9fHuPSkkthxcIrRhwqt/4L75HApeQSLl/Zl+5z0ra9Jf28+13OC6A6/rP6VVZfAFiWn/uwZP+BVyCGamYWKMSjGJEH498CKENw2pQ2b91vjoJxf0Lf8zDE1vYP0pFewcuwWCKDV5qcMfqImPrvTZQLjK0Ai2tIjDb4Gng72qBhP++O29XnXGptIW37p5v8xJXFRPr+Oh3ncGfsM67aI5mgw1SYKwSQNFKyRJ1nPHt/Qf0uwW1z47YKhjnTKxP0esM8y48m0yj9c6hh3cF0/ZV1OssCQ2ROkjkxbHq+rTewXortfHPasQE4ZYCUrC2HKsbZPDXG4TpF6tNCcLcICwoUtIK6sQOSkmV9eBhYicZgbyUwmzGYEoiU1BJZZAfp49QpaS3BkbHULkPb6MP75FcIbOBpNZEtwAuxVojI/uIG2kw76G74hKb01Q/uddqSriUVphjAyOyrKyBQQZ2G55d0MBWTMEj5jyjSlYV0KzYG9A9vDt5nWb9gyCydFz711EcDBV7L/xEZAD9vMJkC0bnxEtvh7nts/2BaOJgF+BhQhssCGSkspkCDWoHNCdSgT4J5mgmO4Dziy0LUM4S/TgoDYuk8HA2zk+jnSb656BZjdeQG1ZCpoGMAPXZDYBWUiHMeQTGBNJApKB6EPmsOlg5hlH+5DSgMHGfEtuYc0mwAYpu7z+H6ZtpRKxSIAzfO2RWQxLYECFRpreZAjzWom+d/Tl4waLdSGEhcQMPIpzLapoZ1xDckAHBh38hWYHC7oEnleRRPCpmYB7+3VAGBDJymAA86Zkl4MccLoISSqn2WeFsS4pM4TLX7A8YCcVZrV8X4sTuUAUKzjidVf56l6FPG6bjbO0MWAq+R3iHGccFD972610MT8IS5ATqXoYf0QqXjO/7V93TLFkNdCRLdwF+42VurOfjRj/iKmdipL06/RxpxvsMExHV2oI2wYmZ0Ug+CuRoIl1hAs/CrbRmTnaTk3oWG46NfA5+BZhHqbYZE2sFWs81DRNguxTTOzCRzCVIYlSQ+ZlpPKIUXEzCBDMKR4P7tAM1EsTMcrkYjq2eiGeeJt6EWDGAiXMY2mCNCgCBlBWCifWTJhsA5H6aHwXjHxxXbg51wyDNBIGE4xFrpA1WBuhNK87K0AMuKw4UwQ7UHv3XvzffvF0ZUEi775lY3yCKDXaOKqRBO6ZZ8lJbOcf824/Nq0c5h3u1UpIMSjrex4cXyDpaaTY2ZJPpCjvz+wrEGqn6LO+k3I9CfspcABkUL2pzeOfooAdHBzV0eiERbrUBlW3oSmdOYkJSWALTP9//9OCl/IsjEGZYrCCZqLNHRgHhGtF5uNpI5RICgitMmNn3RARPhOcncafhatShUFUjlSLBcDb987sLkFrDOPvDR3GTwLqFsAJFQBiX0NRAA5mD4GYAPis2gLnZ7POCS7JdQv81CRRIJIW7pwap/IssMiLdvOEIhSzwtAzPwfSRrZuwvshCIyPl1s0ncXxoVjX/jPsfq928ZIvSRRXYvZYiA/9IPX+EWotGLCRzccAzSnC8UavOGcZlTG3wDpCu3ILEBMJozXYg3Hxs4DyaFWYc6AJ2EOTtvN/jC3ScTw30fAcuLWtLomvPTZehC0ZiriE9v8jX9JxBDsRZJ5NLuFSTqY5wrPrlGd2r4XZOJ2uxOdrVGmSLOVxbG+PdrgG6nPMdIh3ngg3SJR2xV6qj3XGPlcjqcmAeMtu8WeFnDhb+7+3//lLXcXRThxwCsqqWiA4wMWwHARiuKu4ecsIdgIhIYTATsEgk7QG1KJyHE3V3oug2Jh51iNoac0MPiTw9HBfOl1jg9YLycRH9XaRxGM8PN/8KwjbcXAJ0Zh8D4aYy9rsFbW6C9G7qknsk7Ry2ALSyfMU4b3ZONNkAtXwQFzumjMU8bp7ML/A4frN34TRwnPuumAKXDQzbcfspPb1o9kslsSUIk1Fwa/YE4XzqiERbQkDrleU1CRRInPDXGojfKVkShiOgQ8zzuwXFQCOpEJdya6tz4MI+xpLoPIWewsl6e9JsXv623sJvaUYMxlm2+w8enbsoXNUdCLf36MFgsqWK7UD5JoT4tm8iOO6qWUnl3/r5f/7xckY7bHai42Tut/ScG+WTtxVvbWk59ivY7f1n5MfzEUu9yeg3GeJ0UiMYVhI7ywNnJTMTd2sd7IDUj5Z2KBtCk/G1Sx/zbaE63AHx0O3TuAXgBYUIFkIaBF8JAEV/Q1hH5XW/cO97KvNVLn/8zwskGNet2XbkG23HkXuMc+Ei7TlWlzGXht1LdtwPTcYFCWsF2PgqPxYHhtO2mkjwGewGqg2UoDDPY1Uw+OHITZePkmCO6jHrSmPwPSbCZszYWeIYa5LbvGjjqAvg9XXWecEGe5oKNW7ATtF93DHvXxhmBbiQ15dtBr6xdWLw5BnlWLlQZoUtNyN3NJtVovK9Um4ofYMKJbcgEJWPwi8V+wpuUIm/SOWbKEsm+nsnjwBO8+y7TiI3yhoXVsC1FrBo1H/pNSyKdMLkGu1lnrnUJclYTXcsl2noA8PdYN8iGTbLn8F6n2xDOGuprIRWMhOM8pi/TkvCRToQkoYWzFiqxAWH2aLh1pityDj6j6c8Hesys8Eh8uGTwZVcuZbFVfPslupGp9g1ct8XM9HWQo+86Mm/plnZcRzasoiZw9HQbdW2uLmxM996kq8UjG1X/UlBu+YVBnQ4eT9Tc8IO/VMj22wP7OM6wAPikZ2YHvEpK5nbMhbIrgLQdlluJNoYo0x3vNbkFmO7BTFeYUGaGKAOybd6yF03+m9Ld4o7NcFIdKC6u3m+CXd6ML2ghaam3NRWnAc9jmzKbXCKA8HWbcstuEftuhPwaxBmKexGYaFja8HM8CtG81C4GH9OpMRf0X04e/ivh4m26vBMOOPR3fpKO9KVkgS0Tmc9poLMKYZSimfbRvK657ADnhpUA6BJIW9iaubMO22kH+IckY5Xkh7NFtPns3PzRCXpRD9r43azxPyYe+aHeWDvJLflHPFig9gfbYspRX36Jjb8OZLXWsBb7E2w9V/6+Gr76tMsncG1SKQWBu+Jzgpwc2Ub/FMR2tNVxIPSeDxJWOs7Yogns65yJoRLTAvMsSBDrzv4KDFF79IrCx5K3xhT6azAZAuC5tPK5N0FsHVQCtdbvrEt5Z+fPt3/8ODlgoJgnColijh6jbMf6bh452CpTtjiWfFiXwPxhxx6wA4BqCsp9Ni87GlZhqGjMGusr6RCBJMNvHayhK8GlMDc43/18HooA9eyAcIZCKMd1MskvLDqLwVzLTWf8pkoxz6I/D+yZAXTTj+ehOmR1Zb26fb+h8/v79Oaf4A12mmDOa0KKy4fM/STVG4A/0kjZl5q5DXcXHXhgtKmSxdpowCX/jDtMObzaacuu0LoHL2cTwxnGJl2wvdpNUY07aljECfLqi5yvLDuRvPej56J5b3uw8d3Pbb0ajVBF6+HsbOwLvoZO+8kNcrrOEkb5rJSv7YLtDjrvbKJVLnWPK+U/LrPCJfaX9sjBJBwemJc+aR9dU89VmrvVYAMqJIJf3GNTy6dfz48fEQBxlmckzzxsFLaSO3Xu5aNWh0rPkMATbPS04gaPf56dxkiAY9X0CPxmds4JcoKxAwQb+MBjdbBKmtcuunPY3RhK2nXGz/1nMD6/clczN9TeZy69adoY24mO3X5ZGULbYtBg9/b4sEWCyaYWuBKb6TxiRGX60mlHPduOIlUN9lpl/a7YNm3fNBwErImOgBQKJznxT5XsGZy9KU4owAetbgFrUQkT6EnUqzYOrcVxbPU/Um6UyQMHE+sIrLBYg36Jmg8ZEitw7L7CsJ1hqAtf1rcwpZ5ksbkqWWK3ttAWmpfGNJ4TUtOQZsEOcfrcfe2vF0DetVU6V8nCw3DJ/hjBHoMsCXWq0O9XNC2qGlnmGwTIzP6VG0bmGyFfORA18GV3jaf61pEx9cocOZLk470WfSLzLFWdFDXvLzC2TbDGYpE6y9eR320gZ0FvjeQEzm67NQRemjRbc5K36ASsLYqWErc/fFVZRYL5kamR9DvVhqM2uXvc9ivOxP7mjNgsumAmDA/t1mhgGnOwRhQS3pBZQvO9CYI3tFEgSYysmKkw8tA4KWkuXNdNxhnApZE/7iRGlCi5K+bCrr3gO8kZav9W7J9nx6Ywa9PsZe3b9SYi9FjDtKE23apGXQ0rZA7DHqv8AcCdoFCKzFYIHqJVpBubWhTf+ncRG8QCFpJJtyyZo3ffNqD6awjg/iwoqY1Hx/XWBlicJHWemdBDQtLBUN9bB2GG7MyMTkOnZehaTFU5NZbx/K66jfCUzY4wQR7+FlabSd4m1lfleU8bwW+i6wqLUYGrid+WwEjCismWOoi6HtVQ6ua8UNTzuhw+cP5NTTenzxYXPOvX/hcOjBw5fIQl9SmI/BXUOP8KvSSmaY7vVkEl94gbAyUVT8u9FlwtgXPgL4JDUTuHb8zrhArKw4lCBMyCyohtML420XDdWp1XIEeZEhR6hOTgu+b6zukgM4LWbgsqkVMOcWHTRP/+xjOOmR9w1n7XX9sEFcVYIVKyw2reLgorXdD/KSgc44NCMJGB0XvmTaKFTYZuOcmsV8P7uf6khEl04Q/xkK6YcS8yX5/QDch9DmIeRYttl4KftqyqQGWWAPcsE866nlcwiyZ//qtplpYaa42MqFwyUFsSpmQ8oYNUpd2XSU8eUjk7t38/dcJVnrEuIDJJiLd1Q89MrNBQorvnS3vO1JldJxtd9lZ0iIOmLqWGfw3kRT+PsoYLhXeNYtKXf86rtNMKy2d4+zZTP6Y0bBTOgOTdefhgkzFBsQxaFO2vOiCf6oANX2l95Xo6dsS0/YiWrsQvuzv9yZ6A9sA91vZEw5o03TjFDq2DeY47PY7C/WU68f2Ubcf//VJMN9ChS7ujoyy5D9nDWsKR3+2xGIKLxoEXbhe0/WLbyaYbclniZpIkMqF5ZqI6c++HAeYz78OX+IYdSgRLkx4cQJGp4XtIV6usFwPG67YxFMn74Eb3GjW30eLfZ2tpUonSfdNCWYjqaedFlKveORSgz5NY2s2f2SY8LzAGmgef94X+8t382wOxHWAG3/v1xmdV6+IPyYcj/CtFRYGKAq0kZYc+B5R689Zxiff3n7sDWIaNpo5ZqS8w09uOknffmzfmt9zodxpJFGMugLCVozkDllpzZS5+UCqqaejxLQtoETxpKRmOop8gObgEPLhEd55bHWG08i9sHt/Am0eyOlCmKCdse33B2eRw2AaVaBQYckWTAds/aNKHGvdOQPrf0kwphBSEP8rC4jiffhdaX/KNj2noAq91tjElqF4mhX5swU7zFODs7Th6jSKe7uwO5f3+KUkT4ddvx3Lm+8GorT0Yc5rPcb7Hf5kqvz/AAAA//8lUd33" } diff --git a/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go b/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go index d25701236e42..62b2b82dbd7a 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go +++ b/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go @@ -124,7 +124,7 @@ func (r *metricsRequester) getFilterForMetric(serviceName, m string) (f string) } f = fmt.Sprintf("%s AND resource.label.location=starts_with(\"%s\")", f, zone) } - case gcp.ServicePubsub, gcp.ServiceLoadBalancing, gcp.ServiceCloudFunctions, gcp.ServiceFirestore: + case gcp.ServicePubsub, gcp.ServiceLoadBalancing, gcp.ServiceCloudFunctions, gcp.ServiceFirestore, gcp.ServiceDataproc: return case gcp.ServiceStorage: if r.config.Region == "" { diff --git a/x-pack/metricbeat/module/gcp/metrics/response_parser.go b/x-pack/metricbeat/module/gcp/metrics/response_parser.go index e0f968f77a55..e4d6399f7825 100644 --- a/x-pack/metricbeat/module/gcp/metrics/response_parser.go +++ b/x-pack/metricbeat/module/gcp/metrics/response_parser.go @@ -218,6 +218,29 @@ var reMapping = map[string]string{ "document.delete_count.value": "document.delete.count", "document.read_count.value": "document.read.count", "document.write_count.value": "document.write.count", + // gcp.dataproc + "batch.spark.executors.value": "batch.spark.executors.count", + "cluster.hdfs.datanodes.value": "cluster.hdfs.datanodes.count", + "cluster.hdfs.storage_capacity.value": "cluster.hdfs.storage_capacity.value", + "cluster.hdfs.storage_utilization.value": "cluster.hdfs.storage_utilization.value", + "cluster.hdfs.unhealthy_blocks.value": "cluster.hdfs.unhealthy_blocks.count", + "cluster.job.completion_time.value": "cluster.job.completion_time.value", + "cluster.job.duration.value": "cluster.job.duration.value", + "cluster.job.failed_count.value": "cluster.job.failed.count", + "cluster.job.running_count.value": "cluster.job.running.count", + "cluster.job.submitted_count.value": "cluster.job.submitted.count", + "cluster.operation.completion_time.value": "cluster.operation.completion_time.value", + "cluster.operation.duration.value": "cluster.operation.duration.value", + "cluster.operation.failed_count.value": "cluster.operation.failed.count", + "cluster.operation.running_count.value": "cluster.operation.running.count", + "cluster.operation.submitted_count.value": "cluster.operation.submitted.count", + "cluster.yarn.allocated_memory_percentage.value": "cluster.yarn.allocated_memory_percentage.value", + "cluster.yarn.apps.value": "cluster.yarn.apps.count", + "cluster.yarn.containers.value": "cluster.yarn.containers.count", + "cluster.yarn.memory_size.value": "cluster.yarn.memory_size.value", + "cluster.yarn.nodemanagers.value": "cluster.yarn.nodemanagers.count", + "cluster.yarn.pending_memory_size.value": "cluster.yarn.pending_memory_size.value", + "cluster.yarn.virtual_cores.value": "cluster.yarn.virtual_cores.count", } func remap(l *logp.Logger, s string) string { diff --git a/x-pack/metricbeat/module/gcp/module.yml b/x-pack/metricbeat/module/gcp/module.yml index e07d9656454d..41bbb8759051 100644 --- a/x-pack/metricbeat/module/gcp/module.yml +++ b/x-pack/metricbeat/module/gcp/module.yml @@ -6,6 +6,7 @@ metricsets: - storage - gke - firestore + - dataproc dashboards: - id: Metricbeat-gcp-gke-overview file: 1ae960c0-f9f8-11eb-bc38-79936db7c106.json diff --git a/x-pack/metricbeat/modules.d/gcp.yml.disabled b/x-pack/metricbeat/modules.d/gcp.yml.disabled index 638846d0b96e..43a0c84aa59b 100644 --- a/x-pack/metricbeat/modules.d/gcp.yml.disabled +++ b/x-pack/metricbeat/modules.d/gcp.yml.disabled @@ -15,6 +15,7 @@ - pubsub - loadbalancing - firestore + - dataproc zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" From 86541f9825d2b63ac90660c76a5b5efb6ce8471d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 3 Feb 2022 03:44:00 -0800 Subject: [PATCH 2/2] Update metricbeat docs --- metricbeat/docs/fields.asciidoc | 204 ++++++++++++++++++ metricbeat/docs/modules/gcp.asciidoc | 5 + metricbeat/docs/modules/gcp/dataproc.asciidoc | 24 +++ metricbeat/docs/modules_list.asciidoc | 3 +- 4 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 metricbeat/docs/modules/gcp/dataproc.asciidoc diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 1519dfc20878..6be1c217c9fd 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -34857,6 +34857,210 @@ type: long -- +[float] +=== dataproc + +Google Cloud Dataproc metrics + + +*`gcp.dataproc.batch.spark.executors.count`*:: ++ +-- +Indicates the number of Batch Spark executors. + +type: long + +-- + +*`gcp.dataproc.cluster.hdfs.datanodes.count`*:: ++ +-- +Indicates the number of HDFS DataNodes that are running inside a cluster. + +type: long + +-- + +*`gcp.dataproc.cluster.hdfs.storage_capacity.value`*:: ++ +-- +Indicates capacity of HDFS system running on cluster in GB. + +type: double + +-- + +*`gcp.dataproc.cluster.hdfs.storage_utilization.value`*:: ++ +-- +The percentage of HDFS storage currently used. + +type: double + +-- + +*`gcp.dataproc.cluster.hdfs.unhealthy_blocks.count`*:: ++ +-- +Indicates the number of unhealthy blocks inside the cluster. + +type: long + +-- + +*`gcp.dataproc.cluster.job.completion_time.value`*:: ++ +-- +The time jobs took to complete from the time the user submits a job to the time Dataproc reports it is completed. + +type: long + +-- + +*`gcp.dataproc.cluster.job.duration.value`*:: ++ +-- +The time jobs have spent in a given state. + +type: long + +-- + +*`gcp.dataproc.cluster.job.failed.count`*:: ++ +-- +Indicates the number of jobs that have failed on a cluster. + +type: long + +-- + +*`gcp.dataproc.cluster.job.running.count`*:: ++ +-- +Indicates the number of jobs that are running on a cluster. + +type: long + +-- + +*`gcp.dataproc.cluster.job.submitted.count`*:: ++ +-- +Indicates the number of jobs that have been submitted to a cluster. + +type: long + +-- + +*`gcp.dataproc.cluster.operation.completion_time.value`*:: ++ +-- +The time operations took to complete from the time the user submits a operation to the time Dataproc reports it is completed. + +type: long + +-- + +*`gcp.dataproc.cluster.operation.duration.value`*:: ++ +-- +The time operations have spent in a given state. + +type: long + +-- + +*`gcp.dataproc.cluster.operation.failed.count`*:: ++ +-- +Indicates the number of operations that have failed on a cluster. + +type: long + +-- + +*`gcp.dataproc.cluster.operation.running.count`*:: ++ +-- +Indicates the number of operations that are running on a cluster. + +type: long + +-- + +*`gcp.dataproc.cluster.operation.submitted.count`*:: ++ +-- +Indicates the number of operations that have been submitted to a cluster. + +type: long + +-- + +*`gcp.dataproc.cluster.yarn.allocated_memory_percentage.value`*:: ++ +-- +The percentage of YARN memory is allocated. + +type: double + +-- + +*`gcp.dataproc.cluster.yarn.apps.count`*:: ++ +-- +Indicates the number of active YARN applications. + +type: long + +-- + +*`gcp.dataproc.cluster.yarn.containers.count`*:: ++ +-- +Indicates the number of YARN containers. + +type: long + +-- + +*`gcp.dataproc.cluster.yarn.memory_size.value`*:: ++ +-- +Indicates the YARN memory size in GB. + +type: double + +-- + +*`gcp.dataproc.cluster.yarn.nodemanagers.count`*:: ++ +-- +Indicates the number of YARN NodeManagers running inside cluster. + +type: long + +-- + +*`gcp.dataproc.cluster.yarn.pending_memory_size.value`*:: ++ +-- +The current memory request, in GB, that is pending to be fulfilled by the scheduler. + +type: double + +-- + +*`gcp.dataproc.cluster.yarn.virtual_cores.count`*:: ++ +-- +Indicates the number of virtual cores in YARN. + +type: long + +-- + [float] === firestore diff --git a/metricbeat/docs/modules/gcp.asciidoc b/metricbeat/docs/modules/gcp.asciidoc index 86b6239ae2ab..ff481025ed97 100644 --- a/metricbeat/docs/modules/gcp.asciidoc +++ b/metricbeat/docs/modules/gcp.asciidoc @@ -276,6 +276,7 @@ metricbeat.modules: - pubsub - loadbalancing - firestore + - dataproc zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" @@ -340,6 +341,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -356,6 +359,8 @@ include::gcp/billing.asciidoc[] include::gcp/compute.asciidoc[] +include::gcp/dataproc.asciidoc[] + include::gcp/firestore.asciidoc[] include::gcp/gke.asciidoc[] diff --git a/metricbeat/docs/modules/gcp/dataproc.asciidoc b/metricbeat/docs/modules/gcp/dataproc.asciidoc new file mode 100644 index 000000000000..e289049ca43e --- /dev/null +++ b/metricbeat/docs/modules/gcp/dataproc.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-gcp-dataproc]] +[role="xpack"] +=== Google Cloud Platform dataproc metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/gcp/dataproc/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/gcp/dataproc/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index e4f6d15aea9c..5e2952bc3ed0 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -120,8 +120,9 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.8+| .8+| |<> beta[] +.9+| .9+| |<> beta[] |<> beta[] +|<> beta[] |<> beta[] |<> beta[] |<> beta[]