From 94101489e40c9864ffe18f1884c060ad448d3c17 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 27 Nov 2018 09:21:24 -0800 Subject: [PATCH] Index cluster.id, node.id, and node.name in elasticsearch/node_stats metricset (#9168) This PR teaches the `elasticsearch/node_stats` metricset to index the Elasticsearch `cluster_uuid` as the module-level `cluster.id` field, as well as the node ID and node `name` fields as the metricset-level `node.id` and `node.name` fields, respectively. --- metricbeat/docs/fields.asciidoc | 30 ++++++---- .../module/elasticsearch/_meta/fields.yml | 10 ++++ metricbeat/module/elasticsearch/fields.go | 2 +- .../elasticsearch/node/_meta/fields.yml | 4 -- .../elasticsearch/node_stats/_meta/data.json | 56 ++++++++++--------- .../module/elasticsearch/node_stats/data.go | 35 ++++++++++-- .../elasticsearch/node_stats/data_test.go | 2 +- .../elasticsearch/node_stats/data_xpack.go | 10 +--- .../elasticsearch/node_stats/node_stats.go | 14 ++++- 9 files changed, 104 insertions(+), 59 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 8f6afd25871..1faa2447212 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -5389,6 +5389,26 @@ type: keyword Elasticsearch state id. +-- + +*`elasticsearch.node.id`*:: ++ +-- +type: keyword + +Node ID + + +-- + +*`elasticsearch.node.name`*:: ++ +-- +type: keyword + +Node name. + + -- [float] @@ -5928,16 +5948,6 @@ node -*`elasticsearch.node.name`*:: -+ --- -type: keyword - -Node name. - - --- - *`elasticsearch.node.version`*:: + -- diff --git a/metricbeat/module/elasticsearch/_meta/fields.yml b/metricbeat/module/elasticsearch/_meta/fields.yml index 6824317ff74..d457c6f0d43 100644 --- a/metricbeat/module/elasticsearch/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/_meta/fields.yml @@ -24,3 +24,13 @@ type: keyword description: > Elasticsearch state id. + + - name: node.id + type: keyword + description: > + Node ID + + - name: node.name + type: keyword + description: > + Node name. diff --git a/metricbeat/module/elasticsearch/fields.go b/metricbeat/module/elasticsearch/fields.go index 54222a420ef..8ea1f9417a6 100644 --- a/metricbeat/module/elasticsearch/fields.go +++ b/metricbeat/module/elasticsearch/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsW91u2zgWvvdTHOSqBRI9gC/2pth2s0CzxTYdYDAYqLR0bDHhj0JSTjxPPyAlOZREWYotxy7GvovlnO87h+eXpG7gETdzQEa0oYlGopJsBmCoYTiHq3/731/NAFLUiaK5oVLM4V8zAIDGb4DLtGA4A1DIkGicwwINmQFoNIaKlZ7DH1das6truMqMya/+tM8yqUycSLGkqzksCdNWwpIiS/XcgdyAIBy7RO3HbHKcw0rJIq++CbBsivNFJqzQBlVk/9o+rKU+4uZZqtT7Pii7/DQtUcl1KNGsF5amxwCl6Q5IbYjBiYGdTAfbQU1UB8hfrAGYT0pqfVPrpTBnNCH2hw5Se79te1z9aS+8T44hSVE1HvVx7BPli6MixZfO037zjtC//twRjiCXFeMepJoHJy+xxqdYyF4yTIrVfky+khfKCw4anwoUCYIo+AKVJSdzVOXiSAEmw5qtzkhL75rpUjImn3+tJag5DyyCUzoqjXOEZbjbWt0a2oHBMzUZLS2/m9t2oXT8rKgxKI7K8BWu5IUpfKhDGdOPQIWRjvXWtKU+SyX5bj/ylTKUY6ypSDC2uSlWSNKI6yNodk85XgMVwPU1OMQmewsPSzRJhh0leumvmFwQFicZJo+5pMIcgfgXhwGvGLAmrEAbrk3T7ywe+qCEXqXyKdK3lVG0F3hXpA+YyOdWaPiwUojiGjZoTXMNCtOPUZCIkCmGeYRy2ACLOyvMcaCuwkajUuB2oWQRcJ0djjPoNvfSEOZleaes9f7aJ3qYcGKfTkrlNaWUwm+Q0RVdMBxNKiUN75qSkhU9wMOrUjSZzmNuS3Fn7DOVwiMWyKWfbtLut80INt9dNem3znBbEbbQBJWkZadS+x1meqWUK8qJoh0vOgatEmszTG/b11l72oCIOHKpNtFiYwJMD3Gxr04wFNrWWKk8yE7xajdDby5abQH7FKvWgAmHlapby6gaLkNoxq7gZF11KhMdvVcMpDIpOArj/Gzby/YHgiOXIkODoc5+cnol0ptpaiMVRpr+hT3BMEB1KRUnZg59/zxaFUuhnhscZ6uAk7qDPK6cru/mBCWxLewwsZ155r1M+8qfe9lpsSlntIpqwNzhjBUpTOQa1ebUqYu246rXmgOWKgtxrVZjv8jHs+KnS5UtUCspDFtVuSDyQkqGpD0mDyDfqwKBtspnGFsbsppQ5//X2jq5AUfzsQ1RKzRRzyrvhX/vRLpuuH+VS9hM6nZCKYHpG7tgH9MKBZKmCrXeiT5tVfYpNGtzeNFloRKc1PDfncjdhq9gJzO8jzls+Ap9WsP7FJqGb2ZUXfBWiJ8kofb17pcObRJ6lw7t0qEN8n9bhwaX2eoSuZfI/QUjd7sdzaIHuTik8nN2nDlqr47nh6BPBQJn8CAX/a2eIWbCNuu/clGKDKOlxJDYebGOciUT1BrT2E5eKo1D3r3vHPmtFl5uuuO668ghTlSsCaNpnBKDk/K5z/xj8FJh7Y5jAanJUAEBTrWmYmUJYeklIO337m+TEQOJLFgKQhpYIOREaUwDA0THsW3Te4hbt/7/9Hubd90u3gdbo9JUtkfxQ/EqqWHIhzUfXe+HQui3r3ArlnLcCc2Q1kOajyBUkwoawGdQVYEMSR5RQc3J6sF/kORgGTRKgNVhuNj6SnDyclodOHnZXwUhxemX4k6KmwmWo9bllCuyVWX8qryO864cRZzJ5JGw8Gyw1w7i7bIWDlY2pu7WSWm0YCE4/IKGlRJPcTtj4kNul6Wpd9LdlnxWgxa+UG1cua9HmfMdss5oNPnVhpJ63hvYOYAJZ9W9ZlEHOpDFHtbcGjXKpWSTRa1NntW8ZuXuFbiS9YdF+EbIKCv9j6Vdat116+cHzSu3vcsJQ0sKI1yyrVTPT9z93D4X9AnnSB7PhPE3JI9jKcfnY2hHm4+ztu0nzoT2j7K12ZmjNrIIEjk45n63gi9Rdw6ML1F3blGnC7Wma9n/tsABgfe9kn2JvXNgfIm9U8debwe8SqJEMoaJkWqyLvjLJ9gKDUfdiB645rVrF/DAdvgVAVbJvomhb1aEMYs+gug2Ae3tWQMgjS7oyFYvG6J/tt2DsbjUgcs6cEgQfqYMQW+0QQ5h0UNB6A7+T7LjsLWKwtMccdcEyJpQRhbsfVm030LLUaRUrGJD9OOsDf2Gvc6fIYE/IZHCECo0EKgegH3gS/KDdL/NUY3KxFL1vX/89kPIWycSuiK9W2dSURMOqH0OYQPimrf8gkgHXPCL4LNUgC+E58wqVJgbTvKctqg33gilIn4qsMDuq6B7n/ZS7vbSnNiOj7ZfUn2zU1bvvjrnOdDLjnaj2WRUA9Vub3HE7ebgW9DTHLY7JrsvVk95A+LebaYSg2OwFTKZEGMTizubmfiycVZdePVeACe6Bq3ePI5mfwcAAP//BHh+hA==" + return "eJzsXN1u2zgWvvdTHOSqBRI9gC/2prvtZoFmi226wGAwUGnx2GbCH4WknHiefkBKcmSJshRJjl2MfVfL+c53fnkOSfUGHnE7B+TEWJYYJDpZzwAssxzncPWv6vdXMwCKJtEstUzJOfxjBgCw9xsQimYcZwAaORKDc1igJTMAg9YyuTJz+P3KGH51DVdra9OrP9yztdI2TpRcstUcloQbh7BkyKmZeyE3IInAJlH3sdsU57DSKkuLbwIs9+GqkAnPjEUduX/tHpaoj7h9VppWvg9i5599SxS4Xko0axXL6DGEMnpApLHE4sSCPWZYrFR0vLQ7RRFu/9mCPt53Hj93VcNsiW5gV6OtA/mTVsbclI7RmHKWEPdDbzNT+W09ZcpPPXKr5DgSinrvURvHNqgqHJMUXxpP2y3aQ//yc0cEgloWjFsklTwEeYkNPsVStZLhSq6GMflKXpjIBBh8ylAmCDITC9SOnEpR585REuwaS7ZmTWp6l0yXinP1/Gu5oOTc4QSvdJQb5whuuNtZ3RnaC4NnZtcst/xhbjtHmfhZM2tRHpXhq7icF1L4UKYy0o/ApFWe9c60uT5LrcThOKoqZZnA2DCZYOyKa6yR0EiYI2h2zwReA5MgzDV4ifvsnXhYok3W2FCilf6KqwXhcbLG5DFVTNojEP/iZcCrDNgQnqFL133TH1z9zKiCXpTyKcq3w8jqDj6U6R0mqnLLDHxYaUR5DVt0prkGjfRjFCTiltEwj1AN62DhFlPjOTDfIkS9SuDOUSoLhM6BwOkMm3tlCa9Uea+si/4yJlqYCOKeTkrltaTk4DfI2YotOPYmRcledE1JyUF38KisUiyZLmJuc7gzjplC4R4O8uWnWbTbbdODzXe/mrRbp7utCFtogpWkZqdc+wNmeqWUaiaIZo0oOgatXNa2m96ur3P2dAkRCRRKb6PF1gaYjgmxrx4YMuPWWKUrIhuLV70ZevOiVQcYsljVpiwYt1TdOka1kasqzToPTtZVU5WY6L1ygKokEyitj7NdL9ueCJ4cRY4WQ5395PRySW+maazSGBn2J7YkQwfVpdKC2Dm0/XFvVRyFcm7wnJ0CHvUAeVx5Xd8tCHJiO7HdxA7Wmfcy7St/UalOi20+oxVUA+YOV6xIY6I2qLenLl2snlet1uywVL4Ql2r5Da+QPAc/XamsCXVIYbHFKheUvFCKI6mPyR2S73WGwGrLZ1i2sWQ1oc7/K7X1uIFAq8q2RK/QRi1eHiT/3kP6brjdy7nYtTL1gpILZm/sgqsyHSgQSjUac1D6tKtylcL+2hx2usp0gpMa/ruHPGz4Quxkhq/K7DZ8IX1aw1cphPeh84pqMlFL8ZMU1Lbe/dKhTULv0qFdOrRO/m/r0OAyW10y95K5v2Dm7rajefSgFmNWfsGPM0cN6nh+SPaUIQgOD2rR3upZYidss/6jFjlkWBollsQ+ik2UapWgMUhjN3lpGoeie+gc+a0EzzfdcdMM5BAnJjeEMxpTYnFSPvfr6jF4rrDxx7GAzK5RAwHBjGFy5QhhHiWg3Pf+33ZNLCQq4xSksrBASIk2SAMDRCOwXdM7Jqxrfz8ksDeoDVP16XhEoPkLJQVq2K8PG9F7Ce6K6v9/hVu5VP0OTbq07tK8B6GSVNAAVQZFYV4jSSMmmT1Zif43khQcg72q7HToXv+qSgjyclodBHkZroJU8vSuuFPyZgJ3lLqc0iM7Vfp75XXC9itEJLhKHgkPt+uDNvVulyU4OGyk/iJIbrRgbR5/Z8KhxFNcmJj43NlXaVY5fK4jn9Xsgy/MWL8Cl9PF+c49ZzQt/GpzQjmCdQzzMOH4OGg89EI7qtjDRjijRqlSfLKsdcWzGKEc7qDEVbw9LcKXNHpZ6b+cNqk1/dbOD/Zvwba6E7pcCj1Csq5Uy0/8ldm2EKwSTpE8ngnjb0ge+1KOz8fQnrboZ23XT5wJ7R95a3OwRm1VFiQyOud+c8CXrDsHxpesO7esM5nesI1qv8A/IvG+F9iX3DsHxpfcO3XutXbAqyRKFOeYWKUn64K/fIIdaDjrevTAJa9Du4Aj2+FXCbBKhhaGtlkR+ji9B9FdARocWR1C9rqgI1s9b4j+3nYP5uLSBO7PwJgk/Mw4gtkaiwLC0F1J6M/iT7LjsLOKxtOcOpcEyIYwThb8fVnUXwxLUVImV7El5nFWF/2Gvc6fIcCfkChpCZMGCBQPwD2oIlWTdNjmqEFtY6XbXgl++7ngrYeEJmTlIpjSzIYTasi5aABu/+JdUNKIO3cRfFYa8IWIlDuFMnsjSJqyGvW9lzSZjJ8yzLD5dubgA1gm/F6ah23EaP290TcHZfE6qg+ekVF2tEvGds0MMOP3FntcOA6+mDzN+bdncviu85SXEu79Ziqx2Ee2Rq4SYl1hCf3fBxNQ8XdQK+9kE1MKLV4GjmZ/BQAA///0i5UN" } diff --git a/metricbeat/module/elasticsearch/node/_meta/fields.yml b/metricbeat/module/elasticsearch/node/_meta/fields.yml index 1874c97162d..c1fc72631ad 100644 --- a/metricbeat/module/elasticsearch/node/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/node/_meta/fields.yml @@ -4,10 +4,6 @@ node release: beta fields: - - name: name - type: keyword - description: > - Node name. - name: version type: keyword description: > diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/data.json b/metricbeat/module/elasticsearch/node_stats/_meta/data.json index a654b83e14c..cb33996adf0 100644 --- a/metricbeat/module/elasticsearch/node_stats/_meta/data.json +++ b/metricbeat/module/elasticsearch/node_stats/_meta/data.json @@ -1,43 +1,45 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, "elasticsearch": { "cluster": { - "name": "elasticsearch" + "id": "3LbUkLkURz--FR-YO0wLNA", + "name": "es1" }, "node": { - "name": "523zXyT6TRWiqXcQItnkyQ", + "id": "FMRmkE3HTU6xxxoFK-06Ww", + "name": "es1_1", "stats": { "fs": { "summary": { "available": { - "bytes": 47081816064 + "bytes": 350828584960 }, "free": { - "bytes": 52054323200 + "bytes": 354770468864 }, "total": { - "bytes": 250790436864 + "bytes": 499963170816 } } }, "indices": { "docs": { - "count": 231, - "deleted": 56 + "count": 30880, + "deleted": 124 }, "segments": { - "count": 16, + "count": 39, "memory": { - "bytes": 105245 + "bytes": 300797 } }, "store": { "size": { - "bytes": 444882 + "bytes": 15205991 } } }, @@ -46,14 +48,14 @@ "collectors": { "old": { "collection": { - "count": 2, - "ms": 139 + "count": 3, + "ms": 219 } }, "young": { "collection": { - "count": 56, - "ms": 1288 + "count": 505, + "ms": 2439 } } } @@ -62,44 +64,44 @@ "pools": { "old": { "max": { - "bytes": 724828160 + "bytes": 715849728 }, "peak": { - "bytes": 225180248 + "bytes": 543519960 }, "peak_max": { - "bytes": 724828160 + "bytes": 715849728 }, "used": { - "bytes": 225180248 + "bytes": 382281744 } }, "survivor": { "max": { - "bytes": 34865152 + "bytes": 35782656 }, "peak": { - "bytes": 34865152 + "bytes": 35782656 }, "peak_max": { - "bytes": 34865152 + "bytes": 35782656 }, "used": { - "bytes": 5280456 + "bytes": 6418816 } }, "young": { "max": { - "bytes": 279183360 + "bytes": 286326784 }, "peak": { - "bytes": 279183360 + "bytes": 286326784 }, "peak_max": { - "bytes": 279183360 + "bytes": 286326784 }, "used": { - "bytes": 188134984 + "bytes": 118870448 } } } diff --git a/metricbeat/module/elasticsearch/node_stats/data.go b/metricbeat/module/elasticsearch/node_stats/data.go index 41423d7caa6..8002d93873f 100644 --- a/metricbeat/module/elasticsearch/node_stats/data.go +++ b/metricbeat/module/elasticsearch/node_stats/data.go @@ -19,6 +19,9 @@ package node_stats import ( "encoding/json" + "fmt" + + "github.com/elastic/beats/metricbeat/helper/elastic" "github.com/joeshaw/multierror" "github.com/pkg/errors" @@ -32,6 +35,7 @@ import ( var ( schema = s.Schema{ + "name": c.Str("name"), "jvm": c.Dict("jvm", s.Schema{ "mem": c.Dict("mem", s.Schema{ "pools": c.Dict("pools", s.Schema{ @@ -103,11 +107,10 @@ var ( ) type nodesStruct struct { - ClusterName string `json:"cluster_name"` - Nodes map[string]map[string]interface{} `json:"nodes"` + Nodes map[string]map[string]interface{} `json:"nodes"` } -func eventsMapping(r mb.ReporterV2, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { nodeData := &nodesStruct{} err := json.Unmarshal(content, nodeData) @@ -118,7 +121,7 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { } var errs multierror.Errors - for name, node := range nodeData.Nodes { + for id, node := range nodeData.Nodes { event := mb.Event{} event.RootFields = common.MapStr{} @@ -126,10 +129,11 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { event.ModuleFields = common.MapStr{ "node": common.MapStr{ - "name": name, + "id": id, }, "cluster": common.MapStr{ - "name": nodeData.ClusterName, + "name": info.ClusterName, + "id": info.ClusterID, }, } @@ -138,7 +142,26 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { event.Error = errors.Wrap(err, "failure to apply node schema") r.Event(event) errs = append(errs, event.Error) + continue + } + + name, err := event.MetricSetFields.GetValue("name") + if err != nil { + event.Error = elastic.MakeErrorForMissingField("name", elastic.Elasticsearch) + r.Event(event) + errs = append(errs, event.Error) + continue + } + + nameStr, ok := name.(string) + if !ok { + event.Error = fmt.Errorf("name is not a string") + r.Event(event) + errs = append(errs, event.Error) + continue } + event.ModuleFields.Put("node.name", nameStr) + event.MetricSetFields.Delete("name") r.Event(event) } diff --git a/metricbeat/module/elasticsearch/node_stats/data_test.go b/metricbeat/module/elasticsearch/node_stats/data_test.go index cb73845d9ec..670b67cf289 100644 --- a/metricbeat/module/elasticsearch/node_stats/data_test.go +++ b/metricbeat/module/elasticsearch/node_stats/data_test.go @@ -26,5 +26,5 @@ import ( ) func TestStats(t *testing.T) { - elasticsearch.TestMapper(t, "./_meta/test/node_stats.*.json", eventsMapping) + elasticsearch.TestMapperWithInfo(t, "./_meta/test/node_stats.*.json", eventsMapping) } diff --git a/metricbeat/module/elasticsearch/node_stats/data_xpack.go b/metricbeat/module/elasticsearch/node_stats/data_xpack.go index 6510fc3cfbf..bb4dcb5b3af 100644 --- a/metricbeat/module/elasticsearch/node_stats/data_xpack.go +++ b/metricbeat/module/elasticsearch/node_stats/data_xpack.go @@ -163,7 +163,7 @@ var ( } ) -func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { +func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error { nodesStruct := struct { ClusterName string `json:"cluster_name"` Nodes map[string]map[string]interface{} `json:"nodes"` @@ -181,12 +181,6 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { // of ES and it's not know if the request will be routed to the same node as before. var errs multierror.Errors for nodeID, node := range nodesStruct.Nodes { - clusterID, err := elasticsearch.GetClusterID(m.HTTP, m.HTTP.GetURI(), nodeID) - if err != nil { - errs = append(errs, errors.Wrap(err, "could not fetch cluster id")) - continue - } - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HTTP.GetURI()) if err != nil { errs = append(errs, errors.Wrap(err, "error determining if connected Elasticsearch node is master")) @@ -205,7 +199,7 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { event.RootFields = common.MapStr{ "timestamp": time.Now(), - "cluster_uuid": clusterID, + "cluster_uuid": info.ClusterID, "interval_ms": m.Module().Config().Period.Nanoseconds() / 1000 / 1000, "type": "node_stats", "node_stats": nodeData, diff --git a/metricbeat/module/elasticsearch/node_stats/node_stats.go b/metricbeat/module/elasticsearch/node_stats/node_stats.go index c355fe51fd9..a8d81082292 100644 --- a/metricbeat/module/elasticsearch/node_stats/node_stats.go +++ b/metricbeat/module/elasticsearch/node_stats/node_stats.go @@ -63,17 +63,27 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } + info, err := elasticsearch.GetInfo(m.HTTP, m.getServiceURI()) + if err != nil { + elastic.ReportAndLogError(err, r, m.Log) + return + } + if m.XPack { - err = eventsMappingXPack(r, m, content) + err = eventsMappingXPack(r, m, *info, content) if err != nil { m.Log.Error(err) return } } else { - err = eventsMapping(r, content) + err = eventsMapping(r, *info, content) if err != nil { elastic.ReportAndLogError(err, r, m.Log) return } } } + +func (m *MetricSet) getServiceURI() string { + return m.HostData().SanitizedURI + nodeStatsPath +}