From db36db046357d5e0de371f49eb6f7363bb69bd78 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jsorianopastor@gmail.com>
Date: Fri, 17 Apr 2020 12:15:54 +0200
Subject: [PATCH] Add fields validation for histogram subfields (#17759)

Add validation to histogram subfields in a way that they don't need to
be added to the mappings, but they can still be checked. This is helpful
to avoid having to add exceptions to all the modules based on Prometheus
when Elasticsearch types are used.

(cherry picked from commit 5983411ddf0203433725f819adfb25dc1dc841e9)
---
 CHANGELOG-developer.next.asciidoc                         | 1 +
 libbeat/mapping/field.go                                  | 4 ++++
 libbeat/tests/system/beat/beat.py                         | 4 ++++
 .../module/prometheus/collector/_meta/testdata/config.yml | 8 +-------
 .../module/prometheus/collector/collector_test.go         | 3 +++
 5 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
index 1f004c42d31..d5fb9a22dcd 100644
--- a/CHANGELOG-developer.next.asciidoc
+++ b/CHANGELOG-developer.next.asciidoc
@@ -71,3 +71,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - `supported-versions.yml` can be used in metricbeat python system tests to obtain the build args for docker compose builds. {pull}14520[14520]
 - Add support for MODULE environment variable in `mage goIntegTest` in metricbeat to run integration tests for a single module. {pull}17147[17147]
 - Add support for a `TEST_TAGS` environment variable to add tags for tests selection following go build tags semantics, this environment variable is used by mage test targets to add build tags. Python tests can also be tagged with a decorator (`@beat.tag('sometag')`). {pull}16937[16937] {pull}17075[17075]
+- Add fields validation for histogram subfields. {pull}17759[17759]
diff --git a/libbeat/mapping/field.go b/libbeat/mapping/field.go
index 66a580dac5e..7b2ba52e618 100644
--- a/libbeat/mapping/field.go
+++ b/libbeat/mapping/field.go
@@ -354,6 +354,10 @@ func (f Fields) getKeys(namespace string) []string {
 		} else {
 			keys = append(keys, field.Fields.getKeys(fieldName)...)
 		}
+		if field.ObjectType == "histogram" {
+			keys = append(keys, fieldName+".values")
+			keys = append(keys, fieldName+".counts")
+		}
 	}
 
 	return keys
diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py
index 160df982f56..fa8e0ad3cb4 100644
--- a/libbeat/tests/system/beat/beat.py
+++ b/libbeat/tests/system/beat/beat.py
@@ -573,6 +573,10 @@ def extract_fields(doc_list, name):
                     if field.get("type") in ["object", "geo_point"]:
                         dictfields.append(newName)
 
+                if field.get("type") == "object" and field.get("object_type") == "histogram":
+                    fields.append(newName + ".values")
+                    fields.append(newName + ".counts")
+
                 if field.get("type") == "alias":
                     aliases.append(newName)
 
diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml
index e7b2a5e67b4..380627ba09f 100644
--- a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml
+++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml
@@ -1,13 +1,7 @@
 type: http
 url: "/metrics"
 suffix: plain
-omit_documented_fields_check:
-   # these are not mapped by this module but the oss one
-  - prometheus.labels.*
-  # histogram values & counts are not mapped (it's part of the type data)
-  - '*.histogram.values'
-  - '*.histogram.counts'
 remove_fields_from_comparison: ["prometheus.labels.instance"]
 module:
   use_types: true
-  rate_counters: true
\ No newline at end of file
+  rate_counters: true
diff --git a/x-pack/metricbeat/module/prometheus/collector/collector_test.go b/x-pack/metricbeat/module/prometheus/collector/collector_test.go
index 0a3950bdc75..5f42875dd76 100644
--- a/x-pack/metricbeat/module/prometheus/collector/collector_test.go
+++ b/x-pack/metricbeat/module/prometheus/collector/collector_test.go
@@ -12,6 +12,9 @@ import (
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
 
 	_ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus"
+
+	// Import common fields for validation
+	_ "github.com/elastic/beats/v7/metricbeat/module/prometheus"
 )
 
 func TestData(t *testing.T) {