Cherry-pick elastic#17759 to 7.7: Add fields validation for histogram…

… subfields (elastic#17786)

Add validation to histogram subfields in a way that they don't need to
be added to the mappings, but they can still be checked. This is helpful
to avoid having to add exceptions to all the modules based on Prometheus
when Elasticsearch types are used.

(cherry picked from commit 4912b78)

CHANGELOG-developer.next.asciidoc

@@ -60,3 +60,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Compare event by event in `testadata` framework to avoid sorting problems {pull}13747[13747]
 - Added a `default_field` option to fields in fields.yml to offer a way to exclude fields from the default_field list. {issue}14262[14262] {pull}14341[14341]
 - `supported-versions.yml` can be used in metricbeat python system tests to obtain the build args for docker compose builds. {pull}14520[14520]
+- Add fields validation for histogram subfields. {pull}17759[17759]

libbeat/mapping/field.go

@@ -354,6 +354,10 @@ func (f Fields) getKeys(namespace string) []string {
 		} else {
 			keys = append(keys, field.Fields.getKeys(fieldName)...)
 		}
+		if field.ObjectType == "histogram" {
+			keys = append(keys, fieldName+".values")
+			keys = append(keys, fieldName+".counts")
+		}
 	}
 
 	return keys

libbeat/tests/system/beat/beat.py

@@ -573,6 +573,10 @@ def extract_fields(doc_list, name):
                     if field.get("type") in ["object", "geo_point"]:
                         dictfields.append(newName)
 
+                if field.get("type") == "object" and field.get("object_type") == "histogram":
+                    fields.append(newName + ".values")
+                    fields.append(newName + ".counts")
+
                 if field.get("type") == "alias":
                     aliases.append(newName)
 

x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml

@@ -1,13 +1,7 @@
 type: http
 url: "/metrics"
 suffix: plain
-omit_documented_fields_check:
-   # these are not mapped by this module but the oss one
-  - prometheus.labels.*
-  # histogram values & counts are not mapped (it's part of the type data)
-  - '*.histogram.values'
-  - '*.histogram.counts'
 remove_fields_from_comparison: ["prometheus.labels.instance"]
 module:
   use_types: true
-  rate_counters: true
+  rate_counters: true

x-pack/metricbeat/module/prometheus/collector/collector_test.go

@@ -12,6 +12,9 @@ import (
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
 
 	_ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus"
+
+	// Import common fields for validation
+	_ "github.com/elastic/beats/v7/metricbeat/module/prometheus"
 )
 
 func TestData(t *testing.T) {