Track Lucene field usage (elastic#74227)

Adds a field usage API that reports shard-level statistics about which Lucene fields have been accessed, and which
parts of the Lucene data structures have been accessed.

Field usage statistics are automatically captured when queries are runnning on a cluster. A shard-level search request
that accesses a given field, even if multiple times during that request, is counted as a single use.

docs/reference/indices/field-usage-stats.asciidoc

@@ -0,0 +1,167 @@
+[[field-usage-stats]]
+=== Field usage stats API
+++++
+<titleabbrev>Field usage stats</titleabbrev>
+++++
+
+experimental[]
+
+Returns field usage information for each shard and field
+of an index.
+Field usage statistics are automatically captured when
+queries are running on a cluster. A shard-level search
+request that accesses a given field, even if multiple times
+during that request, is counted as a single use.
+
+[source,console]
+--------------------------------------------------
+GET /my-index-000001/_field_usage_stats
+--------------------------------------------------
+// TEST[setup:messages]
+
+[[field-usage-stats-api-request]]
+==== {api-request-title}
+
+`GET /<index>/_field_usage_stats
+
+[[field-usage-stats-api-request-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have the `manage`
+<<privileges-list-indices,index privilege>> for the target index or index alias.
+
+[[field-usage-stats-api-path-params]]
+==== {api-path-parms-title}
+
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index]
+
+[[field-usage-stats-api-query-params]]
+==== {api-query-parms-title}
+
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices]
+
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards]
+
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable]
+
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards]
+
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms]
+
+`fields`::
++
+--
+(Optional, string)
+Comma-separated list or wildcard expressions of fields
+to include in the statistics.
+--
+
+[[field-usage-stats-api-example]]
+==== {api-examples-title}
+
+//////////////////////////
+
+[source,console]
+--------------------------------------------------
+POST /my-index-000001/_search
+{
+  "query" : {
+    "match" : { "context" : "bar" }
+  },
+  "aggs": {
+    "message_stats": {
+      "string_stats": {
+        "field": "message.keyword",
+        "show_distribution": true
+      }
+    }
+  }
+}
+--------------------------------------------------
+// TEST[setup:messages]
+
+//////////////////////////
+
+The following request retrieves field usage information of index `my-index-000001`
+on the currently available shards.
+
+[source,console]
+--------------------------------------------------
+GET /my-index-000001/_field_usage_stats
+--------------------------------------------------
+// TEST[continued]
+
+The API returns the following response:
+
+[source,console-response]
+--------------------------------------------------
+{
+    "_shards": {
+        "total": 1,
+        "successful": 1,
+        "failed": 0
+    },
+    "my-index-000001": {
+        "shards": [
+            {
+                "tracking_id": "MpOl0QlTQ4SYYhEe6KgJoQ",
+                "tracking_started_at_millis": 1625558985010,
+                "routing": {
+                    "state": "STARTED",
+                    "primary": true,
+                    "node": "gA6KeeVzQkGURFCUyV-e8Q",
+                    "relocating_node": null
+                },
+                "stats" : {
+                    "all_fields": {
+                        "any": "6", <1>
+                        "inverted_index": {
+                            "terms" : 1,
+                            "postings" : 1,
+                            "proximity" : 1, <2>
+                            "positions" : 0,
+                            "term_frequencies" : 1,
+                            "offsets" : 0,
+                            "payloads" : 0
+                        },
+                        "stored_fields" : 2,
+                        "doc_values" : 1,
+                        "points" : 0,
+                        "norms" : 1,
+                        "term_vectors" : 0
+                    },
+                    "fields": {
+                        "_id": {
+                            "any" : 1,
+                            "inverted_index": {
+                                "terms" : 1,
+                                "postings" : 1,
+                                "proximity" : 1,
+                                "positions" : 0,
+                                "term_frequencies" : 1,
+                                "offsets" : 0,
+                                "payloads" : 0
+                            },
+                            "stored_fields" : 1,
+                            "doc_values" : 0,
+                            "points" : 0,
+                            "norms" : 0,
+                            "term_vectors" : 0
+                        },
+                        "_source": {...},
+                        "context": {...},
+                        "message.keyword": {...}
+                    }
+                }
+            }
+        ]
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/: \{\.\.\.\}/: $body.$_path/]
+// TESTRESPONSE[s/: (\-)?[0-9]+/: $body.$_path/]
+// TESTRESPONSE[s/: "[^"]*"/: $body.$_path/]
+<1> denotes any kind of use of the field, either inverted index,
+    or stored fields, or doc values, etc.
+<2> denotes any kind of use of either positions, offsets or
+    payloads.

rest-api-spec/src/main/resources/rest-api-spec/api/indices.field_usage_stats.json

@@ -0,0 +1,57 @@
+{
+  "indices.field_usage_stats": {
+    "documentation": {
+      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-field-usage-stats.html",
+      "description": "Returns the field usage stats for each field of an index"
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/{index}/_field_usage_stats",
+          "methods": [
+            "GET"
+          ],
+          "parts": {
+            "index": {
+              "type": "string",
+              "description": "A comma-separated list of index names; use `_all` or empty string to perform the operation on all indices"
+            }
+          }
+        }
+      ]
+    },
+    "params": {
+      "fields":{
+        "type":"list",
+        "description":"A comma-separated list of fields to include in the stats if only a subset of fields should be returned (supports wildcards)"
+      },
+      "ignore_unavailable": {
+        "type": "boolean",
+        "description": "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
+      },
+      "allow_no_indices": {
+        "type": "boolean",
+        "description": "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
+      },
+      "expand_wildcards": {
+        "type": "enum",
+        "options": [
+          "open",
+          "closed",
+          "hidden",
+          "none",
+          "all"
+        ],
+        "default": "open",
+        "description": "Whether to expand wildcard expression to concrete indices that are open, closed or both."
+      }
+    }
+  }
+}