Scripting: Per context stats in script in _nodes/stats (elastic#59266)

Updated `_nodes/stats`:
 * Update `script` in `_node/stats` to include stats per context:

```
      "script": {
        "compilations": 1,
        "cache_evictions": 0,
        "compilation_limit_triggered": 0,
        "contexts":[
          {
            "context": "aggregation_selector",
            "compilations": 0,
            "cache_evictions": 0,
            "compilation_limit_triggered": 0
          },

```

Refs: elastic#50152
Backport: elastic#59625

server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java

@@ -117,10 +117,13 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             adaptiveSelectionStats = null;
         }
+        scriptCacheStats = null;
         if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
-            scriptCacheStats = in.readOptionalWriteable(ScriptCacheStats::new);
-        } else {
-            scriptCacheStats = null;
+            if (in.getVersion().before(Version.V_7_9_0)) {
+                scriptCacheStats = in.readOptionalWriteable(ScriptCacheStats::new);
+            } else if (scriptStats != null) {
+                scriptCacheStats = scriptStats.toScriptCacheStats();
+            }
         }
     }
 
@@ -271,8 +274,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(ingestStats);
         if (out.getVersion().onOrAfter(Version.V_6_1_0)) {
             out.writeOptionalWriteable(adaptiveSelectionStats);
-        }
-        if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
+        } if (out.getVersion().onOrAfter(Version.V_7_8_0) && out.getVersion().before(Version.V_7_9_0)) {
             out.writeOptionalWriteable(scriptCacheStats);
         }
     }

server/src/main/java/org/elasticsearch/script/ScriptCache.java

@@ -136,6 +136,10 @@ public ScriptStats stats() {
         return scriptMetrics.stats();
     }
 
+    public ScriptContextStats stats(String context) {
+        return scriptMetrics.stats(context);
+    }
+
     /**
      * Check whether there have been too many compilations within the last minute, throwing a circuit breaking exception if so.
      * This is a variant of the token bucket algorithm: https://en.wikipedia.org/wiki/Token_bucket

server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java

@@ -32,8 +32,9 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
+// This class is deprecated in favor of ScriptStats and ScriptContextStats.  It is removed in 8.
 public class ScriptCacheStats implements Writeable, ToXContentFragment {
-    private final Map<String,ScriptStats> context;
+    private final Map<String, ScriptStats> context;
     private final ScriptStats general;
 
     public ScriptCacheStats(Map<String, ScriptStats> context) {
@@ -135,7 +136,19 @@ public ScriptStats sum() {
         if (general != null) {
             return general;
         }
-        return ScriptStats.sum(context.values());
+        long compilations = 0;
+        long cacheEvictions = 0;
+        long compilationLimitTriggered = 0;
+        for (ScriptStats stat: context.values()) {
+            compilations += stat.getCompilations();
+            cacheEvictions += stat.getCacheEvictions();
+            compilationLimitTriggered += stat.getCompilationLimitTriggered();
+        }
+        return new ScriptStats(
+            compilations,
+            cacheEvictions,
+            compilationLimitTriggered
+        );
     }
 
     static final class Fields {

server/src/main/java/org/elasticsearch/script/ScriptContextStats.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ScriptContextStats implements Writeable, ToXContentFragment, Comparable<ScriptContextStats> {
+    private final String context;
+    private final long compilations;
+    private final long cacheEvictions;
+    private final long compilationLimitTriggered;
+
+    public ScriptContextStats(String context, long compilations, long cacheEvictions, long compilationLimitTriggered) {
+        this.context = Objects.requireNonNull(context);
+        this.compilations = compilations;
+        this.cacheEvictions = cacheEvictions;
+        this.compilationLimitTriggered = compilationLimitTriggered;
+    }
+
+    public ScriptContextStats(StreamInput in) throws IOException {
+        context = in.readString();
+        compilations = in.readVLong();
+        cacheEvictions = in.readVLong();
+        compilationLimitTriggered = in.readVLong();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(context);
+        out.writeVLong(compilations);
+        out.writeVLong(cacheEvictions);
+        out.writeVLong(compilationLimitTriggered);
+    }
+
+    public String getContext() {
+        return context;
+    }
+
+    public long getCompilations() {
+        return compilations;
+    }
+
+    public long getCacheEvictions() {
+        return cacheEvictions;
+    }
+
+    public long getCompilationLimitTriggered() {
+        return compilationLimitTriggered;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Fields.CONTEXT, getContext());
+        builder.field(Fields.COMPILATIONS, getCompilations());
+        builder.field(Fields.CACHE_EVICTIONS, getCacheEvictions());
+        builder.field(Fields.COMPILATION_LIMIT_TRIGGERED, getCompilationLimitTriggered());
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int compareTo(ScriptContextStats o) {
+        return this.context.compareTo(o.context);
+    }
+
+    static final class Fields {
+        static final String CONTEXT = "context";
+        static final String COMPILATIONS = "compilations";
+        static final String CACHE_EVICTIONS = "cache_evictions";
+        static final String COMPILATION_LIMIT_TRIGGERED = "compilation_limit_triggered";
+    }
+}

server/src/main/java/org/elasticsearch/script/ScriptMetrics.java

@@ -26,10 +26,6 @@ public class ScriptMetrics {
     final CounterMetric cacheEvictionsMetric = new CounterMetric();
     final CounterMetric compilationLimitTriggered = new CounterMetric();
 
-    public ScriptStats stats() {
-        return new ScriptStats(compilationsMetric.count(), cacheEvictionsMetric.count(), compilationLimitTriggered.count());
-    }
-
     public void onCompilation() {
         compilationsMetric.inc();
     }
@@ -41,4 +37,17 @@ public void onCacheEviction() {
     public void onCompilationLimit() {
         compilationLimitTriggered.inc();
     }
+
+    public ScriptStats stats() {
+        return new ScriptStats(compilationsMetric.count(), cacheEvictionsMetric.count(), compilationLimitTriggered.count());
+    }
+
+    public ScriptContextStats stats(String context) {
+        return new ScriptContextStats(
+            context,
+            compilationsMetric.count(),
+            cacheEvictionsMetric.count(),
+            compilationLimitTriggered.count()
+        );
+    }
 }

server/src/main/java/org/elasticsearch/script/ScriptService.java

@@ -658,7 +658,12 @@ ScriptStats stats() {
             if (general != null) {
                 return general.stats();
             }
-            return ScriptStats.sum(contextCache.values().stream().map(AtomicReference::get).map(ScriptCache::stats)::iterator);
+            List<ScriptContextStats> contextStats = new ArrayList<>(contextCache.size());
+            for (Map.Entry<String, AtomicReference<ScriptCache>> entry : contextCache.entrySet()) {
+                ScriptCache cache = entry.getValue().get();
+                contextStats.add(cache.stats(entry.getKey()));
+            }
+            return new ScriptStats(contextStats);
         }
 
         ScriptCacheStats cacheStats() {

server/src/main/java/org/elasticsearch/script/ScriptStats.java

@@ -27,22 +27,52 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 public class ScriptStats implements Writeable, ToXContentFragment {
+    private final List<ScriptContextStats> contextStats;
     private final long compilations;
     private final long cacheEvictions;
     private final long compilationLimitTriggered;
 
+    public ScriptStats(List<ScriptContextStats> contextStats) {
+        ArrayList<ScriptContextStats> ctxStats = new ArrayList<>(contextStats.size());
+        ctxStats.addAll(contextStats);
+        ctxStats.sort(ScriptContextStats::compareTo);
+        this.contextStats = Collections.unmodifiableList(ctxStats);
+        long compilations = 0;
+        long cacheEvictions = 0;
+        long compilationLimitTriggered = 0;
+        for (ScriptContextStats stats: contextStats) {
+            compilations += stats.getCompilations();
+            cacheEvictions += stats.getCacheEvictions();
+            compilationLimitTriggered += stats.getCompilationLimitTriggered();
+        }
+        this.compilations = compilations;
+        this.cacheEvictions = cacheEvictions;
+        this.compilationLimitTriggered = compilationLimitTriggered;
+    }
+
     public ScriptStats(long compilations, long cacheEvictions, long compilationLimitTriggered) {
+        this.contextStats = Collections.emptyList();
         this.compilations = compilations;
         this.cacheEvictions = cacheEvictions;
         this.compilationLimitTriggered = compilationLimitTriggered;
     }
 
+    public ScriptStats(ScriptContextStats context) {
+        this(context.getCompilations(), context.getCacheEvictions(), context.getCompilationLimitTriggered());
+    }
+
     public ScriptStats(StreamInput in) throws IOException {
         compilations = in.readVLong();
         cacheEvictions = in.readVLong();
         compilationLimitTriggered = in.getVersion().onOrAfter(Version.V_7_0_0) ? in.readVLong() : 0;
+        contextStats = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readList(ScriptContextStats::new) : Collections.emptyList();
     }
 
     @Override
@@ -52,6 +82,13 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_7_0_0)) {
             out.writeVLong(compilationLimitTriggered);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_9_0)) {
+            out.writeList(contextStats);
+        }
+    }
+
+    public List<ScriptContextStats> getContextStats() {
+        return contextStats;
     }
 
     public long getCompilations() {
@@ -66,36 +103,32 @@ public long getCompilationLimitTriggered() {
         return compilationLimitTriggered;
     }
 
+    public ScriptCacheStats toScriptCacheStats() {
+        if (contextStats.isEmpty()) {
+            return new ScriptCacheStats(this);
+        }
+        Map<String, ScriptStats> contexts = new HashMap<>(contextStats.size());
+        for (ScriptContextStats contextStats : contextStats) {
+            contexts.put(contextStats.getContext(), new ScriptStats(contextStats));
+        }
+        return new ScriptCacheStats(contexts);
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.SCRIPT_STATS);
-        builder.field(Fields.COMPILATIONS, getCompilations());
-        builder.field(Fields.CACHE_EVICTIONS, getCacheEvictions());
-        builder.field(Fields.COMPILATION_LIMIT_TRIGGERED, getCompilationLimitTriggered());
+        builder.field(Fields.COMPILATIONS, compilations);
+        builder.field(Fields.CACHE_EVICTIONS, cacheEvictions);
+        builder.field(Fields.COMPILATION_LIMIT_TRIGGERED, compilationLimitTriggered);
         builder.endObject();
         return builder;
     }
 
     static final class Fields {
         static final String SCRIPT_STATS = "script";
+        static final String CONTEXTS = "contexts";
         static final String COMPILATIONS = "compilations";
         static final String CACHE_EVICTIONS = "cache_evictions";
         static final String COMPILATION_LIMIT_TRIGGERED = "compilation_limit_triggered";
     }
-
-    public static ScriptStats sum(Iterable<ScriptStats> stats) {
-        long compilations = 0;
-        long cacheEvictions = 0;
-        long compilationLimitTriggered = 0;
-        for (ScriptStats stat: stats) {
-            compilations += stat.compilations;
-            cacheEvictions += stat.cacheEvictions;
-            compilationLimitTriggered += stat.compilationLimitTriggered;
-        }
-        return new ScriptStats(
-            compilations,
-            cacheEvictions,
-            compilationLimitTriggered
-        );
-    }
 }

server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java

@@ -316,7 +316,7 @@ public void testSerialization() throws IOException {
                 ScriptCacheStats deserializedScriptCacheStats = deserializedNodeStats.getScriptCacheStats();
                 if (scriptCacheStats == null) {
                     assertNull(deserializedScriptCacheStats);
-                } else {
+                } else if (deserializedScriptCacheStats.getContextStats() != null) {
                     Map<String, ScriptStats> deserialized = deserializedScriptCacheStats.getContextStats();
                     long evictions = 0;
                     long limited = 0;
@@ -514,16 +514,7 @@ public static NodeStats createNodeStats() {
             }
             adaptiveSelectionStats = new AdaptiveSelectionStats(nodeConnections, nodeStats);
         }
-        ScriptCacheStats scriptCacheStats = null;
-        if (frequently()) {
-            int numContents = randomIntBetween(0, 20);
-            Map<String,ScriptStats> stats = new HashMap<>(numContents);
-            for (int i = 0; i < numContents; i++) {
-                String context = randomValueOtherThanMany(stats::containsKey, () -> randomAlphaOfLength(12));
-                stats.put(context, new ScriptStats(randomLongBetween(0, 1024), randomLongBetween(0, 1024), randomLongBetween(0, 1024)));
-            }
-            scriptCacheStats = new ScriptCacheStats(stats);
-        }
+        ScriptCacheStats scriptCacheStats = scriptStats != null ? scriptStats.toScriptCacheStats() : null;
         //TODO NodeIndicesStats are not tested here, way too complicated to create, also they need to be migrated to Writeable yet
         return new NodeStats(node, randomNonNegativeLong(), null, osStats, processStats, jvmStats, threadPoolStats,
                 fsInfo, transportStats, httpStats, allCircuitBreakerStats, scriptStats, discoveryStats,

server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java

@@ -37,7 +37,6 @@
 import org.junit.Before;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -585,7 +584,6 @@ public void testCacheHolderChangeSettings() throws IOException {
         );
         assertEquals(contexts.keySet(), scriptService.cacheHolder.get().contextCache.keySet());
 
-        String d = randomValueOtherThanMany(Arrays.asList(a, b, c)::contains, () -> randomFrom(contextNames));
         assertEquals(new ScriptCache.CompilationRate(aRate),
                      scriptService.cacheHolder.get().contextCache.get(a).get().rate);
         assertEquals(new ScriptCache.CompilationRate(bRate),