Script: Time series compile and cache evict metrics (#79078)

Collects compilation and cache eviction metrics for
each script context.

Metrics are available in _nodes/stats in 5m/15m/1d
buckets.

Refs: #62899

docs/reference/cluster/nodes-stats.asciidoc

@@ -2032,10 +2032,47 @@ Contains script statistics for the node.
 (integer)
 Total number of inline script compilations performed by the node.
 
+`compilations_history`::
+(object)
+Contains this recent history of script compilations
+
+.Properties of `compilations_history`
+[%collapsible%open]
+=======
+`5m`::
+(long)
+The number of script compilations in the last five minutes.
+`15m`::
+(long)
+The number of script compilations in the last fifteen minutes.
+`24h`::
+(long)
+The number of script compilations in the last twenty-four hours.
+=======
+
 `cache_evictions`::
 (integer)
 Total number of times the script cache has evicted old data.
 
+
+`cache_evictions_history`::
+(object)
+Contains this recent history of script cache evictions
+
+.Properties of `cache_evictions`
+[%collapsible%open]
+=======
+`5m`::
+(long)
+The number of script cache evictions in the last five minutes.
+`15m`::
+(long)
+The number of script cache evictions in the last fifteen minutes.
+`24h`::
+(long)
+The number of script cache evictions in the last twenty-four hours.
+=======
+
 `compilation_limit_triggered`::
 (integer)
 Total number of times the <<script-compilation-circuit-breaker,script

docs/reference/modules/indices/circuit_breaker.asciidoc

@@ -132,6 +132,14 @@ documentation for more information.
     that are allowed to be compiled. Defaults to `150/5m`,
     meaning 150 every 5 minutes.
 
+If the cluster regularly hits the given `max_compilation_rate`, it's possible the
+script cache is undersized, use <<cluster-nodes-stats,Nodes Stats>> to inspect
+the number of recent cache evictions, `script.cache_evictions_history` and
+compilations `script.compilations_history`.  If there are a large
+number of recent cache evictions or compilations, the script cache may be
+undersized, consider doubling the size of the script cache via the setting
+`script.cache.max_size`.
+
 [[regex-circuit-breaker]]
 [discrete]
 ==== Regex circuit breaker

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java

@@ -48,7 +48,7 @@ public boolean execute(Token token) {
         };
 
         @SuppressWarnings("unchecked")
-        ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap()) {
+        ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L) {
             @Override
             public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryType> context) {
                 assertEquals(context, AnalysisPredicateScript.CONTEXT);

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java

@@ -48,7 +48,7 @@ public boolean execute(Token token) {
         };
 
         @SuppressWarnings("unchecked")
-        ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap()) {
+        ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L) {
             @Override
             public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryType> context) {
                 assertEquals(context, AnalysisPredicateScript.CONTEXT);

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java

@@ -144,7 +144,8 @@ public void testInlineIsCompiled() throws Exception {
                     return null;
                 }), Collections.emptyMap())
             ),
-            new HashMap<>(ScriptModule.CORE_CONTEXTS)
+            new HashMap<>(ScriptModule.CORE_CONTEXTS),
+            () -> 1L
         );
         factory = new ScriptProcessor.Factory(scriptService);
 

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java

@@ -49,7 +49,8 @@ public void setupScripting() {
                     return null;
                 }), Collections.emptyMap())
             ),
-            new HashMap<>(ScriptModule.CORE_CONTEXTS)
+            new HashMap<>(ScriptModule.CORE_CONTEXTS),
+            () -> 1L
         );
         script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap());
         ingestScript = scriptService.compile(script, IngestScript.CONTEXT).newInstance(script.getParams());

qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java

@@ -32,7 +32,7 @@ public abstract class AbstractScriptTestCase extends ESTestCase {
     public void init() throws Exception {
         MustacheScriptEngine engine = new MustacheScriptEngine();
         Map<String, ScriptEngine> engines = Collections.singletonMap(engine.getType(), engine);
-        scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
+        scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L);
     }
 
     protected TemplateScript.Factory compile(String template) {

server/src/main/java/org/elasticsearch/node/Node.java

@@ -198,6 +198,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
+import java.util.function.LongSupplier;
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -420,7 +421,12 @@ protected Node(
             client = new NodeClient(settings, threadPool);
 
             final ScriptModule scriptModule = new ScriptModule(settings, pluginsService.filterPlugins(ScriptPlugin.class));
-            final ScriptService scriptService = newScriptService(settings, scriptModule.engines, scriptModule.contexts);
+            final ScriptService scriptService = newScriptService(
+                settings,
+                scriptModule.engines,
+                scriptModule.contexts,
+                threadPool::absoluteTimeInMillis
+            );
             AnalysisModule analysisModule = new AnalysisModule(this.environment, pluginsService.filterPlugins(AnalysisPlugin.class));
             // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool
             // so we might be late here already
@@ -1487,8 +1493,13 @@ protected SearchService newSearchService(
     /**
      * Creates a new the ScriptService. This method can be overwritten by tests to inject mock implementations.
      */
-    protected ScriptService newScriptService(Settings settings, Map<String, ScriptEngine> engines, Map<String, ScriptContext<?>> contexts) {
-        return new ScriptService(settings, engines, contexts);
+    protected ScriptService newScriptService(
+        Settings settings,
+        Map<String, ScriptEngine> engines,
+        Map<String, ScriptContext<?>> contexts,
+        LongSupplier timeProvider
+    ) {
+        return new ScriptService(settings, engines, contexts, timeProvider);
     }
 
     /**

server/src/main/java/org/elasticsearch/script/ScriptCache.java

@@ -23,6 +23,7 @@
 import java.util.Objects;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.LongSupplier;
 
 /**
  * Script cache and compilation rate limiter.
@@ -44,7 +45,13 @@ public class ScriptCache {
     private final double compilesAllowedPerNano;
     private final String contextRateSetting;
 
-    ScriptCache(int cacheMaxSize, TimeValue cacheExpire, CompilationRate maxCompilationRate, String contextRateSetting) {
+    ScriptCache(
+        int cacheMaxSize,
+        TimeValue cacheExpire,
+        CompilationRate maxCompilationRate,
+        String contextRateSetting,
+        LongSupplier timeProvider
+    ) {
         this.cacheSize = cacheMaxSize;
         this.cacheExpire = cacheExpire;
         this.contextRateSetting = contextRateSetting;
@@ -63,7 +70,7 @@ public class ScriptCache {
 
         this.rate = maxCompilationRate;
         this.compilesAllowedPerNano = ((double) rate.count) / rate.time.nanos();
-        this.scriptMetrics = new ScriptMetrics();
+        this.scriptMetrics = new ScriptMetrics(timeProvider);
         this.tokenBucketState = new AtomicReference<TokenBucketState>(new TokenBucketState(this.rate.count));
     }
 

server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java

@@ -133,7 +133,7 @@ public ScriptStats sum() {
             cacheEvictions += stat.getCacheEvictions();
             compilationLimitTriggered += stat.getCompilationLimitTriggered();
         }
-        return new ScriptStats(compilations, cacheEvictions, compilationLimitTriggered);
+        return new ScriptStats(compilations, cacheEvictions, compilationLimitTriggered, null, null);
     }
 
     static final class Fields {

server/src/main/java/org/elasticsearch/script/ScriptContextStats.java

@@ -28,15 +28,13 @@ public class ScriptContextStats implements Writeable, ToXContentFragment, Compar
 
     public ScriptContextStats(
         String context,
-        long compilations,
-        long cacheEvictions,
         long compilationLimitTriggered,
         TimeSeries compilationsHistory,
         TimeSeries cacheEvictionsHistory
     ) {
         this.context = Objects.requireNonNull(context);
-        this.compilations = compilations;
-        this.cacheEvictions = cacheEvictions;
+        this.compilations = compilationsHistory.total;
+        this.cacheEvictions = cacheEvictionsHistory.total;
         this.compilationLimitTriggered = compilationLimitTriggered;
         this.compilationsHistory = compilationsHistory;
         this.cacheEvictionsHistory = cacheEvictionsHistory;
@@ -47,12 +45,15 @@ public ScriptContextStats(StreamInput in) throws IOException {
         compilations = in.readVLong();
         cacheEvictions = in.readVLong();
         compilationLimitTriggered = in.readVLong();
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
             compilationsHistory = new TimeSeries(in);
             cacheEvictionsHistory = new TimeSeries(in);
+        } else if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            compilationsHistory = new TimeSeries(in).withTotal(compilations);
+            cacheEvictionsHistory = new TimeSeries(in).withTotal(cacheEvictions);
         } else {
-            compilationsHistory = null;
-            cacheEvictionsHistory = null;
+            compilationsHistory = new TimeSeries(compilations);
+            cacheEvictionsHistory = new TimeSeries(cacheEvictions);
         }
     }
 
@@ -68,65 +69,6 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    public static class TimeSeries implements Writeable, ToXContentFragment {
-        public final long fiveMinutes;
-        public final long fifteenMinutes;
-        public final long twentyFourHours;
-
-        public TimeSeries() {
-            this.fiveMinutes = 0;
-            this.fifteenMinutes = 0;
-            this.twentyFourHours = 0;
-        }
-
-        public TimeSeries(long fiveMinutes, long fifteenMinutes, long twentyFourHours) {
-            assert fiveMinutes >= 0;
-            this.fiveMinutes = fiveMinutes;
-            assert fifteenMinutes >= fiveMinutes;
-            this.fifteenMinutes = fifteenMinutes;
-            assert twentyFourHours >= fifteenMinutes;
-            this.twentyFourHours = twentyFourHours;
-        }
-
-        public TimeSeries(StreamInput in) throws IOException {
-            fiveMinutes = in.readVLong();
-            fifteenMinutes = in.readVLong();
-            twentyFourHours = in.readVLong();
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(Fields.FIVE_MINUTES, fiveMinutes);
-            builder.field(Fields.FIFTEEN_MINUTES, fifteenMinutes);
-            builder.field(Fields.TWENTY_FOUR_HOURS, twentyFourHours);
-            return builder;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeVLong(fiveMinutes);
-            out.writeVLong(fifteenMinutes);
-            out.writeVLong(twentyFourHours);
-        }
-
-        public boolean isEmpty() {
-            return twentyFourHours == 0;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            TimeSeries that = (TimeSeries) o;
-            return fiveMinutes == that.fiveMinutes && fifteenMinutes == that.fifteenMinutes && twentyFourHours == that.twentyFourHours;
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(fiveMinutes, fifteenMinutes, twentyFourHours);
-        }
-    }
-
     public String getContext() {
         return context;
     }
@@ -158,15 +100,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(Fields.COMPILATIONS, getCompilations());
 
         TimeSeries series = getCompilationsHistory();
-        if (series != null && series.isEmpty() == false) {
+        if (series != null && series.areTimingsEmpty() == false) {
             builder.startObject(Fields.COMPILATIONS_HISTORY);
             series.toXContent(builder, params);
             builder.endObject();
         }
 
         builder.field(Fields.CACHE_EVICTIONS, getCacheEvictions());
         series = getCacheEvictionsHistory();
-        if (series != null && series.isEmpty() == false) {
+        if (series != null && series.areTimingsEmpty() == false) {
             builder.startObject(Fields.CACHE_EVICTIONS_HISTORY);
             series.toXContent(builder, params);
             builder.endObject();

server/src/main/java/org/elasticsearch/script/ScriptMetrics.java

@@ -10,35 +10,45 @@
 
 import org.elasticsearch.common.metrics.CounterMetric;
 
+import java.util.function.LongSupplier;
+
 public class ScriptMetrics {
-    final CounterMetric compilationsMetric = new CounterMetric();
-    final CounterMetric cacheEvictionsMetric = new CounterMetric();
     final CounterMetric compilationLimitTriggered = new CounterMetric();
+    final TimeSeriesCounter compilations;
+    final TimeSeriesCounter cacheEvictions;
+
+    public ScriptMetrics(LongSupplier timeProvider) {
+        compilations = new TimeSeriesCounter(timeProvider);
+        cacheEvictions = new TimeSeriesCounter(timeProvider);
+    }
 
     public void onCompilation() {
-        compilationsMetric.inc();
+        compilations.inc();
     }
 
     public void onCacheEviction() {
-        cacheEvictionsMetric.inc();
+        cacheEvictions.inc();
     }
 
     public void onCompilationLimit() {
         compilationLimitTriggered.inc();
     }
 
     public ScriptStats stats() {
-        return new ScriptStats(compilationsMetric.count(), cacheEvictionsMetric.count(), compilationLimitTriggered.count());
+        TimeSeries compilationsTimeSeries = compilations.timeSeries();
+        TimeSeries cacheEvictionsTimeSeries = cacheEvictions.timeSeries();
+        return new ScriptStats(
+            compilationsTimeSeries.total,
+            cacheEvictionsTimeSeries.total,
+            compilationLimitTriggered.count(),
+            compilationsTimeSeries,
+            cacheEvictionsTimeSeries
+        );
     }
 
     public ScriptContextStats stats(String context) {
-        return new ScriptContextStats(
-            context,
-            compilationsMetric.count(),
-            cacheEvictionsMetric.count(),
-            compilationLimitTriggered.count(),
-            new ScriptContextStats.TimeSeries(),
-            new ScriptContextStats.TimeSeries()
-        );
+        TimeSeries compilationsTimeSeries = compilations.timeSeries();
+        TimeSeries cacheEvictionsTimeSeries = cacheEvictions.timeSeries();
+        return new ScriptContextStats(context, compilationLimitTriggered.count(), compilationsTimeSeries, cacheEvictionsTimeSeries);
     }
 }