From 30e15ba8387f2130976ca017553b2ca6f2f5de0e Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 3 Nov 2021 13:13:42 -0500 Subject: [PATCH] Script: Time series compile and cache evict metrics (#79078) Collects compilation and cache eviction metrics for each script context. Metrics are available in _nodes/stats in 5m/15m/1d buckets. Refs: #62899 --- docs/reference/cluster/nodes-stats.asciidoc | 37 ++ .../modules/indices/circuit_breaker.asciidoc | 8 + .../PredicateTokenScriptFilterTests.java | 2 +- .../ScriptedConditionTokenFilterTests.java | 2 +- .../common/ScriptProcessorFactoryTests.java | 3 +- .../ingest/common/ScriptProcessorTests.java | 3 +- .../ingest/AbstractScriptTestCase.java | 2 +- .../java/org/elasticsearch/node/Node.java | 17 +- .../org/elasticsearch/script/ScriptCache.java | 11 +- .../script/ScriptCacheStats.java | 2 +- .../script/ScriptContextStats.java | 78 +-- .../elasticsearch/script/ScriptMetrics.java | 36 +- .../elasticsearch/script/ScriptService.java | 25 +- .../org/elasticsearch/script/ScriptStats.java | 55 +- .../org/elasticsearch/script/TimeSeries.java | 111 ++++ .../script/TimeSeriesCounter.java | 476 +++++++++++++++ .../cluster/node/stats/NodeStatsTests.java | 20 +- .../action/update/UpdateRequestTests.java | 2 +- .../elasticsearch/index/IndexModuleTests.java | 2 +- .../index/mapper/MappingParserTests.java | 2 +- .../index/mapper/TestScriptEngine.java | 2 +- .../ingest/ConditionalProcessorTests.java | 9 +- .../ingest/IngestServiceTests.java | 3 +- .../ingest/TrackingResultProcessorTests.java | 12 +- .../script/ScriptCacheTests.java | 36 +- .../script/ScriptLanguagesInfoTests.java | 6 +- .../script/ScriptServiceTests.java | 2 +- .../script/ScriptStatsTests.java | 86 ++- .../script/TimeSeriesCounterTests.java | 548 ++++++++++++++++++ .../missing/MissingAggregatorTests.java | 2 +- .../bucket/nested/NestedAggregatorTests.java | 2 +- .../bucket/terms/TermsAggregatorTests.java | 2 +- .../metrics/AvgAggregatorTests.java | 2 +- .../metrics/InternalScriptedMetricTests.java | 2 +- .../metrics/MaxAggregatorTests.java | 2 +- ...edianAbsoluteDeviationAggregatorTests.java | 2 +- .../metrics/MinAggregatorTests.java | 2 +- .../ScriptedMetricAggregatorTests.java | 4 +- .../metrics/StatsAggregatorTests.java | 2 +- .../metrics/SumAggregatorTests.java | 2 +- .../metrics/ValueCountAggregatorTests.java | 2 +- .../pipeline/BucketScriptAggregatorTests.java | 2 +- .../pipeline/MovFnAggrgatorTests.java | 2 +- .../search/sort/AbstractSortTestCase.java | 7 +- .../snapshots/SnapshotResiliencyTests.java | 2 +- .../ingest/TestTemplateService.java | 2 +- .../java/org/elasticsearch/node/MockNode.java | 10 +- .../script/MockScriptService.java | 2 +- .../boxplot/BoxplotAggregatorTests.java | 2 +- .../multiterms/MultiTermsAggregatorTests.java | 2 +- .../analytics/rate/RateAggregatorTests.java | 2 +- .../StringStatsAggregatorTests.java | 2 +- .../topmetrics/TopMetricsAggregatorTests.java | 2 +- .../analytics/ttest/TTestAggregatorTests.java | 2 +- .../support/mapper/TemplateRoleNameTests.java | 24 +- .../WildcardServiceProviderResolverTests.java | 3 +- .../authc/ldap/ActiveDirectoryRealmTests.java | 3 +- .../security/authc/ldap/LdapRealmTests.java | 3 +- .../mapper/NativeRoleMappingStoreTests.java | 3 +- .../watcher/support/WatcherTemplateTests.java | 2 +- .../watcher/test/WatcherMockScriptPlugin.java | 2 +- .../test/integration/SearchInputTests.java | 2 +- .../integration/SearchTransformTests.java | 2 +- .../script/ScriptTransformTests.java | 2 +- 64 files changed, 1475 insertions(+), 234 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/script/TimeSeries.java create mode 100644 server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java create mode 100644 server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index 7e15acb99d4bd..253890cd2a175 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -2032,10 +2032,47 @@ Contains script statistics for the node. (integer) Total number of inline script compilations performed by the node. +`compilations_history`:: +(object) +Contains this recent history of script compilations + +.Properties of `compilations_history` +[%collapsible%open] +======= +`5m`:: +(long) +The number of script compilations in the last five minutes. +`15m`:: +(long) +The number of script compilations in the last fifteen minutes. +`24h`:: +(long) +The number of script compilations in the last twenty-four hours. +======= + `cache_evictions`:: (integer) Total number of times the script cache has evicted old data. + +`cache_evictions_history`:: +(object) +Contains this recent history of script cache evictions + +.Properties of `cache_evictions` +[%collapsible%open] +======= +`5m`:: +(long) +The number of script cache evictions in the last five minutes. +`15m`:: +(long) +The number of script cache evictions in the last fifteen minutes. +`24h`:: +(long) +The number of script cache evictions in the last twenty-four hours. +======= + `compilation_limit_triggered`:: (integer) Total number of times the <> to inspect +the number of recent cache evictions, `script.cache_evictions_history` and +compilations `script.compilations_history`. If there are a large +number of recent cache evictions or compilations, the script cache may be +undersized, consider doubling the size of the script cache via the setting +`script.cache.max_size`. + [[regex-circuit-breaker]] [discrete] ==== Regex circuit breaker diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java index b2a68c7cc25c6..9bc9aaf78f9e5 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java @@ -48,7 +48,7 @@ public boolean execute(Token token) { }; @SuppressWarnings("unchecked") - ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap()) { + ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L) { @Override public FactoryType compile(Script script, ScriptContext context) { assertEquals(context, AnalysisPredicateScript.CONTEXT); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java index c747d9af6ad30..a032cdd0b3081 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java @@ -48,7 +48,7 @@ public boolean execute(Token token) { }; @SuppressWarnings("unchecked") - ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap()) { + ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L) { @Override public FactoryType compile(Script script, ScriptContext context) { assertEquals(context, AnalysisPredicateScript.CONTEXT); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java index 2f025802a6138..8042253b8b1df 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java @@ -144,7 +144,8 @@ public void testInlineIsCompiled() throws Exception { return null; }), Collections.emptyMap()) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L ); factory = new ScriptProcessor.Factory(scriptService); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java index 5e12c15d2bba7..6add04b9b9b61 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java @@ -49,7 +49,8 @@ public void setupScripting() { return null; }), Collections.emptyMap()) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L ); script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()); ingestScript = scriptService.compile(script, IngestScript.CONTEXT).newInstance(script.getParams()); diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java index fb2bf9e2aee2b..eca978353fb0d 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java @@ -32,7 +32,7 @@ public abstract class AbstractScriptTestCase extends ESTestCase { public void init() throws Exception { MustacheScriptEngine engine = new MustacheScriptEngine(); Map engines = Collections.singletonMap(engine.getType(), engine); - scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } protected TemplateScript.Factory compile(String template) { diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 0d77f6db77898..c79f39225bd14 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -198,6 +198,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Function; +import java.util.function.LongSupplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -420,7 +421,12 @@ protected Node( client = new NodeClient(settings, threadPool); final ScriptModule scriptModule = new ScriptModule(settings, pluginsService.filterPlugins(ScriptPlugin.class)); - final ScriptService scriptService = newScriptService(settings, scriptModule.engines, scriptModule.contexts); + final ScriptService scriptService = newScriptService( + settings, + scriptModule.engines, + scriptModule.contexts, + threadPool::absoluteTimeInMillis + ); AnalysisModule analysisModule = new AnalysisModule(this.environment, pluginsService.filterPlugins(AnalysisPlugin.class)); // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool // so we might be late here already @@ -1487,8 +1493,13 @@ protected SearchService newSearchService( /** * Creates a new the ScriptService. This method can be overwritten by tests to inject mock implementations. */ - protected ScriptService newScriptService(Settings settings, Map engines, Map> contexts) { - return new ScriptService(settings, engines, contexts); + protected ScriptService newScriptService( + Settings settings, + Map engines, + Map> contexts, + LongSupplier timeProvider + ) { + return new ScriptService(settings, engines, contexts, timeProvider); } /** diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCache.java b/server/src/main/java/org/elasticsearch/script/ScriptCache.java index a55b37c7433ae..95aad9b086671 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptCache.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptCache.java @@ -23,6 +23,7 @@ import java.util.Objects; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.LongSupplier; /** * Script cache and compilation rate limiter. @@ -44,7 +45,13 @@ public class ScriptCache { private final double compilesAllowedPerNano; private final String contextRateSetting; - ScriptCache(int cacheMaxSize, TimeValue cacheExpire, CompilationRate maxCompilationRate, String contextRateSetting) { + ScriptCache( + int cacheMaxSize, + TimeValue cacheExpire, + CompilationRate maxCompilationRate, + String contextRateSetting, + LongSupplier timeProvider + ) { this.cacheSize = cacheMaxSize; this.cacheExpire = cacheExpire; this.contextRateSetting = contextRateSetting; @@ -63,7 +70,7 @@ public class ScriptCache { this.rate = maxCompilationRate; this.compilesAllowedPerNano = ((double) rate.count) / rate.time.nanos(); - this.scriptMetrics = new ScriptMetrics(); + this.scriptMetrics = new ScriptMetrics(timeProvider); this.tokenBucketState = new AtomicReference(new TokenBucketState(this.rate.count)); } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java b/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java index 4efdaf81b9986..baf8384daf8d2 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java @@ -133,7 +133,7 @@ public ScriptStats sum() { cacheEvictions += stat.getCacheEvictions(); compilationLimitTriggered += stat.getCompilationLimitTriggered(); } - return new ScriptStats(compilations, cacheEvictions, compilationLimitTriggered); + return new ScriptStats(compilations, cacheEvictions, compilationLimitTriggered, null, null); } static final class Fields { diff --git a/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java b/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java index c2b484e324567..acde189431c44 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java @@ -28,15 +28,13 @@ public class ScriptContextStats implements Writeable, ToXContentFragment, Compar public ScriptContextStats( String context, - long compilations, - long cacheEvictions, long compilationLimitTriggered, TimeSeries compilationsHistory, TimeSeries cacheEvictionsHistory ) { this.context = Objects.requireNonNull(context); - this.compilations = compilations; - this.cacheEvictions = cacheEvictions; + this.compilations = compilationsHistory.total; + this.cacheEvictions = cacheEvictionsHistory.total; this.compilationLimitTriggered = compilationLimitTriggered; this.compilationsHistory = compilationsHistory; this.cacheEvictionsHistory = cacheEvictionsHistory; @@ -47,12 +45,15 @@ public ScriptContextStats(StreamInput in) throws IOException { compilations = in.readVLong(); cacheEvictions = in.readVLong(); compilationLimitTriggered = in.readVLong(); - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_8_1_0)) { compilationsHistory = new TimeSeries(in); cacheEvictionsHistory = new TimeSeries(in); + } else if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + compilationsHistory = new TimeSeries(in).withTotal(compilations); + cacheEvictionsHistory = new TimeSeries(in).withTotal(cacheEvictions); } else { - compilationsHistory = null; - cacheEvictionsHistory = null; + compilationsHistory = new TimeSeries(compilations); + cacheEvictionsHistory = new TimeSeries(cacheEvictions); } } @@ -68,65 +69,6 @@ public void writeTo(StreamOutput out) throws IOException { } } - public static class TimeSeries implements Writeable, ToXContentFragment { - public final long fiveMinutes; - public final long fifteenMinutes; - public final long twentyFourHours; - - public TimeSeries() { - this.fiveMinutes = 0; - this.fifteenMinutes = 0; - this.twentyFourHours = 0; - } - - public TimeSeries(long fiveMinutes, long fifteenMinutes, long twentyFourHours) { - assert fiveMinutes >= 0; - this.fiveMinutes = fiveMinutes; - assert fifteenMinutes >= fiveMinutes; - this.fifteenMinutes = fifteenMinutes; - assert twentyFourHours >= fifteenMinutes; - this.twentyFourHours = twentyFourHours; - } - - public TimeSeries(StreamInput in) throws IOException { - fiveMinutes = in.readVLong(); - fifteenMinutes = in.readVLong(); - twentyFourHours = in.readVLong(); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(Fields.FIVE_MINUTES, fiveMinutes); - builder.field(Fields.FIFTEEN_MINUTES, fifteenMinutes); - builder.field(Fields.TWENTY_FOUR_HOURS, twentyFourHours); - return builder; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(fiveMinutes); - out.writeVLong(fifteenMinutes); - out.writeVLong(twentyFourHours); - } - - public boolean isEmpty() { - return twentyFourHours == 0; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TimeSeries that = (TimeSeries) o; - return fiveMinutes == that.fiveMinutes && fifteenMinutes == that.fifteenMinutes && twentyFourHours == that.twentyFourHours; - } - - @Override - public int hashCode() { - return Objects.hash(fiveMinutes, fifteenMinutes, twentyFourHours); - } - } - public String getContext() { return context; } @@ -158,7 +100,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.COMPILATIONS, getCompilations()); TimeSeries series = getCompilationsHistory(); - if (series != null && series.isEmpty() == false) { + if (series != null && series.areTimingsEmpty() == false) { builder.startObject(Fields.COMPILATIONS_HISTORY); series.toXContent(builder, params); builder.endObject(); @@ -166,7 +108,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.CACHE_EVICTIONS, getCacheEvictions()); series = getCacheEvictionsHistory(); - if (series != null && series.isEmpty() == false) { + if (series != null && series.areTimingsEmpty() == false) { builder.startObject(Fields.CACHE_EVICTIONS_HISTORY); series.toXContent(builder, params); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java index c4d26bc861c8c..c08d760108eb2 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java @@ -10,17 +10,24 @@ import org.elasticsearch.common.metrics.CounterMetric; +import java.util.function.LongSupplier; + public class ScriptMetrics { - final CounterMetric compilationsMetric = new CounterMetric(); - final CounterMetric cacheEvictionsMetric = new CounterMetric(); final CounterMetric compilationLimitTriggered = new CounterMetric(); + final TimeSeriesCounter compilations; + final TimeSeriesCounter cacheEvictions; + + public ScriptMetrics(LongSupplier timeProvider) { + compilations = new TimeSeriesCounter(timeProvider); + cacheEvictions = new TimeSeriesCounter(timeProvider); + } public void onCompilation() { - compilationsMetric.inc(); + compilations.inc(); } public void onCacheEviction() { - cacheEvictionsMetric.inc(); + cacheEvictions.inc(); } public void onCompilationLimit() { @@ -28,17 +35,20 @@ public void onCompilationLimit() { } public ScriptStats stats() { - return new ScriptStats(compilationsMetric.count(), cacheEvictionsMetric.count(), compilationLimitTriggered.count()); + TimeSeries compilationsTimeSeries = compilations.timeSeries(); + TimeSeries cacheEvictionsTimeSeries = cacheEvictions.timeSeries(); + return new ScriptStats( + compilationsTimeSeries.total, + cacheEvictionsTimeSeries.total, + compilationLimitTriggered.count(), + compilationsTimeSeries, + cacheEvictionsTimeSeries + ); } public ScriptContextStats stats(String context) { - return new ScriptContextStats( - context, - compilationsMetric.count(), - cacheEvictionsMetric.count(), - compilationLimitTriggered.count(), - new ScriptContextStats.TimeSeries(), - new ScriptContextStats.TimeSeries() - ); + TimeSeries compilationsTimeSeries = compilations.timeSeries(); + TimeSeries cacheEvictionsTimeSeries = cacheEvictions.timeSeries(); + return new ScriptContextStats(context, compilationLimitTriggered.count(), compilationsTimeSeries, cacheEvictionsTimeSeries); } } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java index 6ad520d5b8480..b741692618988 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java @@ -47,6 +47,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import java.util.function.LongSupplier; import java.util.stream.Collectors; public class ScriptService implements Closeable, ClusterStateApplier, ScriptCompiler { @@ -169,6 +170,7 @@ public class ScriptService implements Closeable, ClusterStateApplier, ScriptComp private final Map engines; private final Map> contexts; + private final LongSupplier timeProvider; private ClusterState clusterState; @@ -177,7 +179,12 @@ public class ScriptService implements Closeable, ClusterStateApplier, ScriptComp // package private for tests final AtomicReference cacheHolder = new AtomicReference<>(); - public ScriptService(Settings settings, Map engines, Map> contexts) { + public ScriptService( + Settings settings, + Map engines, + Map> contexts, + LongSupplier timeProvider + ) { this.engines = Collections.unmodifiableMap(Objects.requireNonNull(engines)); this.contexts = Collections.unmodifiableMap(Objects.requireNonNull(contexts)); @@ -274,6 +281,7 @@ public ScriptService(Settings settings, Map engines, Map context) { rate = new ScriptCache.CompilationRate(ScriptContext.DEFAULT_COMPILATION_RATE_LIMIT); } - return new ScriptCache(cacheSize, cacheExpire, rate, rateSetting.getKey()); + return new ScriptCache(cacheSize, cacheExpire, rate, rateSetting.getKey(), timeProvider); } /** @@ -759,9 +768,15 @@ static class CacheHolder { final ScriptCache general; final Map> contextCache; - CacheHolder(int cacheMaxSize, TimeValue cacheExpire, ScriptCache.CompilationRate maxCompilationRate, String contextRateSetting) { + CacheHolder( + int cacheMaxSize, + TimeValue cacheExpire, + ScriptCache.CompilationRate maxCompilationRate, + String contextRateSetting, + LongSupplier timeProvider + ) { contextCache = null; - general = new ScriptCache(cacheMaxSize, cacheExpire, maxCompilationRate, contextRateSetting); + general = new ScriptCache(cacheMaxSize, cacheExpire, maxCompilationRate, contextRateSetting, timeProvider); } CacheHolder(Map context) { diff --git a/server/src/main/java/org/elasticsearch/script/ScriptStats.java b/server/src/main/java/org/elasticsearch/script/ScriptStats.java index c6e9bb908dd59..f9d365db126a7 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptStats.java @@ -8,6 +8,7 @@ package org.elasticsearch.script; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -26,6 +27,8 @@ public class ScriptStats implements Writeable, ToXContentFragment { private final long compilations; private final long cacheEvictions; private final long compilationLimitTriggered; + private final TimeSeries compilationsHistory; + private final TimeSeries cacheEvictionsHistory; public ScriptStats(List contextStats) { ArrayList ctxStats = new ArrayList<>(contextStats.size()); @@ -43,30 +46,60 @@ public ScriptStats(List contextStats) { this.compilations = compilations; this.cacheEvictions = cacheEvictions; this.compilationLimitTriggered = compilationLimitTriggered; + this.compilationsHistory = new TimeSeries(compilations); + this.cacheEvictionsHistory = new TimeSeries(cacheEvictions); } - public ScriptStats(long compilations, long cacheEvictions, long compilationLimitTriggered) { + public ScriptStats( + long compilations, + long cacheEvictions, + long compilationLimitTriggered, + TimeSeries compilationsHistory, + TimeSeries cacheEvictionsHistory + ) { this.contextStats = Collections.emptyList(); this.compilations = compilations; this.cacheEvictions = cacheEvictions; this.compilationLimitTriggered = compilationLimitTriggered; + this.compilationsHistory = compilationsHistory == null ? new TimeSeries(compilations) : compilationsHistory; + this.cacheEvictionsHistory = cacheEvictionsHistory == null ? new TimeSeries(cacheEvictions) : cacheEvictionsHistory; } public ScriptStats(ScriptContextStats context) { - this(context.getCompilations(), context.getCacheEvictions(), context.getCompilationLimitTriggered()); + this( + context.getCompilations(), + context.getCacheEvictions(), + context.getCompilationLimitTriggered(), + context.getCompilationsHistory(), + context.getCacheEvictionsHistory() + ); } public ScriptStats(StreamInput in) throws IOException { - compilations = in.readVLong(); - cacheEvictions = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_8_1_0)) { + compilationsHistory = new TimeSeries(in); + cacheEvictionsHistory = new TimeSeries(in); + compilations = compilationsHistory.total; + cacheEvictions = cacheEvictionsHistory.total; + } else { + compilations = in.readVLong(); + cacheEvictions = in.readVLong(); + compilationsHistory = new TimeSeries(compilations); + cacheEvictionsHistory = new TimeSeries(cacheEvictions); + } compilationLimitTriggered = in.readVLong(); contextStats = in.readList(ScriptContextStats::new); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(compilations); - out.writeVLong(cacheEvictions); + if (out.getVersion().onOrAfter(Version.V_8_1_0)) { + compilationsHistory.writeTo(out); + cacheEvictionsHistory.writeTo(out); + } else { + out.writeVLong(compilations); + out.writeVLong(cacheEvictions); + } out.writeVLong(compilationLimitTriggered); out.writeList(contextStats); } @@ -104,6 +137,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.COMPILATIONS, compilations); builder.field(Fields.CACHE_EVICTIONS, cacheEvictions); builder.field(Fields.COMPILATION_LIMIT_TRIGGERED, compilationLimitTriggered); + if (compilationsHistory != null && compilationsHistory.areTimingsEmpty() == false) { + builder.startObject(ScriptContextStats.Fields.COMPILATIONS_HISTORY); + compilationsHistory.toXContent(builder, params); + builder.endObject(); + } + if (cacheEvictionsHistory != null && cacheEvictionsHistory.areTimingsEmpty() == false) { + builder.startObject(ScriptContextStats.Fields.COMPILATIONS_HISTORY); + cacheEvictionsHistory.toXContent(builder, params); + builder.endObject(); + } builder.startArray(Fields.CONTEXTS); for (ScriptContextStats contextStats : contextStats) { contextStats.toXContent(builder, params); diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeries.java b/server/src/main/java/org/elasticsearch/script/TimeSeries.java new file mode 100644 index 0000000000000..8399a65a57e08 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/TimeSeries.java @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * A response class representing a snapshot of a {@link org.elasticsearch.script.TimeSeriesCounter} at a point in time. + */ +public class TimeSeries implements Writeable, ToXContentFragment { + public final long fiveMinutes; + public final long fifteenMinutes; + public final long twentyFourHours; + public final long total; + + public TimeSeries(long total) { + this.fiveMinutes = 0; + this.fifteenMinutes = 0; + this.twentyFourHours = 0; + this.total = total; + } + + public TimeSeries(long fiveMinutes, long fifteenMinutes, long twentyFourHours, long total) { + this.fiveMinutes = fiveMinutes; + this.fifteenMinutes = fifteenMinutes; + this.twentyFourHours = twentyFourHours; + this.total = total; + } + + TimeSeries withTotal(long total) { + return new TimeSeries(fiveMinutes, fifteenMinutes, twentyFourHours, total); + } + + public TimeSeries(StreamInput in) throws IOException { + fiveMinutes = in.readVLong(); + fifteenMinutes = in.readVLong(); + twentyFourHours = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_8_1_0)) { + total = in.readVLong(); + } else { + total = 0; + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // total is omitted from toXContent as it's written at a higher level by ScriptContextStats + builder.field(ScriptContextStats.Fields.FIVE_MINUTES, fiveMinutes); + builder.field(ScriptContextStats.Fields.FIFTEEN_MINUTES, fifteenMinutes); + builder.field(ScriptContextStats.Fields.TWENTY_FOUR_HOURS, twentyFourHours); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(fiveMinutes); + out.writeVLong(fifteenMinutes); + out.writeVLong(twentyFourHours); + if (out.getVersion().onOrAfter(Version.V_8_1_0)) { + out.writeVLong(total); + } + } + + public boolean areTimingsEmpty() { + return fiveMinutes == 0 && fifteenMinutes == 0 && twentyFourHours == 0; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TimeSeries that = (TimeSeries) o; + return fiveMinutes == that.fiveMinutes + && fifteenMinutes == that.fifteenMinutes + && twentyFourHours == that.twentyFourHours + && total == that.total; + } + + @Override + public int hashCode() { + return Objects.hash(fiveMinutes, fifteenMinutes, twentyFourHours, total); + } + + @Override + public String toString() { + return "TimeSeries{" + + "fiveMinutes=" + + fiveMinutes + + ", fifteenMinutes=" + + fifteenMinutes + + ", twentyFourHours=" + + twentyFourHours + + ", total=" + + total + + '}'; + } +} diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java new file mode 100644 index 0000000000000..642c6e6547abb --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -0,0 +1,476 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.Arrays; +import java.util.concurrent.atomic.LongAdder; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.LongSupplier; + +/** + * Provides a counter with a history of 5m/15m/24h. + * + * Callers increment the counter and query the current state of the TimeSeries. + * + * {@link TimeSeriesCounter#inc()} increments the counter to indicate an event happens "now", with metadata about "now" + * from the given {@link TimeSeriesCounter#timeProvider}. + * + * {@link TimeSeriesCounter#timeSeries()} provides a snapshot of the counters at the current time, given by + * {@link TimeSeriesCounter#timeProvider}. The snapshot includes the number of events in the last five minutes, the last fifteen + * minutes, the last twenty-four hours and an all-time count. + * + * Caveats: + * * If the timeProvider produces a time stamp value, {@code t[j]} that occurs _before_ an earlier invocation {@code t[j]} (where j is an + * invocation that *happens-before*, in the java memory model sense), the time stamp is treated as occurring at the latest time seen, + * {@code t[latest]}, EXCEPT if {@code t[latest] - t[j]} is earlier than any time covered by the twenty-four counter. If that occurs, + * time goes backwards more than 24 hours (between 24:00 and 23:45 depending on the circumstance) the history is reset but total is + * retained. + * * All counters have a resolution: + * - 5m resolution is 15s + * - 15m resolution is 90s + * - 24h resolution is 15m + * The counter will drop events between one second and the resolution minus one second during a bucket rollover. + */ +public class TimeSeriesCounter { + public static final int SECOND = 1; + public static final int MINUTE = 60; + public static final int HOUR = 60 * MINUTE; + protected LongAdder adder = new LongAdder(); + + protected ReadWriteLock lock = new ReentrantReadWriteLock(); + + protected Counter fiveMinutes = new Counter(15 * SECOND, 5 * MINUTE); + protected Counter fifteenMinutes = new Counter(90 * SECOND, 15 * MINUTE); + protected Counter twentyFourHours = new Counter(15 * MINUTE, 24 * HOUR); + protected final LongSupplier timeProvider; + + /** + * Create a TimeSeriesCounter with the given {@code timeProvider}. + * + * The {@code timeProvider} must return positive values, in milliseconds. In live code, this is expected + * to be {@link System#currentTimeMillis()} or a proxy such as {@link ThreadPool#absoluteTimeInMillis()}. + */ + public TimeSeriesCounter(LongSupplier timeProvider) { + this.timeProvider = timeProvider; + } + + /** + * Increment counters at timestamp t, any increment more than 24hours before the current time + * series resets all historical counters, but the total counter is still increments. + */ + public void inc() { + long now = now(); + adder.increment(); + lock.writeLock().lock(); + try { + // If time has skewed more than a day in the past, reset the histories, but not the adder. + // Counters clamp all times before the end of the current bucket as happening in the current + // bucket, so this reset avoids pegging counters to their current buckets for too long. + if (now < twentyFourHours.earliestTimeInCounter()) { + fiveMinutes.reset(now); + fifteenMinutes.reset(now); + twentyFourHours.reset(now); + } else { + fiveMinutes.inc(now); + fifteenMinutes.inc(now); + twentyFourHours.inc(now); + } + } finally { + lock.writeLock().unlock(); + } + } + + /** + * Get the value of the counters for the last 5 minutes, last 15 minutes and the last 24 hours from + * t. May include events up to resolution before those durations due to counter granularity. + */ + public TimeSeries timeSeries() { + long now = now(); + lock.readLock().lock(); + try { + return new TimeSeries(fiveMinutes.sum(now), fifteenMinutes.sum(now), twentyFourHours.sum(now), count()); + } finally { + lock.readLock().unlock(); + } + } + + // The current time, in seconds, from the timeProvider, which emits milliseconds. Clamped to zero or positive numbers. + protected long now() { + long t = timeProvider.getAsLong(); + if (t < 0) { + t = 0; + } else { + t /= 1000; + } + return t; + } + + /** + * The total number of events for all time covered by the counters. + */ + public long count() { + long total = adder.sum(); + return total < 0 ? 0 : total; + } + + /* + * Keeps track event counts over a duration. Events are clamped to buckets, either the current bucket or a future + * bucket. A bucket represents all events over a period of resolution number of seconds. + */ + public static class Counter { + + /* + * In the following diagrams, we take a duration of 100 and resolution of 20. + * + * |___________________________________________________________| + * duration = 100 + * + * |___________|___________|___________|___________|___________| + * buckets = 5 + * + * |___________| + * resolution = 20 + * + * Action: inc(235) - Increment the counter at time 235 seconds. + * + * While there is only one `buckets` array, it's useful to view the array as overlapping three + * epoch (time at bucket[0]), the last epoch, the present epoch and the future epoch. + * + * Past + * [_] [_] [2] [3] [4] + * |___________|___________|___________|___________|___________| + * 140[e]-> 160-> 180-> 199 + * + * Present + * [0] [1][b] [2][g] [3] [4] + * |___________|_____1_____|___________|___________|___________| + * 200[a]-> 220-> 240[d]-> 260-> 280-> 299 + * + * Future + * [0] [_] [_] [_] [_] + * |___________|___________|___________|___________|___________| + * 300[c]-> 320[f] + * + * [a] Beginning of the current epoch + * startOfCurrentEpoch = 200 = (t / duration) * duration = (235 / 100) * 100 + * Start time of bucket zero, this is used to anchor the bucket ring in time. Without `startOfCurrentEpoch`, + * it would be impossible to distinguish between two times that are `duration` away from each other. + * In this example, the first inc used time 235, since startOfCurrentEpoch is rounded down to the nearest + * duration (100), it is 200. + * + * [b] The current bucket + * curBucket = 1 = (t / resolution) % buckets.length = (235 / 20) % 5 + * The latest active bucket in the bucket ring. The bucket of a timestamp is determined by the `resolution`. + * In this case the `resolution` is 20, so each bucket covers 20 seconds, the first covers 200-219, the + * second covers 220->239, the third 240->259 and so on. 235 is in the second bucket, at index 1. + * + * [c] Beginning of the next epoch + * nextEpoch() = 300 = startOfCurrentEpoch + duration = 200 + 100 + * The first time of the next epoch, this indicates when `startOfCurrentEpoch` should be updated. When `curBucket` + * advances to or past zero, `startOfCurrentEpoch` must be updated to `nextEpoch()` + * + * [d] Beginning of the next bucket + * nextBucketStartTime() = 240 = startOfCurrentEpoch + ((curBucket + 1) * resolution) = 200 + ((1 + 1) * 20 + * The first time of the next bucket, when a timestamp is greater than or equal to this time, we must update + * the `curBucket` and potentially the `startOfCurrentEpoch`. + * + * [e] The earliest time to sum + * earliestTimeInCounter() = 140 = nextBucketStartTime() - duration = 240 - 100 + * `curBucket` covers the latest timestamp seen by the `Counter`. Since the counter keeps a history, when a + * caller calls `sum(t)`, the `Counter` must clamp the range to the earliest time covered by its current state. + * The times proceed backwards for `buckets.length - 1`. + * **Important** this is likely _before_ the `startOfCurrentEpoch`. `startOfCurrentEpoch` is the timestamp of bucket[0]. + * + * [f] The counter is no longer valid at this time + * counterExpired() = 320 = startOfCurrentEpoch + (curBucket * resolution) + duration = 200 + (1 * 20) + 100 + * Where `earliestTimeInCounter()` is looking backwards, to the history covered by the counter, `counterExpired()` + * looks forward to when current counter has expired. Since `curBucket` represents the latest time in this counter, + * `counterExpired()` is `duration` from the start of the time covered from `curBucket` + * + * [g] The next bucket in the bucket ring + * nextBucket(curBucket) = 2 = (i + 1) % buckets.length = (1 + 1) % 5 + * Since `buckets` is a ring, the next bucket may wrap around. + * + * ------------------------------------------------------------------------------------------------------------------ + * + * Action: inc(238) - since this inc is within the current bucket, it is incremented and nothing else changes + * + * Present + * [0] [1][b] [2][g] [3] [4] + * |___________|_____2_____|___________|___________|___________| + * 200[a]-> 220-> 240[d]-> 260-> 280-> 299 + * + * ------------------------------------------------------------------------------------------------------------------ + * + * Action: inc(165) - only the current bucket is incremented, so increments from a timestamp in the past are + * clamped to the current bucket. This makes `inc(long)` dependent on the ordering of timestamps, + * but it avoids revising a history that may have already been exposed via `sum(long)`. + * + * Present + * [0] [1][b] [2][g] [3] [4] + * |___________|_____3_____|___________|___________|___________| + * 200[a]-> 220-> 240[d]-> 260-> 280-> 299 + * + * ------------------------------------------------------------------------------------------------------------------ + * + * Action: inc(267) - 267 is in bucket 3, so bucket 2 is zeroed and skipped. Bucket 2 is zeroed because it may have + * had contents that were relevant for timestamps 140 - 159. + * + * The `startOfCurrentEpoch`[a], does not change while `curBucket`[b] is now bucket 3. + * + * `nextEpoch()`[c] does not change as there hasn't been a rollover. + * + * `nextBucketStartTime()`[d] is now 280, the start time of bucket 4. + * + * `earliestTimeInCounter()`[e] is now 180, bucket 2 was zeroed, erasing the history from 140-159 and + * bucket 3 was set to 1, now representing 260-279 rather than 160-179. + * + * `counterExpired()`[f] is now 360. Bucket 3 in the current epoch represents 260->279, an + * `inc(long)` any of time (260 + `duration`) or beyond would require clearing all `buckets` in the + * `Counter` and any `sum(long)` that starts at 360 or later does not cover the valid time range for + * this state of the counter. + * + * `nextBucket(curBucket)`[g] is now 4, the bucket after 3. + * + * + * Past + * [_] [_] [_] [_] [4] + * |___________|___________|___________|___________|___________| + * 180[e]-> 199 + * + * Present + * [0] [1] [2] [3][b] [4][g] + * |___________|_____3_____|___________|______1____|___________| + * 200[a]-> 220-> 240-> 260-> 280[d]-> 299 + * + * Future + * [0] [1] [2] [_] [_] + * |___________|___________|___________|___________|___________| + * 300[c]-> 320-> 340-> 360[f]-> + * + * ------------------------------------------------------------------------------------------------------------------ + * + * Action: inc(310) - 310 is in bucket 0, so bucket 4 is zeroed and skipped, as it may have had contents + * for timestamps 180-199. + * + * The `startOfCurrentEpoch`[a], is now 300 as the `Counter` has rolled through bucket 0. + * + * `curBucket`[b] is now bucket 0. + * + * `nextEpoch()`[c] is now 400 because `startOfCurrentEpoch`[a] has changed. + * + * `nextBucketStartTime()`[d] is now 320, the start time of bucket 1 in this new epoch. + * + * `earliestTimeInCounter()`[e] is now 220, bucket 4 was zeroed, erasing the history from 180-199 and + * bucket 0 was set to 1, now representing 300-319 due to the epoch change, rather than 200-219, so + * 220 is the earliest time available in the `Counter`. + * + * `counterExpired()`[f] is now 400. Bucket 0 in the current epoch represents 300-319, an + * `inc(long)` any of time (300 + `duration`) or beyond would require clearing all `buckets` in the + * `Counter` and any `sum(long)` that starts at 400 or later does not cover the valid time range for + * this state of the counter. + * + * `nextBucket(curBucket)`[g] is now 1, the bucket after 0. + * + * + * Past + * [_] [1] [2] [3] [4] + * |___________|_____3_____|___________|______1____|___________| + * 220[e]-> 240-> 260-> 280-> 299 + * + * Present + * [0][b] [1][g] [2] [3] [4] + * |_____1_____|___________|___________|___________|___________| + * 300[a]-> 320[d]-> 340-> 360-> 380-> 399 + * + * Future + * [_] [_] [_] [_] [_] + * |___________|___________|___________|___________|___________| + * 400[c][f]-> + * + * ------------------------------------------------------------------------------------------------------------------ + * + * Action: inc(321) - 321 is in bucket 1, so the previous contents of bucket 1 is replaced with the value 1. + * + * The `startOfCurrentEpoch`[a] remains 300. + * + * `curBucket`[b] is now bucket 1. + * + * `nextEpoch()`[c] remains 400. + * + * `nextBucketStartTime()`[d] is now 340, the start time of bucket 2. + * + * `earliestTimeInCounter()`[e] is now 240 as bucket 1 now represents 320-339 rather than 220-239. + * + * `counterExpired()`[f] is now 420. Bucket 1 in the current epoch represents 320-339, an + * `inc(long)` any of time (320 + `duration`) or beyond would require clearing all `buckets` in the + * `Counter` and any `sum(long)` that starts at 420 or later does not cover the valid time range for + * this state of the counter. + * + * `nextBucket(curBucket)`[g] is now 2, the bucket after 1. + * + * Past + * [_] [_] [2] [3] [4] + * |___________|___________|___________|______1____|___________| + * 240[e]-> 260-> 280-> 299 + * + * Present + * [0] [1][b] [2][g] [3] [4] + * |_____1_____|_____1_____|___________|___________|___________| + * 300[a]-> 320-> 340[d]-> 360-> 380-> 399 + * + * Future + * [0] [_] [_] [_] [_] + * |_____0_____|___________|___________|___________|___________| + * 400[c]-> 420[f]-> + * + * + * ------------------------------------------------------------------------------------------------------------------ + * + * Action: sum(321) - This is a sum at the exact time of the last update, because of the `earliestTimeInCounter` check, + * it starts at bucket 2, which is after the current bucket index, 1, but the earliest time covered by + * the counter, 240 to 259. + * 1) calculate start = 321 - duration = 321 - 100 = 221 + * 2) start is before the nextBucketStartTime (340), so sum does not terminate early + * 3) start is before the earliestTimeInCounter (240) -> start = 240 + * 3) Iterate from bucket(start) = bucket(240) = 2 until curBucket 1, summing the following + * bucket 2 = 0, bucket 3 = 1, bucket 4 = 0, bucket 0 = 1 -> 1 + 1 = 2 + * 4) return that with the context of bucket 1 = 1 -> 2 + 1 = 3 + * + * Action: sum(465) - This sum is so far in the future, it does not overlap with any bucket in range + * 1) calculate start = 465 - duration = 465 - 100 = 365 + * 2) start is greater than or equal to the nextBucketStartTime (340), so we know the counter has no contexts + * -> return 0 + * + * Action: sum(439) - This sum starts _after_ the last update, which is at 321, but because of the bucket resolution + * sum still catches the value bucket 1, times 320 to 339. + * 1) calculate start = 439 - duration = 439 - 100 = 339 + * 2) start is before nextBucketStartTime(340), so sum does not terminate early + * 3) start is after earliestTimeInCounter (240), so it is now updated + * 4) bucket(start) = 1 which is curBucket, so the for loop falls through + * 5) return total = 0 + buckets[curBucket] = 0 + 1 = 1 + */ + protected final long resolution; + protected final long duration; + protected final long[] buckets; + + // The start time of buckets[0]. bucket(t + (i * duration)) is the same for all i. startOfCurrentEpoch allows the counter + // to differentiate between those times. + protected long startOfCurrentEpoch; + protected int curBucket = 0; + + /** + * Create a Counter that covers duration seconds at the given resolution. Duration must be divisible by resolution. + */ + public Counter(long resolution, long duration) { + if (resolution <= 0) { + throw new IllegalArgumentException("resolution [" + resolution + "] must be greater than zero"); + } else if (duration <= 0) { + throw new IllegalArgumentException("duration [" + duration + "] must be greater than zero"); + } else if (duration % resolution != 0) { + throw new IllegalArgumentException("duration [" + duration + "] must divisible by resolution [" + resolution + "]"); + } + this.resolution = resolution; + this.duration = duration; + this.buckets = new long[(int) (duration / resolution)]; + this.startOfCurrentEpoch = 0; + assert buckets.length > 0; + } + + /** + * Increment the counter at time {@code now}, expressed in seconds. + */ + public void inc(long now) { + if (now < nextBucketStartTime()) { + buckets[curBucket]++; + } else if (now >= counterExpired()) { + reset(now); + } else { + int dstBucket = bucket(now); + for (int i = nextBucket(curBucket); i != dstBucket; i = nextBucket(i)) { + buckets[i] = 0; + } + curBucket = dstBucket; + buckets[curBucket] = 1; + if (now >= nextEpoch()) { + startOfCurrentEpoch = epoch(now); + } + } + } + + /** + * sum for the duration of the counter until {@code now}. + */ + public long sum(long now) { + long start = now - duration; + if (start >= nextBucketStartTime()) { + return 0; + } + + if (start < earliestTimeInCounter()) { + start = earliestTimeInCounter(); + } + + long total = 0; + for (int i = bucket(start); i != curBucket; i = nextBucket(i)) { + total += buckets[i]; + } + return total + buckets[curBucket]; + } + + /** + * Reset the counter. Next counter begins at now. + */ + void reset(long now) { + Arrays.fill(buckets, 0); + startOfCurrentEpoch = epoch(now); + curBucket = bucket(now); + buckets[curBucket] = 1; + } + + // The time at bucket[0] for the given timestamp. + long epoch(long t) { + return (t / duration) * duration; + } + + // What is the start time of the next epoch? + long nextEpoch() { + return startOfCurrentEpoch + duration; + } + + // What is the earliest time covered by this counter? Counters do not extend before zero. + long earliestTimeInCounter() { + long time = nextBucketStartTime() - duration; + return time <= 0 ? 0 : time; + } + + // When does this entire counter expire? + long counterExpired() { + return startOfCurrentEpoch + (curBucket * resolution) + duration; + } + + // bucket for the given time + int bucket(long t) { + return (int) (t / resolution) % buckets.length; + } + + // the next bucket in the circular bucket array + int nextBucket(int i) { + return (i + 1) % buckets.length; + } + + // When does the next bucket start? + long nextBucketStartTime() { + return startOfCurrentEpoch + ((curBucket + 1) * resolution); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index 364f74bdc902a..144c9f0843441 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.script.ScriptCacheStats; import org.elasticsearch.script.ScriptContextStats; import org.elasticsearch.script.ScriptStats; +import org.elasticsearch.script.TimeSeries; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPoolStats; @@ -285,8 +286,8 @@ public void testSerialization() throws IOException { compilations += generatedStats.getCompilations(); assertEquals(generatedStats.getCompilations(), deserStats.getCompilations()); - assertEquals(generatedStats.getCacheEvictionsHistory(), deserStats.getCacheEvictionsHistory()); - assertEquals(generatedStats.getCompilationsHistory(), deserStats.getCompilationsHistory()); + assertEquals(generatedStats.getCacheEvictions(), deserStats.getCacheEvictions()); + assertEquals(generatedStats.getCompilations(), deserStats.getCompilations()); } assertEquals(evictions, scriptStats.getCacheEvictions()); assertEquals(limited, scriptStats.getCompilationLimitTriggered()); @@ -718,13 +719,9 @@ public static NodeStats createNodeStats() { List stats = new ArrayList<>(numContents); HashSet contexts = new HashSet<>(); for (int i = 0; i < numContents; i++) { - long compile = randomLongBetween(0, 1024); - long eviction = randomLongBetween(0, 1024); String context = randomValueOtherThanMany(contexts::contains, () -> randomAlphaOfLength(12)); contexts.add(context); - stats.add( - new ScriptContextStats(context, compile, eviction, randomLongBetween(0, 1024), randomTimeSeries(), randomTimeSeries()) - ); + stats.add(new ScriptContextStats(context, randomLongBetween(0, 1024), randomTimeSeries(), randomTimeSeries())); } scriptStats = new ScriptStats(stats); } @@ -872,14 +869,15 @@ public static NodeStats createNodeStats() { ); } - private static ScriptContextStats.TimeSeries randomTimeSeries() { + private static TimeSeries randomTimeSeries() { if (randomBoolean()) { - long day = randomLongBetween(0, 1024); + long total = randomLongBetween(0, 1024); + long day = total >= 1 ? randomLongBetween(0, total) : 0; long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; - return new ScriptContextStats.TimeSeries(five, fifteen, day); + return new TimeSeries(five, fifteen, day, day); } else { - return new ScriptContextStats.TimeSeries(); + return new TimeSeries(randomLongBetween(0, 1024)); } } diff --git a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index a72f4653fcd04..4634f892c12dd 100644 --- a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -112,7 +112,7 @@ public void setUp() throws Exception { scripts.put("return", vars -> null); final MockScriptEngine engine = new MockScriptEngine("mock", scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(engine.getType(), engine); - ScriptService scriptService = new ScriptService(baseSettings, engines, ScriptModule.CORE_CONTEXTS); + ScriptService scriptService = new ScriptService(baseSettings, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); updateHelper = new UpdateHelper(scriptService); } diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 0c3e3a412ea2f..315022ef7f4e3 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -164,7 +164,7 @@ public void setUp() throws Exception { circuitBreakerService = new NoneCircuitBreakerService(); PageCacheRecycler pageCacheRecycler = new PageCacheRecycler(settings); bigArrays = new BigArrays(pageCacheRecycler, circuitBreakerService, CircuitBreaker.REQUEST); - scriptService = new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap()); + scriptService = new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L); clusterService = ClusterServiceUtils.createClusterService(threadPool); nodeEnvironment = new NodeEnvironment(settings, environment); mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index 920abe4786b27..9c09c6ad5e664 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -29,7 +29,7 @@ public class MappingParserTests extends MapperServiceTestCase { private static MappingParser createMappingParser(Settings settings) { - ScriptService scriptService = new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap()); + ScriptService scriptService = new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L); IndexSettings indexSettings = createIndexSettings(Version.CURRENT, settings); IndexAnalyzers indexAnalyzers = createIndexAnalyzers(); SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, Collections.emptyMap()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TestScriptEngine.java b/server/src/test/java/org/elasticsearch/index/mapper/TestScriptEngine.java index 7a4c16fe6b95a..5fd2ed03d9ff7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TestScriptEngine.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TestScriptEngine.java @@ -28,7 +28,7 @@ protected Object buildScriptFactory(ScriptContext context) { public Set> getSupportedContexts() { return Set.of(context); } - }), Map.of(context.name, context)); + }), Map.of(context.name, context), () -> 1L); } @Override diff --git a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java index ad685a5abb243..e44a790a1fd0d 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java @@ -59,7 +59,8 @@ public void testChecksCondition() throws Exception { Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L ); Map document = new HashMap<>(); LongSupplier relativeTimeProvider = mock(LongSupplier.class); @@ -150,7 +151,8 @@ public void testTypeDeprecation() throws Exception { return true; }), Collections.emptyMap()) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L ); LongSupplier relativeTimeProvider = mock(LongSupplier.class); @@ -258,7 +260,8 @@ private static void assertMutatingCtxThrows(Consumer> mutati return false; }), Collections.emptyMap()) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L ); Map document = new HashMap<>(); ConditionalProcessor processor = new ConditionalProcessor( diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 19820dcd07cba..c52e29a516f5b 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -546,7 +546,8 @@ public void testGetProcessorsInPipelineComplexConditional() throws Exception { return true; }), Collections.emptyMap()) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L ); Map processors = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java index 0f70afd8402b0..7280294ad7260 100644 --- a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java @@ -165,7 +165,8 @@ public void testActualCompoundProcessorWithOnFailureAndTrueCondition() throws Ex Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> true), Collections.emptyMap()) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L ); RuntimeException exception = new RuntimeException("fail"); TestProcessor failProcessor = new TestProcessor("fail", "test", null, exception); @@ -253,7 +254,8 @@ public void testActualCompoundProcessorWithFalseConditional() throws Exception { Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap()) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L ); CompoundProcessor compoundProcessor = new CompoundProcessor( @@ -380,7 +382,8 @@ public void testActualPipelineProcessorWithTrueConditional() throws Exception { Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> true), Collections.emptyMap()) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L ); Pipeline pipeline1 = new Pipeline( @@ -479,7 +482,8 @@ public void testActualPipelineProcessorWithFalseConditional() throws Exception { Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap()) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L ); Pipeline pipeline1 = new Pipeline( diff --git a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java index 97e82173b4c7e..9e4c77ed7be40 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java @@ -13,9 +13,12 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; +import java.util.function.LongSupplier; import java.util.stream.Collectors; public class ScriptCacheTests extends ESTestCase { + private static final LongSupplier time = () -> 1L; + // even though circuit breaking is allowed to be configured per minute, we actually weigh this over five minutes // simply by multiplying by five, so even setting it to one, requires five compilations to break public void testCompilationCircuitBreaking() throws Exception { @@ -33,27 +36,29 @@ public void testCompilationCircuitBreaking() throws Exception { size, expire, new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), - rateSettingName + rateSettingName, + () -> 1L ); cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), rateSettingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), rateSettingName, time); cache.checkCompilationLimit(); // should pass cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); int count = randomIntBetween(5, 50); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), rateSettingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), rateSettingName, time); for (int i = 0; i < count; i++) { cache.checkCompilationLimit(); // should pass } expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), rateSettingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), rateSettingName, time); expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); cache = new ScriptCache( size, expire, new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), - rateSettingName + rateSettingName, + () -> 1L ); int largeLimit = randomIntBetween(1000, 10000); for (int i = 0; i < largeLimit; i++) { @@ -65,26 +70,33 @@ public void testGeneralCompilationCircuitBreaking() throws Exception { final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY); final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY); String settingName = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(); - ScriptCache cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), settingName); + ScriptCache cache = new ScriptCache( + size, + expire, + new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), + settingName, + () -> 1L + ); cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), settingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), settingName, () -> 1L); cache.checkCompilationLimit(); // should pass cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); int count = randomIntBetween(5, 50); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), settingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), settingName, () -> 1L); for (int i = 0; i < count; i++) { cache.checkCompilationLimit(); // should pass } expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), settingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), settingName, () -> 1L); expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); cache = new ScriptCache( size, expire, new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), - settingName + settingName, + () -> 1L ); int largeLimit = randomIntBetween(1000, 10000); for (int i = 0; i < largeLimit; i++) { @@ -99,7 +111,7 @@ public void testUnlimitedCompilationRate() { final Integer size = ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY); final TimeValue expire = ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY); String settingName = ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context).getKey(); - ScriptCache cache = new ScriptCache(size, expire, ScriptCache.UNLIMITED_COMPILATION_RATE, settingName); + ScriptCache cache = new ScriptCache(size, expire, ScriptCache.UNLIMITED_COMPILATION_RATE, settingName, time); ScriptCache.TokenBucketState initialState = cache.tokenBucketState.get(); for (int i = 0; i < 3000; i++) { cache.checkCompilationLimit(); @@ -113,7 +125,7 @@ public void testGeneralUnlimitedCompilationRate() { final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY); final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY); String settingName = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(); - ScriptCache cache = new ScriptCache(size, expire, ScriptCache.UNLIMITED_COMPILATION_RATE, settingName); + ScriptCache cache = new ScriptCache(size, expire, ScriptCache.UNLIMITED_COMPILATION_RATE, settingName, () -> 1L); ScriptCache.TokenBucketState initialState = cache.tokenBucketState.get(); for (int i = 0; i < 3000; i++) { cache.checkCompilationLimit(); diff --git a/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java b/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java index 1a0e5c207e1c5..d9186758f9025 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java @@ -60,7 +60,7 @@ private ScriptService getMockScriptService(Settings settings) { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(settings, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(settings, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public interface MiscContext { @@ -86,7 +86,7 @@ public void testOnlyScriptEngineContextsReturned() { Map> mockAndMiscContexts = new HashMap<>(mockContexts); mockAndMiscContexts.put(miscContext, new ScriptContext<>(miscContext, MiscContext.class)); - ScriptService ss = new ScriptService(Settings.EMPTY, engines, mockAndMiscContexts); + ScriptService ss = new ScriptService(Settings.EMPTY, engines, mockAndMiscContexts, () -> 1L); ScriptLanguagesInfo info = ss.getScriptLanguages(); assertTrue(info.languageContexts.containsKey(MockScriptEngine.NAME)); @@ -115,7 +115,7 @@ public void testContextsAllowedSettingRespected() { Map> mockAndMiscContexts = new HashMap<>(mockContexts); mockAndMiscContexts.put(miscContext, new ScriptContext<>(miscContext, MiscContext.class)); - ScriptService ss = new ScriptService(settings.build(), engines, mockAndMiscContexts); + ScriptService ss = new ScriptService(settings.build(), engines, mockAndMiscContexts, () -> 1L); ScriptLanguagesInfo info = ss.getScriptLanguages(); assertTrue(info.languageContexts.containsKey(MockScriptEngine.NAME)); diff --git a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index 2ce18979099f6..e8cf7501ab593 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -79,7 +79,7 @@ public void setup() throws IOException { private void buildScriptService(Settings additionalSettings) throws IOException { Settings finalSettings = Settings.builder().put(baseSettings).put(additionalSettings).build(); - scriptService = new ScriptService(finalSettings, engines, contexts) { + scriptService = new ScriptService(finalSettings, engines, contexts, () -> 1L) { @Override Map getScriptsFromClusterState() { Map scripts = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java index abaf64088f5da..4bf3b74be7d49 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java @@ -28,15 +28,8 @@ public class ScriptStatsTests extends ESTestCase { public void testXContent() throws IOException { List contextStats = List.of( - new ScriptContextStats( - "contextB", - 100, - 201, - 302, - new ScriptContextStats.TimeSeries(1000, 1001, 1002), - new ScriptContextStats.TimeSeries(2000, 2001, 2002) - ), - new ScriptContextStats("contextA", 1000, 2010, 3020, null, new ScriptContextStats.TimeSeries(0, 0, 0)) + new ScriptContextStats("contextB", 302, new TimeSeries(1000, 1001, 1002, 100), new TimeSeries(2000, 2001, 2002, 201)), + new ScriptContextStats("contextA", 3020, new TimeSeries(1000), new TimeSeries(2010)) ); ScriptStats stats = new ScriptStats(contextStats); final XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); @@ -79,8 +72,7 @@ public void testXContent() throws IOException { } public void testSerializeEmptyTimeSeries() throws IOException { - ScriptContextStats.TimeSeries empty = new ScriptContextStats.TimeSeries(); - ScriptContextStats stats = new ScriptContextStats("c", 1111, 2222, 3333, null, empty); + ScriptContextStats stats = new ScriptContextStats("c", 3333, new TimeSeries(1111), new TimeSeries(2222)); XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); stats.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -96,20 +88,13 @@ public void testSerializeEmptyTimeSeries() throws IOException { } public void testSerializeTimeSeries() throws IOException { - Function mkContextStats = (ts) -> new ScriptContextStats( - "c", - 1111, - 2222, - 3333, - null, - ts - ); + Function mkContextStats = (ts) -> new ScriptContextStats("c", 3333, new TimeSeries(1111), ts); - ScriptContextStats.TimeSeries series = new ScriptContextStats.TimeSeries(0, 0, 5); + TimeSeries series = new TimeSeries(0, 0, 5, 2222); String format = "{\n" + " \"context\" : \"c\",\n" + " \"compilations\" : 1111,\n" - + " \"cache_evictions\" : 2222,\n" + + " \"cache_evictions\" : %d,\n" + " \"cache_evictions_history\" : {\n" + " \"5m\" : %d,\n" + " \"15m\" : %d,\n" @@ -121,31 +106,27 @@ public void testSerializeTimeSeries() throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS); - assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 0, 0, 5))); + assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 2222, 0, 0, 5))); - series = new ScriptContextStats.TimeSeries(0, 7, 1234); + series = new TimeSeries(0, 7, 1234, 5678); builder = XContentFactory.jsonBuilder().prettyPrint(); mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS); - assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 0, 7, 1234))); + assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 5678, 0, 7, 1234))); - series = new ScriptContextStats.TimeSeries(123, 456, 789); + series = new TimeSeries(123, 456, 789, 91011); builder = XContentFactory.jsonBuilder().prettyPrint(); mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS); - assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 123, 456, 789))); - } - - public void testTimeSeriesAssertions() { - expectThrows(AssertionError.class, () -> new ScriptContextStats.TimeSeries(-1, 1, 2)); - expectThrows(AssertionError.class, () -> new ScriptContextStats.TimeSeries(1, 0, 2)); - expectThrows(AssertionError.class, () -> new ScriptContextStats.TimeSeries(1, 3, 2)); + assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 91011, 123, 456, 789))); } public void testTimeSeriesIsEmpty() { - assertTrue((new ScriptContextStats.TimeSeries(0, 0, 0)).isEmpty()); - long day = randomLongBetween(1, 1024); + assertTrue((new TimeSeries(0, 0, 0, 0)).areTimingsEmpty()); + assertTrue((new TimeSeries(0, 0, 0, 123)).areTimingsEmpty()); + long total = randomLongBetween(1, 1024); + long day = randomLongBetween(1, total); long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; - assertFalse((new ScriptContextStats.TimeSeries(five, fifteen, day)).isEmpty()); + assertFalse((new TimeSeries(five, fifteen, day, 0)).areTimingsEmpty()); } public void testTimeSeriesSerialization() throws IOException { @@ -155,8 +136,10 @@ public void testTimeSeriesSerialization() throws IOException { assertEquals(stats.getCompilations(), deserStats.getCompilations()); assertEquals(stats.getCacheEvictions(), deserStats.getCacheEvictions()); assertEquals(stats.getCompilationLimitTriggered(), deserStats.getCompilationLimitTriggered()); - assertNull(deserStats.getCompilationsHistory()); - assertNull(deserStats.getCacheEvictionsHistory()); + assertTrue(deserStats.getCompilationsHistory().areTimingsEmpty()); + assertEquals(stats.getCompilations(), deserStats.getCompilationsHistory().total); + assertTrue(deserStats.getCacheEvictionsHistory().areTimingsEmpty()); + assertEquals(stats.getCacheEvictions(), deserStats.getCacheEvictionsHistory().total); deserStats = serDeser(Version.V_8_0_0, Version.V_8_0_0, stats); assertEquals(stats.getCompilations(), deserStats.getCompilations()); @@ -164,6 +147,20 @@ public void testTimeSeriesSerialization() throws IOException { assertEquals(stats.getCompilationLimitTriggered(), deserStats.getCompilationLimitTriggered()); assertEquals(stats.getCompilationsHistory(), deserStats.getCompilationsHistory()); assertEquals(stats.getCacheEvictionsHistory(), deserStats.getCacheEvictionsHistory()); + + deserStats = serDeser(Version.V_8_1_0, Version.V_7_16_0, stats); + assertEquals(stats.getCompilations(), deserStats.getCompilations()); + assertEquals(stats.getCacheEvictions(), deserStats.getCacheEvictions()); + assertEquals(stats.getCompilationLimitTriggered(), deserStats.getCompilationLimitTriggered()); + assertEquals(new TimeSeries(stats.getCompilationsHistory().total), deserStats.getCompilationsHistory()); + assertEquals(new TimeSeries(stats.getCacheEvictionsHistory().total), deserStats.getCacheEvictionsHistory()); + + deserStats = serDeser(Version.V_8_1_0, Version.V_8_1_0, stats); + assertEquals(stats.getCompilations(), deserStats.getCompilations()); + assertEquals(stats.getCacheEvictions(), deserStats.getCacheEvictions()); + assertEquals(stats.getCompilationLimitTriggered(), deserStats.getCompilationLimitTriggered()); + assertEquals(stats.getCompilationsHistory(), deserStats.getCompilationsHistory()); + assertEquals(stats.getCacheEvictionsHistory(), deserStats.getCacheEvictionsHistory()); } public ScriptContextStats serDeser(Version outVersion, Version inVersion, ScriptContextStats stats) throws IOException { @@ -179,24 +176,17 @@ public ScriptContextStats serDeser(Version outVersion, Version inVersion, Script public ScriptContextStats randomStats() { long[] histStats = { randomLongBetween(0, 2048), randomLongBetween(0, 2048) }; - List timeSeries = new ArrayList<>(); + List timeSeries = new ArrayList<>(); for (int j = 0; j < 2; j++) { if (randomBoolean() && histStats[j] > 0) { long day = randomLongBetween(0, histStats[j]); long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; - timeSeries.add(new ScriptContextStats.TimeSeries(five, fifteen, day)); + timeSeries.add(new TimeSeries(five, fifteen, day, histStats[j])); } else { - timeSeries.add(new ScriptContextStats.TimeSeries()); + timeSeries.add(new TimeSeries(histStats[j])); } } - return new ScriptContextStats( - randomAlphaOfLength(15), - histStats[0], - histStats[1], - randomLongBetween(0, 1024), - timeSeries.get(0), - timeSeries.get(1) - ); + return new ScriptContextStats(randomAlphaOfLength(15), randomLongBetween(0, 1024), timeSeries.get(0), timeSeries.get(1)); } } diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java new file mode 100644 index 0000000000000..1eed55e5112d3 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java @@ -0,0 +1,548 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matcher; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.LongSupplier; + +import static org.elasticsearch.script.TimeSeriesCounter.Counter; +import static org.elasticsearch.script.TimeSeriesCounter.HOUR; +import static org.hamcrest.Matchers.lessThan; + +public class TimeSeriesCounterTests extends ESTestCase { + protected long now; + protected long customCounterResolution; + protected long customCounterDuration; + protected TimeProvider timeProvider = new TimeProvider(); + protected TimeSeriesCounter tsc = new TimeSeriesCounter(timeProvider); + protected final Matcher fiveDelta = lessThan(tsc.fiveMinutes.resolution); + protected final Matcher fifteenDelta = lessThan(tsc.fifteenMinutes.resolution); + protected final Matcher twentyFourDelta = lessThan(tsc.twentyFourHours.resolution); + protected List events; + protected Counter counter; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + now = 1635182590; + customCounterResolution = 45; + customCounterDuration = 900; + reset(); + } + + protected void reset() { + timeProvider = new TimeProvider(); + events = new ArrayList<>(); + tsc = new TimeSeriesCounter(timeProvider); + counter = new Counter(customCounterResolution, customCounterDuration); + } + + public void testCounterNegativeResolution() { + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> new Counter(-20, 200)); + assertEquals("resolution [-20] must be greater than zero", iae.getMessage()); + } + + public void testCounterNegativeDuration() { + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> new Counter(20, -200)); + assertEquals("duration [-200] must be greater than zero", iae.getMessage()); + } + + public void testCounterIndivisibleResolution() { + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> new Counter(3, 101)); + assertEquals("duration [101] must divisible by resolution [3]", iae.getMessage()); + } + + public void testNegativeIncrement() { + inc(-100); + assertEquals(1, timeSeries(0).fiveMinutes); + } + + public void testNegativeSum() { + long t = 60; + // t += 24 * HOUR; + inc(t); + t += 2 * tsc.twentyFourHours.resolution; + inc(t); + TimeSeries ts = timeSeries(t); + assertEquals(2, ts.twentyFourHours); + } + + public void testNegativeStart() { + long t = -1 * 48 * HOUR; + inc(t); + t += 2 * tsc.twentyFourHours.resolution; + inc(t); + TimeSeries ts = timeSeries(t); + assertEquals(2, ts.twentyFourHours); + } + + public void testOnePerSecond() { + long time = now; + long t; + long nextAssertCheck = randomLongBetween(1, HOUR); + long twentyFive = 25 * HOUR; + for (int i = 0; i < twentyFive; i++) { + t = time + i; + inc(t); + + if (i == nextAssertCheck) { + TimeSeries ts = timeSeries(t); + assertThat(five(t) - ts.fiveMinutes, fiveDelta); + assertThat(fifteen(t) - ts.fifteenMinutes, fifteenDelta); + assertThat(twentyFour(t) - ts.twentyFourHours, twentyFourDelta); + assertEquals(i + 1, tsc.count()); + + nextAssertCheck = Math.min(twentyFive, nextAssertCheck + randomLongBetween(HOUR, 3 * HOUR)); + } + } + } + + public void testCounterIncrementSameBucket() { + long resolution = 45; + long duration = 900; + counter.inc(now); + long count = randomLongBetween(resolution / 2, resolution * 2); + // this is the beginning of the current epoch + long start = (now / resolution) * resolution; + for (int i = 1; i < count; i++) { + counter.inc(start + randomLongBetween(0, resolution - 1)); + } + + assertEquals(count, counter.sum(start)); + assertEquals(count, counter.sum(now)); + + long t = 0; + + // Since we only incremented the first bucket, we should have access to that throughout duration + for (; t <= duration; t += resolution) { + assertEquals(count, counter.sum(start + t)); + } + + // Now we've gone past the end of the duration + assertEquals(0, counter.sum(start + t)); + assertEquals(0, counter.sum(start + duration + resolution)); + // The last second for which this counter is valid + assertEquals(count, counter.sum(start + duration + resolution - 1)); + } + + public void testFiveMinuteSameBucket() { + inc(now); + long resolution = tsc.fiveMinutes.resolution; + long duration = tsc.fiveMinutes.duration; + long count = randomLongBetween(1, resolution); + long start = (now / resolution) * resolution; + for (int i = 1; i < count; i++) { + inc(start + i); + } + assertEquals(count, tsc.count()); + assertEquals(count, timeSeries(now).fiveMinutes); + + long t = 0; + for (; t <= duration; t += resolution) { + assertEquals(count, timeSeries(start + t).fiveMinutes); + } + + TimeSeries series = timeSeries(start + t); + assertEquals(0, series.fiveMinutes); + assertEquals(count, series.fifteenMinutes); + assertEquals(count, series.twentyFourHours); + + series = timeSeries(start + duration + resolution); + assertEquals(0, series.fiveMinutes); + assertEquals(count, series.fifteenMinutes); + assertEquals(count, series.twentyFourHours); + assertEquals(count, timeSeries(start + duration + resolution - 1).fiveMinutes); + } + + public void testFifteenMinuteSameBucket() { + inc(now); + long resolution = tsc.fifteenMinutes.resolution; + long duration = tsc.fifteenMinutes.duration; + long start = (now / resolution) * resolution; + long count = randomLongBetween(1, resolution); + for (int i = 1; i < count; i++) { + inc(start + i); + } + assertEquals(count, tsc.count()); + assertEquals(count, timeSeries(now).fifteenMinutes); + + long t = 0; + for (; t <= duration; t += resolution) { + assertEquals(count, timeSeries(start + t).fifteenMinutes); + } + + TimeSeries series = timeSeries(start + t); + assertEquals(0, series.fiveMinutes); + assertEquals(0, series.fifteenMinutes); + assertEquals(count, series.twentyFourHours); + + series = timeSeries(start + duration + resolution); + assertEquals(0, series.fiveMinutes); + assertEquals(0, series.fifteenMinutes); + assertEquals(count, series.twentyFourHours); + assertEquals(count, timeSeries(start + duration + resolution - 1).fifteenMinutes); + } + + public void testTwentyFourHourSameBucket() { + inc(now); + long resolution = tsc.twentyFourHours.resolution; + long duration = tsc.twentyFourHours.duration; + long start = (now / resolution) * resolution; + long count = randomLongBetween(1, resolution); + for (int i = 1; i < count; i++) { + inc(start + i); + } + assertEquals(count, tsc.count()); + assertEquals(count, timeSeries(now).twentyFourHours); + + long t = 0; + for (; t <= duration; t += resolution) { + assertEquals(count, timeSeries(start + t).twentyFourHours); + } + + TimeSeries series = timeSeries(start + t); + assertEquals(0, series.fiveMinutes); + assertEquals(0, series.fifteenMinutes); + assertEquals(0, series.twentyFourHours); + + series = timeSeries(start + duration + resolution); + assertEquals(0, series.fiveMinutes); + assertEquals(0, series.fifteenMinutes); + assertEquals(0, series.twentyFourHours); + assertEquals(count, timeSeries(start + duration + resolution - 1).twentyFourHours); + } + + public void testCounterIncrementBucket() { + long count = customCounterDuration / customCounterResolution; + for (int i = 0; i < count; i++) { + counter.inc(now + i * customCounterResolution); + } + assertEquals(count, counter.sum(now + customCounterDuration)); + assertEquals(count - 1, counter.sum(now + customCounterDuration + customCounterResolution)); + assertEquals(count - 2, counter.sum(now + customCounterDuration + (2 * customCounterResolution))); + counter.inc(now + customCounterDuration); + assertEquals(count, counter.sum(now + customCounterDuration + customCounterResolution)); + } + + public void testFiveMinuteIncrementBucket() { + int count = tsc.fiveMinutes.buckets.length; + long resolution = tsc.fiveMinutes.resolution; + long duration = tsc.fiveMinutes.duration; + for (int i = 0; i < count; i++) { + inc(now + i * resolution); + } + long t = now + duration; + TimeSeries ts = timeSeries(t); + assertEquals(count, ts.fiveMinutes); + assertEquals(count, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + assertEquals(count, tsc.count()); + + t = now + duration + resolution; + ts = timeSeries(t); + assertEquals(count - 1, ts.fiveMinutes); + assertEquals(count, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + + long numRes = 2; + t = now + duration + (numRes * resolution); + ts = timeSeries(t); + assertEquals(count - numRes, ts.fiveMinutes); + assertEquals(count, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + + inc(now + duration); + ts = timeSeries(now + duration + resolution); + assertEquals(count, ts.fiveMinutes); + assertEquals(count + 1, ts.fifteenMinutes); + assertEquals(count + 1, ts.twentyFourHours); + assertEquals(count + 1, tsc.count()); + } + + public void testFifteenMinuteIncrementBucket() { + int count = tsc.fifteenMinutes.buckets.length; + long resolution = tsc.fifteenMinutes.resolution; + long duration = tsc.fifteenMinutes.duration; + for (int i = 0; i < count; i++) { + long t = now + i * resolution; + inc(t); + } + long t = now + duration; + TimeSeries ts = timeSeries(t); + assertEquals(five(t), ts.fiveMinutes); + assertEquals(count, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + + t = now + duration + resolution; + ts = timeSeries(t); + assertEquals(five(t), ts.fiveMinutes); + assertEquals(count - 1, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + + long numRes = 2; + t = now + duration + (numRes * resolution); + ts = timeSeries(t); + assertEquals(five(t), ts.fiveMinutes); + assertEquals(count - numRes, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + + inc(now + duration); + t = now + duration + resolution; + ts = timeSeries(t); + assertEquals(five(t), ts.fiveMinutes); + assertEquals(count, ts.fifteenMinutes); + assertEquals(count + 1, ts.twentyFourHours); + assertEquals(count + 1, tsc.count()); + } + + public void testTwentyFourHourIncrementBucket() { + int count = tsc.twentyFourHours.buckets.length; + long resolution = tsc.twentyFourHours.resolution; + long duration = tsc.twentyFourHours.duration; + for (int i = 0; i < count; i++) { + long t = now + i * resolution; + inc(t); + } + long t = now + duration; + TimeSeries ts = timeSeries(t); + assertEquals(five(t), ts.fiveMinutes); + assertEquals(fifteen(t), ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + + t = now + duration + resolution; + ts = timeSeries(t); + assertEquals(five(t), ts.fiveMinutes); + assertEquals(0, ts.fifteenMinutes); + assertEquals(count - 1, ts.twentyFourHours); + + long numRes = 2; + t = now + duration + (numRes * resolution); + ts = timeSeries(t); + assertEquals(0, ts.fiveMinutes); + assertEquals(0, ts.fifteenMinutes); + assertEquals(count - numRes, ts.twentyFourHours); + + inc(now + duration); + t = now + duration + resolution; + ts = timeSeries(t); + assertEquals(0, ts.fiveMinutes); + assertEquals(1, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + assertEquals(count + 1, tsc.count()); + } + + public void testCounterSkipBuckets() { + int count = (int) (customCounterDuration / customCounterResolution); + for (int skip = 1; skip <= count; skip++) { + reset(); + int increments = 0; + for (int i = 0; (i * skip * customCounterResolution) < customCounterDuration; i++) { + counter.inc(now + (i * skip * customCounterResolution)); + increments++; + } + assertEquals(increments, counter.sum(now + customCounterDuration)); + } + } + + public void testFiveMinuteSkipBucket() { + int count = tsc.fiveMinutes.buckets.length; + long resolution = tsc.fiveMinutes.resolution; + long duration = tsc.fiveMinutes.duration; + for (int skip = 1; skip <= count; skip++) { + tsc = new TimeSeriesCounter(timeProvider); + long increments = 0; + for (int i = 0; (i * skip * resolution) < duration; i++) { + inc(now + (i * skip * resolution)); + increments++; + } + + TimeSeries series = timeSeries(now + duration); + assertEquals(increments, series.fiveMinutes); + assertEquals(increments, series.fifteenMinutes); + assertEquals(increments, series.twentyFourHours); + assertEquals(increments, tsc.count()); + } + } + + public void testFifteenMinuteSkipBuckets() { + int count = tsc.fifteenMinutes.buckets.length; + long resolution = tsc.fifteenMinutes.resolution; + long duration = tsc.fifteenMinutes.duration; + for (int skip = 1; skip <= count; skip++) { + reset(); + for (int i = 0; (i * skip * resolution) < duration; i++) { + inc(now + (i * skip * resolution)); + } + TimeSeries ts = timeSeries(now + duration); + assertEquals(five(now + duration), ts.fiveMinutes); + assertEquals(events.size(), ts.fifteenMinutes); + assertEquals(events.size(), ts.twentyFourHours); + assertEquals(events.size(), tsc.count()); + } + } + + public void testTwentyFourHourSkipBuckets() { + int count = tsc.twentyFourHours.buckets.length; + long resolution = tsc.twentyFourHours.resolution; + long duration = tsc.twentyFourHours.duration; + for (int skip = 1; skip <= count; skip++) { + reset(); + for (int i = 0; (i * skip * resolution) < duration; i++) { + inc(now + (i * skip * resolution)); + } + TimeSeries ts = timeSeries(now + duration); + assertEquals(five(now + duration), ts.fiveMinutes); + assertEquals(events.size(), ts.twentyFourHours); + assertEquals(events.size(), tsc.count()); + } + } + + public void testCounterReset() { + long time = now; + for (int i = 0; i < 20; i++) { + long count = 0; + long withinBucket = randomIntBetween(1, (int) (customCounterResolution / 2)); + time += customCounterResolution + (i * customCounterDuration); + long last = time; + for (int j = 0; j < withinBucket; j++) { + long bucketTime = (time / customCounterResolution) * customCounterResolution; + last = bucketTime + randomLongBetween(0, customCounterResolution - 1); + counter.inc(last); + count++; + } + assertEquals(count, counter.sum(last)); + } + } + + public void testFiveMinuteReset() { + long time = now; + long resolution = tsc.fiveMinutes.resolution; + long duration = tsc.fiveMinutes.duration; + for (int i = 0; i < 20; i++) { + long withinBucket = randomLongBetween(1, resolution); + time += resolution + (i * duration); + for (int j = 0; j < withinBucket; j++) { + inc(time + j); + } + TimeSeries ts = timeSeries(time); + assertThat(five(time) - ts.fiveMinutes, fiveDelta); + assertThat(fifteen(time) - ts.fifteenMinutes, fifteenDelta); + assertThat(twentyFour(time) - ts.twentyFourHours, twentyFourDelta); + assertEquals(events.size(), tsc.count()); + } + } + + public void testFifteenMinuteReset() { + long time = now; + long resolution = tsc.fifteenMinutes.resolution; + long duration = tsc.fifteenMinutes.duration; + for (int i = 0; i < 20; i++) { + long withinBucket = randomLongBetween(1, resolution); + time += resolution + (i * duration); + for (int j = 0; j < withinBucket; j++) { + inc(time + j); + } + TimeSeries ts = timeSeries(time); + assertThat(five(time) - ts.fiveMinutes, fiveDelta); + assertThat(fifteen(time) - ts.fifteenMinutes, fifteenDelta); + assertThat(twentyFour(time) - ts.twentyFourHours, twentyFourDelta); + assertEquals(events.size(), tsc.count()); + } + } + + public void testTwentyFourHourReset() { + long time = now; + long resolution = tsc.twentyFourHours.resolution; + long duration = tsc.twentyFourHours.duration; + for (int i = 0; i < 20; i++) { + long withinBucket = randomLongBetween(1, resolution); + time += resolution + (i * duration); + for (int j = 0; j < withinBucket; j++) { + inc(time + j); + } + TimeSeries ts = timeSeries(time); + assertThat(twentyFour(time) - ts.twentyFourHours, twentyFourDelta); + assertEquals(events.size(), tsc.count()); + } + } + + // Count the last five minutes of events before t + public long five(long t) { + return countLast(t, tsc.fiveMinutes, events); + } + + // Count the last fifteen minutes of events before t + public long fifteen(long t) { + return countLast(t, tsc.fifteenMinutes, events); + } + + // Count the last twenty-four hours of events before t + public long twentyFour(long t) { + return countLast(t, tsc.twentyFourHours, events); + } + + // Count the last set of events that would be recorded by counter + public long countLast(long t, Counter counter, List events) { + long count = 0; + long after = ((t - counter.duration) / counter.resolution) * counter.resolution; + for (long event : events) { + if (event > after) { + count++; + } + } + return count; + } + + protected void inc(long t) { + timeProvider.inc(t); + } + + protected TimeSeries timeSeries(long t) { + return timeProvider.timeSeries(t); + } + + class TimeProvider implements LongSupplier { + public int i = 0; + public boolean useTimeSeries = false; + public long timeSeriesT = 0; + + public void inc(long t) { + int last = i; + events.add(t); + tsc.inc(); + assert i == last + 1; + } + + public TimeSeries timeSeries(long t) { + int last = i; + useTimeSeries = true; + timeSeriesT = t; + TimeSeries ts = tsc.timeSeries(); + assert i == last; + return ts; + } + + @Override + public long getAsLong() { + if (useTimeSeries) { + useTimeSeries = false; + return timeSeriesT * 1000; + } + long event = events.get(i) * 1000; + i++; + return event; + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java index 3d38b40cb0945..b96c5e8f0d355 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java @@ -371,7 +371,7 @@ protected ScriptService getMockScriptService() { }); final MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, deterministicScripts, emptyMap(), emptyMap()); final Map engines = singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } private static List threshold(String fieldName, long threshold, Map vars) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index f4935805cd5c8..271faf4776f0f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -131,7 +131,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testNoDocs() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index b5bb03405d957..0eb3aba58283c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -169,7 +169,7 @@ protected ScriptService getMockScriptService() { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } protected A createAggregator(AggregationBuilder aggregationBuilder, AggregationContext context) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java index 754c14b9ca87b..55a344dd8c1e8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java @@ -122,7 +122,7 @@ protected ScriptService getMockScriptService() { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testNoDocs() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java index ec91f60b63d0c..1e5cbf5a95191 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java @@ -142,7 +142,7 @@ protected ScriptService mockScriptService() { Collections.emptyMap() ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java index 5c6d8606f6e80..52652893ecc1d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java @@ -148,7 +148,7 @@ protected ScriptService getMockScriptService() { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java index bbd209ea06b8b..22275cfc4876a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java @@ -325,6 +325,6 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java index 49824e84f130e..152e278b2c081 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java @@ -147,7 +147,7 @@ protected ScriptService getMockScriptService() { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testNoMatchingField() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java index c92ce332dde7a..efc5946620716 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java @@ -276,7 +276,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } @SuppressWarnings("unchecked") @@ -427,7 +427,7 @@ public void testScriptParamsPassedThrough() throws IOException { public void testAggParamsPassedToReduceScript() throws IOException { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java index cce5f5cfb6afb..05c1667a5cc37 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java @@ -448,6 +448,6 @@ protected ScriptService getMockScriptService() { ); final MockScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, emptyMap()); final Map engines = singletonMap(engine.getType(), engine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java index 58074ca1efe33..8cd8358cae043 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java @@ -406,7 +406,7 @@ protected ScriptService getMockScriptService() { ); final MockScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, emptyMap()); final Map engines = singletonMap(engine.getType(), engine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } private static MappedFieldType defaultFieldType() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java index 068d65dd240a3..63e8b0ce7e651 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java @@ -96,7 +96,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testGeoField() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptAggregatorTests.java index d7a98379547af..61d3014251212 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptAggregatorTests.java @@ -57,7 +57,7 @@ protected ScriptService getMockScriptService() { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testScript() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java index 541ef1cf36000..7dae864ee3340 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java @@ -77,7 +77,7 @@ protected ScriptService getMockScriptService() { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testMatchAllDocs() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 4718d56e15a06..09a36db0fa8b8 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -76,7 +76,12 @@ public static void init() { Settings baseSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); Map, Object>> scripts = Collections.singletonMap(MOCK_SCRIPT_NAME, p -> null); ScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); - scriptService = new ScriptService(baseSettings, Collections.singletonMap(engine.getType(), engine), ScriptModule.CORE_CONTEXTS); + scriptService = new ScriptService( + baseSettings, + Collections.singletonMap(engine.getType(), engine), + ScriptModule.CORE_CONTEXTS, + () -> 1L + ); SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables()); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index b65e145e4610f..dd7a1498e43e7 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1720,7 +1720,7 @@ protected void assertSnapshotOrGenericThread() { ); nodeEnv = new NodeEnvironment(settings, environment); final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()); - final ScriptService scriptService = new ScriptService(settings, emptyMap(), emptyMap()); + final ScriptService scriptService = new ScriptService(settings, emptyMap(), emptyMap(), () -> 1L); client = new NodeClient(settings, threadPool); final SetOnce rerouteServiceSetOnce = new SetOnce<>(); final SnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java index e40ba48d9d38e..25a44f9761b19 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java @@ -32,7 +32,7 @@ public static ScriptService instance(boolean compilationException) { } private TestTemplateService(boolean compilationException) { - super(Settings.EMPTY, Collections.singletonMap(DEFAULT_TEMPLATE_LANG, new MockScriptEngine()), Collections.emptyMap()); + super(Settings.EMPTY, Collections.singletonMap(DEFAULT_TEMPLATE_LANG, new MockScriptEngine()), Collections.emptyMap(), () -> 1L); this.compilationException = compilationException; } diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java index 1b6807335ec94..fa041d51005f2 100644 --- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java +++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java @@ -49,6 +49,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; +import java.util.function.LongSupplier; /** * A node for testing which allows: @@ -157,9 +158,14 @@ protected SearchService newSearchService( } @Override - protected ScriptService newScriptService(Settings settings, Map engines, Map> contexts) { + protected ScriptService newScriptService( + Settings settings, + Map engines, + Map> contexts, + LongSupplier timeProvider + ) { if (getPluginsService().filterPlugins(MockScriptService.TestPlugin.class).isEmpty()) { - return super.newScriptService(settings, engines, contexts); + return super.newScriptService(settings, engines, contexts, timeProvider); } return new MockScriptService(settings, engines, contexts); } diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptService.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptService.java index 7e017ff5e8dba..5f46af5fcf86a 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptService.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptService.java @@ -23,7 +23,7 @@ public class MockScriptService extends ScriptService { public static class TestPlugin extends Plugin {} public MockScriptService(Settings settings, Map engines, Map> contexts) { - super(settings, engines, contexts); + super(settings, engines, contexts, () -> 1L); } @Override diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java index bed0939510c6d..0ad234c425891 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java @@ -79,7 +79,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testNoMatchingField() throws IOException { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregatorTests.java index 926878d167ffb..d14bccd0f6058 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregatorTests.java @@ -112,7 +112,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testIntegersFloatsAndStrings() throws IOException { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java index fe9480f9aa301..6d234cee87899 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java @@ -108,7 +108,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testNoMatchingField() throws IOException { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java index 5b3509e6e59ef..21e75fbbd1dc2 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java @@ -423,6 +423,6 @@ protected ScriptService getMockScriptService() { ); final MockScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, emptyMap()); final Map engines = singletonMap(engine.getType(), engine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java index ac30d0e079d63..042f85b8abd9c 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java @@ -585,7 +585,7 @@ protected ScriptService getMockScriptService() { return field.getValue(); }), emptyMap()); Map engines = singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } @Override diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java index 70f59362c4081..3ef0f4b64a3b4 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java @@ -138,7 +138,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testNoMatchingField() throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java index 5c6fa57269de3..2d89a3612f615 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java @@ -94,7 +94,8 @@ public void testEvaluateRoles() throws Exception { final ScriptService scriptService = new ScriptService( Settings.EMPTY, Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), - ScriptModule.CORE_CONTEXTS + ScriptModule.CORE_CONTEXTS, + () -> 1L ); final ExpressionModel model = new ExpressionModel(); model.defineField("username", "hulk"); @@ -150,7 +151,8 @@ public void testValidate() { final ScriptService scriptService = new ScriptService( Settings.EMPTY, Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), - ScriptModule.CORE_CONTEXTS + ScriptModule.CORE_CONTEXTS, + () -> 1L ); final TemplateRoleName plainString = new TemplateRoleName(new BytesArray("{ \"source\":\"heroes\" }"), Format.STRING); @@ -176,7 +178,8 @@ public void testValidateWillPassWithEmptyContext() { final ScriptService scriptService = new ScriptService( Settings.EMPTY, Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), - ScriptModule.CORE_CONTEXTS + ScriptModule.CORE_CONTEXTS, + () -> 1L ); final BytesReference template = new BytesArray( @@ -206,7 +209,8 @@ public void testValidateWillFailForSyntaxError() { final ScriptService scriptService = new ScriptService( Settings.EMPTY, Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), - ScriptModule.CORE_CONTEXTS + ScriptModule.CORE_CONTEXTS, + () -> 1L ); final BytesReference template = new BytesArray("{ \"source\":\" {{#not-closed}} {{other-variable}} \" }"); @@ -239,7 +243,8 @@ public void testValidateWillCompileButNotExecutePainlessScript() { final ScriptService scriptService = new ScriptService( Settings.EMPTY, Map.of("painless", scriptEngine), - ScriptModule.CORE_CONTEXTS + ScriptModule.CORE_CONTEXTS, + () -> 1L ) { @Override protected StoredScriptSource getScriptFromClusterState(String id) { @@ -268,7 +273,8 @@ public void testValidationWillFailWhenInlineScriptIsNotEnabled() { final ScriptService scriptService = new ScriptService( settings, Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), - ScriptModule.CORE_CONTEXTS + ScriptModule.CORE_CONTEXTS, + () -> 1L ); final BytesReference inlineScript = new BytesArray("{ \"source\":\"\" }"); final IllegalArgumentException e = expectThrows( @@ -283,7 +289,8 @@ public void testValidateWillFailWhenStoredScriptIsNotEnabled() { final ScriptService scriptService = new ScriptService( settings, Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), - ScriptModule.CORE_CONTEXTS + ScriptModule.CORE_CONTEXTS, + () -> 1L ); final ClusterChangedEvent clusterChangedEvent = mock(ClusterChangedEvent.class); final ClusterState clusterState = mock(ClusterState.class); @@ -310,7 +317,8 @@ public void testValidateWillFailWhenStoredScriptIsNotFound() { final ScriptService scriptService = new ScriptService( Settings.EMPTY, Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), - ScriptModule.CORE_CONTEXTS + ScriptModule.CORE_CONTEXTS, + () -> 1L ); final ClusterChangedEvent clusterChangedEvent = mock(ClusterChangedEvent.class); final ClusterState clusterState = mock(ClusterState.class); diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java index b2b2008098d41..e251a2af22ceb 100644 --- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java +++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java @@ -95,7 +95,8 @@ public void setUpResolver() { final ScriptService scriptService = new ScriptService( settings, Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), - ScriptModule.CORE_CONTEXTS + ScriptModule.CORE_CONTEXTS, + () -> 1L ); final ServiceProviderDefaults samlDefaults = new ServiceProviderDefaults("elastic-cloud", NameID.TRANSIENT, Duration.ofMinutes(15)); resolver = new WildcardServiceProviderResolver(settings, scriptService, new SamlServiceProviderFactory(samlDefaults)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java index 487f044772caf..552d9756729f1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java @@ -432,7 +432,8 @@ public void testRealmWithTemplatedRoleMapping() throws Exception { final ScriptService scriptService = new ScriptService( settings, Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), - ScriptModule.CORE_CONTEXTS + ScriptModule.CORE_CONTEXTS, + () -> 1L ); NativeRoleMappingStore roleMapper = new NativeRoleMappingStore(settings, mockClient, mockSecurityIndex, scriptService) { @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java index bcebc32d50173..7647bc4dae9a8 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java @@ -473,7 +473,8 @@ public void testLdapRealmWithTemplatedRoleMapping() throws Exception { final ScriptService scriptService = new ScriptService( defaultGlobalSettings, Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), - ScriptModule.CORE_CONTEXTS + ScriptModule.CORE_CONTEXTS, + () -> 1L ); NativeRoleMappingStore roleMapper = new NativeRoleMappingStore( defaultGlobalSettings, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index dea8e63ad0c4e..1f00be6e15a5d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -122,7 +122,8 @@ public void testResolveRoles() throws Exception { ScriptService scriptService = new ScriptService( Settings.EMPTY, Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), - ScriptModule.CORE_CONTEXTS + ScriptModule.CORE_CONTEXTS, + () -> 1L ); when(securityIndex.isAvailable()).thenReturn(true); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java index 0287b5af5d636..4f1a3e861dd9c 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java @@ -44,7 +44,7 @@ public void init() throws Exception { Watcher.SCRIPT_TEMPLATE_CONTEXT.name, Watcher.SCRIPT_TEMPLATE_CONTEXT ); - ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, contexts); + ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, contexts, () -> 1L); textTemplateEngine = new TextTemplateEngine(scriptService); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherMockScriptPlugin.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherMockScriptPlugin.java index 01ea342b20b5e..f69b28a40ff5d 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherMockScriptPlugin.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherMockScriptPlugin.java @@ -74,6 +74,6 @@ public static ScriptService newMockScriptService(Map engines = new HashMap<>(); engines.put(MockScriptEngine.NAME, new MockScriptEngine(MockScriptEngine.NAME, scripts, CONTEXT_COMPILERS)); Map> contexts = CONTEXTS.stream().collect(Collectors.toMap(o -> o.name, Function.identity())); - return new ScriptService(Settings.EMPTY, engines, contexts); + return new ScriptService(Settings.EMPTY, engines, contexts, () -> 1L); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java index c450ff7ed325c..24951bd971144 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java @@ -78,7 +78,7 @@ public void setup() { Map> contexts = new HashMap<>(); contexts.put(Watcher.SCRIPT_TEMPLATE_CONTEXT.name, Watcher.SCRIPT_TEMPLATE_CONTEXT); contexts.put(WatcherTransformScript.CONTEXT.name, WatcherTransformScript.CONTEXT); - scriptService = new ScriptService(Settings.EMPTY, engines, contexts); + scriptService = new ScriptService(Settings.EMPTY, engines, contexts, () -> 1L); ThreadPool threadPool = mock(ThreadPool.class); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java index 859732e61a0d4..140f44130fd70 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java @@ -68,7 +68,7 @@ public void testParser() throws Exception { final MockScriptEngine engine = new MockScriptEngine("mock", Collections.emptyMap(), Collections.emptyMap()); Map engines = Collections.singletonMap(engine.getType(), engine); - ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); Client client = mock(Client.class); SearchTransformFactory transformFactory = new SearchTransformFactory(Settings.EMPTY, client, xContentRegistry(), scriptService); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/script/ScriptTransformTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/script/ScriptTransformTests.java index dd6889103d2be..77dfbc460937f 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/script/ScriptTransformTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/script/ScriptTransformTests.java @@ -208,6 +208,6 @@ public static ScriptService createScriptService() throws Exception { Map> contexts = new HashMap<>(ScriptModule.CORE_CONTEXTS); contexts.put(WatcherTransformScript.CONTEXT.name, WatcherTransformScript.CONTEXT); contexts.put(Watcher.SCRIPT_TEMPLATE_CONTEXT.name, Watcher.SCRIPT_TEMPLATE_CONTEXT); - return new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap()); + return new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L); } }