Script: Restore the scripting general cache

Restore the scripting general cache in preparation for deprecation of
the context cache, then removal of the context cache in master.

Brings back the general cache settings:
* `script.max_compilations_rate`
* `script.cache.expire`
* `script.cache.max_size`

`script.cache.max_size` and `script.cache.expire` are now dynamic settings.

The context cache is still default, the switch to general cache as default will happen on context cache deprecation.

System script contexts can now opt-out of compilation rate limiting using a flag rather than a sentinel rate limit value.  Duplicated defaults for system contexts have been coalesced.

Other than that, this code is the same as what was in 7.x to make a `master`-first commit and backport strategy easy.

Refs: elastic#62899
Backport: elastic#79414

server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java

@@ -19,8 +19,6 @@
 import java.util.Map;
 import java.util.function.Function;
 
-import static org.elasticsearch.core.TimeValue.timeValueMillis;
-
 /**
  * Abstract base for scripts to execute to build scripted fields. Inspired by
  * {@link AggregationScript} but hopefully with less historical baggage.
@@ -32,33 +30,21 @@ public abstract class AbstractFieldScript extends DocBasedScript {
     public static final int MAX_VALUES = 100;
 
     static <F> ScriptContext<F> newContext(String name, Class<F> factoryClass) {
-        return new ScriptContext<>(
-            name,
-            factoryClass,
-            /*
-             * We rely on the script cache in two ways:
-             * 1. It caches the "heavy" part of mappings generated at runtime.
-             * 2. Mapping updates tend to try to compile the script twice. Not
-             *    for any good reason. They just do.
-             * Thus we use the default 100.
-             */
-            100,
-            timeValueMillis(0),
-            /*
-             * Disable compilation rate limits for runtime fields so we
-             * don't prevent mapping updates because we've performed too
-             * many recently. That'd just be lame. We also compile these
-             * scripts during search requests so this could totally be a
-             * source of runaway script compilations. We think folks will
-             * mostly reuse scripts though.
-             */
-            ScriptCache.UNLIMITED_COMPILATION_RATE.asTuple(),
-            /*
-             * Disable runtime fields scripts from being allowed
-             * to be stored as part of the script meta data.
-             */
-            false
-        );
+        /*
+         * We rely on the script cache in two ways:
+         * 1. It caches the "heavy" part of mappings generated at runtime.
+         * 2. Mapping updates tend to try to compile the script twice. Not
+         *    for any good reason. They just do.
+         * Thus we use the default cache size.
+         *
+         * We also disable compilation rate limits for runtime fields so we
+         * don't prevent mapping updates because we've performed too
+         * many recently. That'd just be lame. We also compile these
+         * scripts during search requests so this could totally be a
+         * source of runaway script compilations. We think folks will
+         * mostly reuse scripts though.
+         */
+        return new ScriptContext<>(name, factoryClass, false);
     }
 
     private static final Map<String, Function<Object, Object>> PARAMS_FUNCTIONS = org.elasticsearch.core.Map.of(

server/src/main/java/org/elasticsearch/script/IngestConditionalScript.java

@@ -8,8 +8,6 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.core.TimeValue;
-
 import java.util.Map;
 
 /**
@@ -20,8 +18,7 @@ public abstract class IngestConditionalScript {
     public static final String[] PARAMETERS = { "ctx" };
 
     /** The context used to compile {@link IngestConditionalScript} factories. */
-    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("processor_conditional", Factory.class,
-        200, TimeValue.timeValueMillis(0), ScriptCache.UNLIMITED_COMPILATION_RATE.asTuple(), true);
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("processor_conditional", Factory.class, true);
 
     /** The generic runtime parameters for the script. */
     private final Map<String, Object> params;

server/src/main/java/org/elasticsearch/script/IngestScript.java

@@ -9,8 +9,6 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.core.TimeValue;
-
 import java.util.Map;
 
 /**
@@ -21,8 +19,7 @@ public abstract class IngestScript {
     public static final String[] PARAMETERS = { "ctx" };
 
     /** The context used to compile {@link IngestScript} factories. */
-    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("ingest", Factory.class,
-        200, TimeValue.timeValueMillis(0), ScriptCache.UNLIMITED_COMPILATION_RATE.asTuple(), true);
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("ingest", Factory.class, true);
 
     /** The generic runtime parameters for the script. */
     private final Map<String, Object> params;

server/src/main/java/org/elasticsearch/script/ScriptCache.java

@@ -44,12 +44,7 @@ public class ScriptCache {
     private final double compilesAllowedPerNano;
     private final String contextRateSetting;
 
-    ScriptCache(
-            int cacheMaxSize,
-            TimeValue cacheExpire,
-            CompilationRate maxCompilationRate,
-            String contextRateSetting
-    ) {
+    ScriptCache(int cacheMaxSize, TimeValue cacheExpire, CompilationRate maxCompilationRate, String contextRateSetting) {
         this.cacheSize = cacheMaxSize;
         this.cacheExpire = cacheExpire;
         this.contextRateSetting = contextRateSetting;
@@ -94,8 +89,10 @@ <FactoryType> FactoryType compile(
                     logger.trace("context [{}]: compiling script, type: [{}], lang: [{}], options: [{}]", context.name, type,
                         lang, options);
                 }
-                // Check whether too many compilations have happened
-                checkCompilationLimit();
+                if (context.compilationRateLimited) {
+                    // Check whether too many compilations have happened
+                    checkCompilationLimit();
+                }
                 Object compiledScript = scriptEngine.compile(id, idOrCode, context, options);
                 // Since the cache key is the script content itself we don't need to
                 // invalidate/check the cache if an indexed script changes.

server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java

@@ -21,7 +21,7 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
-// This class is deprecated in favor of ScriptStats and ScriptContextStats.  It is removed in 8.
+// This class is deprecated in favor of ScriptStats and ScriptContextStats
 public class ScriptCacheStats implements Writeable, ToXContentFragment {
     private final Map<String, ScriptStats> context;
     private final ScriptStats general;

server/src/main/java/org/elasticsearch/script/ScriptContext.java

@@ -47,6 +47,8 @@
  * be {@code boolean needs_score()}.
  */
 public final class ScriptContext<FactoryType> {
+    /** The default compilation rate limit for contexts with compilation rate limiting enabled */
+    public static final Tuple<Integer, TimeValue> DEFAULT_COMPILATION_RATE_LIMIT = new Tuple<>(150, TimeValue.timeValueMinutes(5));
 
     /** A unique identifier for this context. */
     public final String name;
@@ -66,15 +68,15 @@ public final class ScriptContext<FactoryType> {
     /** The default expiration of a script in the cache for the context, if not overridden */
     public final TimeValue cacheExpireDefault;
 
-    /** The default max compilation rate for scripts in this context.  Script compilation is throttled if this is exceeded */
-    public final Tuple<Integer, TimeValue> maxCompilationRateDefault;
+    /** Is compilation rate limiting enabled for this context? */
+    public final boolean compilationRateLimited;
 
     /** Determines if the script can be stored as part of the cluster state. */
     public final boolean allowStoredScript;
 
     /** Construct a context with the related instance and compiled classes with caller provided cache defaults */
-    public ScriptContext(String name, Class<FactoryType> factoryClazz, int cacheSizeDefault, TimeValue cacheExpireDefault,
-                        Tuple<Integer, TimeValue> maxCompilationRateDefault, boolean allowStoredScript) {
+    ScriptContext(String name, Class<FactoryType> factoryClazz, int cacheSizeDefault, TimeValue cacheExpireDefault,
+                  boolean compilationRateLimited, boolean allowStoredScript) {
         this.name = name;
         this.factoryClazz = factoryClazz;
         Method newInstanceMethod = findMethod("FactoryType", factoryClazz, "newInstance");
@@ -98,15 +100,25 @@ public ScriptContext(String name, Class<FactoryType> factoryClazz, int cacheSize
 
         this.cacheSizeDefault = cacheSizeDefault;
         this.cacheExpireDefault = cacheExpireDefault;
-        this.maxCompilationRateDefault = maxCompilationRateDefault;
+        this.compilationRateLimited = compilationRateLimited;
         this.allowStoredScript = allowStoredScript;
     }
 
     /** Construct a context with the related instance and compiled classes with defaults for cacheSizeDefault, cacheExpireDefault and
      *  maxCompilationRateDefault and allow scripts of this context to be stored scripts */
     public ScriptContext(String name, Class<FactoryType> factoryClazz) {
         // cache size default, cache expire default, max compilation rate are defaults from ScriptService.
-        this(name, factoryClazz, 100, TimeValue.timeValueMillis(0), new Tuple<>(75, TimeValue.timeValueMinutes(5)), true);
+        this(name, factoryClazz, 100, TimeValue.timeValueMillis(0), true, true);
+    }
+
+    /**
+     * Construct a context for a system script usage, using the:
+     *  - default cache size (only relevant when context caching enabled): 200
+     *  - default cache expiration: no expiration
+     *  - unlimited compilation rate
+     */
+    public ScriptContext(String name, Class<FactoryType> factoryClazz, boolean allowStoredScript) {
+        this(name, factoryClazz, 200, TimeValue.timeValueMillis(0), false, allowStoredScript);
     }
 
     /** Returns a method with the given name, or throws an exception if multiple are found. */

server/src/main/java/org/elasticsearch/script/ScriptService.java

@@ -231,7 +231,7 @@ void registerClusterSettingsListeners(ClusterSettings clusterSettings) {
 
         // Handle all settings for context and general caches, this flips between general and context caches.
         clusterSettings.addSettingsUpdateConsumer(
-            (settings) -> setCacheHolder(settings),
+            this::setCacheHolder,
             Arrays.asList(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING,
                           SCRIPT_GENERAL_CACHE_EXPIRE_SETTING,
                           SCRIPT_GENERAL_CACHE_SIZE_SETTING,
@@ -565,8 +565,9 @@ void setCacheHolder(Settings settings) {
         } else if (current.general == null) {
             // Flipping to general
             cacheHolder.set(generalCacheHolder(settings));
-        } else if (current.general.rate.equals(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings)) == false) {
-            // General compilation rate changed, that setting is the only dynamically updated general setting
+        } else if (current.general.rate.equals(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings)) == false ||
+                   current.general.cacheExpire.equals(SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(settings)) == false ||
+                   current.general.cacheSize != SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(settings)) {
             cacheHolder.set(generalCacheHolder(settings));
         }
     }
@@ -592,13 +593,15 @@ ScriptCache contextCache(Settings settings, ScriptContext<?> context) {
 
         Setting<ScriptCache.CompilationRate> rateSetting =
             SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context.name);
-        ScriptCache.CompilationRate rate = null;
-        if (SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.get(settings) || compilationLimitsEnabled() == false) {
+        ScriptCache.CompilationRate rate;
+        if (SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.get(settings)
+                || compilationLimitsEnabled() == false
+                || context.compilationRateLimited == false) {
             rate = SCRIPT_COMPILATION_RATE_ZERO;
         } else if (rateSetting.existsOrFallbackExists(settings)) {
             rate = rateSetting.get(settings);
         } else {
-            rate = new ScriptCache.CompilationRate(context.maxCompilationRateDefault);
+            rate = new ScriptCache.CompilationRate(ScriptContext.DEFAULT_COMPILATION_RATE_LIMIT);
         }
 
         return new ScriptCache(cacheSize, cacheExpire, rate, rateSetting.getKey());

server/src/main/java/org/elasticsearch/script/ScriptStats.java

@@ -109,6 +109,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(Fields.COMPILATIONS, compilations);
         builder.field(Fields.CACHE_EVICTIONS, cacheEvictions);
         builder.field(Fields.COMPILATION_LIMIT_TRIGGERED, compilationLimitTriggered);
+        /* TODO(stu): master only
+        builder.startArray(Fields.CONTEXTS);
+        for (ScriptContextStats contextStats: contextStats) {
+            contextStats.toXContent(builder, params);
+        }
+        builder.endArray();
+         */
         builder.endObject();
         return builder;
     }

server/src/main/java/org/elasticsearch/script/TemplateScript.java

@@ -8,8 +8,6 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.core.TimeValue;
-
 import java.util.Map;
 
 /**
@@ -41,6 +39,5 @@ public interface Factory {
     // Remove compilation rate limit for ingest.  Ingest pipelines may use many mustache templates, triggering compilation
     // rate limiting.  MustacheScriptEngine explicitly checks for TemplateScript.  Rather than complicating the implementation there by
     // creating a new Script class (as would be customary), this context is used to avoid the default rate limit.
-    public static final ScriptContext<Factory> INGEST_CONTEXT = new ScriptContext<>("ingest_template", Factory.class,
-            200, TimeValue.timeValueMillis(0), ScriptCache.UNLIMITED_COMPILATION_RATE.asTuple(), true);
+    public static final ScriptContext<Factory> INGEST_CONTEXT = new ScriptContext<>("ingest_template", Factory.class, true);
 }

server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java

@@ -8,14 +8,54 @@
 package org.elasticsearch.script;
 
 import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.stream.Collectors;
+
 public class ScriptCacheTests extends ESTestCase {
     // even though circuit breaking is allowed to be configured per minute, we actually weigh this over five minutes
     // simply by multiplying by five, so even setting it to one, requires five compilations to break
     public void testCompilationCircuitBreaking() throws Exception {
+        String context = randomFrom(
+            ScriptModule.CORE_CONTEXTS.values().stream().filter(
+                c -> c.compilationRateLimited
+            ).collect(Collectors.toList())
+        ).name;
+        final TimeValue expire = ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY);
+        final Integer size = ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY);
+        Setting<ScriptCache.CompilationRate> rateSetting =
+            ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context);
+        ScriptCache.CompilationRate rate =
+            ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY);
+        String rateSettingName = rateSetting.getKey();
+        ScriptCache cache = new ScriptCache(size, expire,
+            new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), rateSettingName);
+        cache.checkCompilationLimit(); // should pass
+        expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
+        cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), rateSettingName);
+        cache.checkCompilationLimit(); // should pass
+        cache.checkCompilationLimit(); // should pass
+        expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
+        int count = randomIntBetween(5, 50);
+        cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), rateSettingName);
+        for (int i = 0; i < count; i++) {
+            cache.checkCompilationLimit(); // should pass
+        }
+        expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
+        cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), rateSettingName);
+        expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
+        cache = new ScriptCache(size, expire,
+                                new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), rateSettingName);
+        int largeLimit = randomIntBetween(1000, 10000);
+        for (int i = 0; i < largeLimit; i++) {
+            cache.checkCompilationLimit();
+        }
+    }
+
+    public void testGeneralCompilationCircuitBreaking() throws Exception {
         final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY);
         final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY);
         String settingName = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey();
@@ -35,14 +75,33 @@ public void testCompilationCircuitBreaking() throws Exception {
         cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), settingName);
         expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
         cache = new ScriptCache(size, expire,
-                                new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), settingName);
+            new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), settingName);
         int largeLimit = randomIntBetween(1000, 10000);
         for (int i = 0; i < largeLimit; i++) {
             cache.checkCompilationLimit();
         }
     }
 
     public void testUnlimitedCompilationRate() {
+        String context = randomFrom(
+            ScriptModule.CORE_CONTEXTS.values().stream().filter(
+                c -> c.compilationRateLimited
+            ).collect(Collectors.toList())
+        ).name;
+        final Integer size = ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY);
+        final TimeValue expire = ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY);
+        String settingName = ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context).getKey();
+        ScriptCache cache = new ScriptCache(size, expire, ScriptCache.UNLIMITED_COMPILATION_RATE, settingName);
+        ScriptCache.TokenBucketState initialState = cache.tokenBucketState.get();
+        for(int i=0; i < 3000; i++) {
+            cache.checkCompilationLimit();
+            ScriptCache.TokenBucketState currentState = cache.tokenBucketState.get();
+            assertEquals(initialState.lastInlineCompileTime, currentState.lastInlineCompileTime);
+            assertEquals(initialState.availableTokens, currentState.availableTokens, 0.0); // delta of 0.0 because it should never change
+        }
+    }
+
+    public void testGeneralUnlimitedCompilationRate() {
         final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY);
         final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY);
         String settingName = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey();