Move painless base whitelist into painless jar (#76262) (#76271)

The base whitelist is shared by all script contexts. However, the text
files exist within painless itself, not the limited spi jar which is
meant for extension. This commit moves the base definition into painless
and reworks context loading in painless to add the base whitelist so
that all contexts get it, regardless of whether they have added an
explicit whitelist.

Co-authored-by: Ryan Ernst <[email protected]>

modules/lang-painless/spi/build.gradle

@@ -14,7 +14,5 @@ archivesBaseName = 'elasticsearch-scripting-painless-spi'
 
 dependencies {
   api project(":server")
+  testImplementation project(":test:framework")
 }
-
-// no tests...yet?
-tasks.named("test").configure { enabled = false }

modules/lang-painless/src/test/java/org/elasticsearch/painless/AnnotationTestObject.java → modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/AnnotationTestObject.java

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.painless;
+package org.elasticsearch.painless.spi;
 
 import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser;
 

modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java

@@ -8,8 +8,6 @@
 
 package org.elasticsearch.painless.spi;
 
-import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser;
-
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
@@ -26,28 +24,6 @@
  */
 public final class Whitelist {
 
-    private static final String[] BASE_WHITELIST_FILES = new String[] {
-        "org.elasticsearch.txt",
-        "org.elasticsearch.net.txt",
-        "org.elasticsearch.script.fields.txt",
-        "java.lang.txt",
-        "java.math.txt",
-        "java.text.txt",
-        "java.time.txt",
-        "java.time.chrono.txt",
-        "java.time.format.txt",
-        "java.time.temporal.txt",
-        "java.time.zone.txt",
-        "java.util.txt",
-        "java.util.function.txt",
-        "java.util.regex.txt",
-        "java.util.stream.txt"
-    };
-
-    public static final List<Whitelist> BASE_WHITELISTS =
-            Collections.singletonList(WhitelistLoader.loadFromResourceFiles(
-                    Whitelist.class, WhitelistAnnotationParser.BASE_ANNOTATION_PARSERS, BASE_WHITELIST_FILES));
-
     /** The {@link ClassLoader} used to look up the whitelisted Java classes, constructors, methods, and fields. */
     public final ClassLoader classLoader;
 

modules/lang-painless/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java → modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java

@@ -8,18 +8,20 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.painless.spi.AnnotationTestObject;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.painless.spi.WhitelistClass;
 import org.elasticsearch.painless.spi.WhitelistLoader;
 import org.elasticsearch.painless.spi.WhitelistMethod;
 import org.elasticsearch.painless.spi.annotation.DeprecatedAnnotation;
 import org.elasticsearch.painless.spi.annotation.NoImportAnnotation;
 import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser;
+import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
 import java.util.Map;
 
-public class WhitelistLoaderTests extends ScriptTestCase {
+public class WhitelistLoaderTests extends ESTestCase {
     public void testUnknownAnnotations() {
         Map<String, WhitelistAnnotationParser> parsers = new HashMap<>(WhitelistAnnotationParser.BASE_ANNOTATION_PARSERS);
 

modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.annotation → modules/lang-painless/spi/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.annotation

@@ -1,7 +1,7 @@
 # whitelist for annotation tests
 
-class org.elasticsearch.painless.AnnotationTestObject @no_import {
+class org.elasticsearch.painless.spi.AnnotationTestObject @no_import {
   void deprecatedMethod() @deprecated[message="use another method"]
   void annotatedTestMethod() @test_annotation[one="one",two="two",three="three"]
   void annotatedMultipleMethod() @test_annotation[one="one",two="two",three="three"] @deprecated[message="test"]
-}
+}

modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.annotation.unknown → modules/lang-painless/spi/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.annotation.unknown

@@ -1,6 +1,6 @@
 # whitelist for annotation tests with unknown annotation
 
-class org.elasticsearch.painless.AnnotationTestObject @no_import {
+class org.elasticsearch.painless.spi.AnnotationTestObject @no_import {
   void unknownAnnotationMethod() @unknownAnnotation
   void unknownAnnotationMethod() @unknownAnootationWithMessage[message="use another method"]
 }

modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.annotation.unknown_with_options → modules/lang-painless/spi/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.annotation.unknown_with_options

@@ -1,5 +1,5 @@
 # whitelist for annotation tests with unknown annotation containing options
 
-class org.elasticsearch.painless.AnnotationTestObject @no_import {
+class org.elasticsearch.painless.spi.AnnotationTestObject @no_import {
   void unknownAnnotationMethod() @unknownAnootationWithMessage[arg="arg value"]
 }

modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.painless.spi.PainlessExtension;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.painless.spi.WhitelistLoader;
+import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -65,6 +66,26 @@
 public final class PainlessPlugin extends Plugin implements ScriptPlugin, ExtensiblePlugin, ActionPlugin {
 
     private static final Map<ScriptContext<?>, List<Whitelist>> whitelists;
+    private static final String[] BASE_WHITELIST_FILES = new String[] {
+        "org.elasticsearch.txt",
+        "org.elasticsearch.net.txt",
+        "org.elasticsearch.script.fields.txt",
+        "java.lang.txt",
+        "java.math.txt",
+        "java.text.txt",
+        "java.time.txt",
+        "java.time.chrono.txt",
+        "java.time.format.txt",
+        "java.time.temporal.txt",
+        "java.time.zone.txt",
+        "java.util.txt",
+        "java.util.function.txt",
+        "java.util.regex.txt",
+        "java.util.stream.txt"
+    };
+    public static final List<Whitelist> BASE_WHITELISTS =
+            Collections.singletonList(WhitelistLoader.loadFromResourceFiles(
+                    PainlessPlugin.class, WhitelistAnnotationParser.BASE_ANNOTATION_PARSERS, BASE_WHITELIST_FILES));
 
     /*
      * Contexts from Core that need custom whitelists can add them to the map below.
@@ -75,22 +96,23 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin, Extens
         Map<ScriptContext<?>, List<Whitelist>> map = new HashMap<>();
 
         // Moving Function Pipeline Agg
-        List<Whitelist> movFn = new ArrayList<>(Whitelist.BASE_WHITELISTS);
-        Whitelist movFnWhitelist = WhitelistLoader.loadFromResourceFiles(Whitelist.class, "org.elasticsearch.aggs.movfn.txt");
+        List<Whitelist> movFn = new ArrayList<>();
+        Whitelist movFnWhitelist = WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, "org.elasticsearch.aggs.movfn.txt");
         movFn.add(movFnWhitelist);
         map.put(MovingFunctionScript.CONTEXT, movFn);
 
         // Functions used for scoring docs
-        List<Whitelist> scoreFn = new ArrayList<>(Whitelist.BASE_WHITELISTS);
-        Whitelist scoreFnWhitelist = WhitelistLoader.loadFromResourceFiles(Whitelist.class, "org.elasticsearch.score.txt");
-        Whitelist scoreFieldWhitelist = WhitelistLoader.loadFromResourceFiles(Whitelist.class, "org.elasticsearch.script.fields.score.txt");
+        List<Whitelist> scoreFn = new ArrayList<>();
+        Whitelist scoreFnWhitelist = WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, "org.elasticsearch.score.txt");
+        Whitelist scoreFieldWhitelist =
+            WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, "org.elasticsearch.script.fields.score.txt");
         scoreFn.add(scoreFnWhitelist);
         scoreFn.add(scoreFieldWhitelist);
         map.put(ScoreScript.CONTEXT, scoreFn);
 
         // Functions available to ingest pipelines
-        List<Whitelist> ingest = new ArrayList<>(Whitelist.BASE_WHITELISTS);
-        Whitelist ingestWhitelist = WhitelistLoader.loadFromResourceFiles(Whitelist.class, "org.elasticsearch.ingest.txt");
+        List<Whitelist> ingest = new ArrayList<>();
+        Whitelist ingestWhitelist = WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, "org.elasticsearch.ingest.txt");
         ingest.add(ingestWhitelist);
         map.put(IngestScript.CONTEXT, ingest);
 
@@ -100,35 +122,38 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin, Extens
             map.put(scriptContext, getRuntimeFieldWhitelist(scriptContext.name));
         }
 
-        List<Whitelist> numSort = new ArrayList<>(Whitelist.BASE_WHITELISTS);
-        Whitelist numSortField = WhitelistLoader.loadFromResourceFiles(Whitelist.class, "org.elasticsearch.script.fields.numbersort.txt");
+        List<Whitelist> numSort = new ArrayList<>();
+        Whitelist numSortField =
+            WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, "org.elasticsearch.script.fields.numbersort.txt");
         numSort.add(numSortField);
         map.put(NumberSortScript.CONTEXT, numSort);
 
-        List<Whitelist> strSort = new ArrayList<>(Whitelist.BASE_WHITELISTS);
-        Whitelist strSortField = WhitelistLoader.loadFromResourceFiles(Whitelist.class, "org.elasticsearch.script.fields.stringsort.txt");
+        List<Whitelist> strSort = new ArrayList<>();
+        Whitelist strSortField =
+            WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, "org.elasticsearch.script.fields.stringsort.txt");
         strSort.add(strSortField);
         map.put(StringSortScript.CONTEXT, strSort);
 
-        List<Whitelist> filter = new ArrayList<>(Whitelist.BASE_WHITELISTS);
-        Whitelist filterWhitelist = WhitelistLoader.loadFromResourceFiles(Whitelist.class, "org.elasticsearch.script.fields.filter.txt");
+        List<Whitelist> filter = new ArrayList<>();
+        Whitelist filterWhitelist =
+            WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, "org.elasticsearch.script.fields.filter.txt");
         filter.add(filterWhitelist);
         map.put(FilterScript.CONTEXT, filter);
 
         // Execute context gets everything
-        List<Whitelist> test = new ArrayList<>(Whitelist.BASE_WHITELISTS);
+        List<Whitelist> test = new ArrayList<>();
         test.add(movFnWhitelist);
         test.add(scoreFnWhitelist);
         test.add(ingestWhitelist);
-        test.add(WhitelistLoader.loadFromResourceFiles(Whitelist.class, "org.elasticsearch.json.txt"));
+        test.add(WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, "org.elasticsearch.json.txt"));
         map.put(PainlessExecuteAction.PainlessTestScript.CONTEXT, test);
 
         whitelists = map;
     }
 
     private static List<Whitelist> getRuntimeFieldWhitelist(String contextName) {
-        List<Whitelist> scriptField = new ArrayList<>(Whitelist.BASE_WHITELISTS);
-        Whitelist whitelist = WhitelistLoader.loadFromResourceFiles(Whitelist.class,
+        List<Whitelist> scriptField = new ArrayList<>();
+        Whitelist whitelist = WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class,
             "org.elasticsearch.script." + contextName + ".txt");
         scriptField.add(whitelist);
         return scriptField;
@@ -143,7 +168,9 @@ public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<
             // we might have a context that only uses the base whitelists, so would not have been filled in by reloadSPI
             List<Whitelist> contextWhitelists = whitelists.get(context);
             if (contextWhitelists == null) {
-                contextWhitelists = new ArrayList<>(Whitelist.BASE_WHITELISTS);
+                contextWhitelists = new ArrayList<>(BASE_WHITELISTS);
+            } else {
+                contextWhitelists.addAll(BASE_WHITELISTS);
             }
             contextsWithWhitelists.put(context, contextWhitelists);
         }
@@ -174,7 +201,7 @@ public void loadExtensions(ExtensionLoader loader) {
             .flatMap(extension -> extension.getContextWhitelists().entrySet().stream())
             .forEach(entry -> {
                 List<Whitelist> existing = whitelists.computeIfAbsent(entry.getKey(),
-                    c -> new ArrayList<>(Whitelist.BASE_WHITELISTS));
+                    c -> new ArrayList<>(BASE_WHITELISTS));
                 existing.addAll(entry.getValue());
             });
     }

modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java

@@ -46,6 +46,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Supplier;
 import java.util.regex.Pattern;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_B_BYTE_IMPLICIT;
@@ -213,6 +214,21 @@ private boolean isValidType(Class<?> type) {
         return type == def.class || classesToPainlessClassBuilders.containsKey(type);
     }
 
+    private Class<?> loadClass(ClassLoader classLoader, String javaClassName, Supplier<String> errorMessage) {
+        try {
+            return Class.forName(javaClassName, true, classLoader);
+        } catch (ClassNotFoundException cnfe) {
+            try {
+                // Painless provides some api classes that are available only through the painless implementation.
+                return Class.forName(javaClassName);
+            } catch (ClassNotFoundException cnfe2) {
+                IllegalArgumentException iae = new IllegalArgumentException(errorMessage.get(), cnfe2);
+                cnfe2.addSuppressed(cnfe);
+                throw iae;
+            }
+        }
+    }
+
     public void addPainlessClass(ClassLoader classLoader, String javaClassName, boolean importClassName) {
         Objects.requireNonNull(classLoader);
         Objects.requireNonNull(javaClassName);
@@ -229,11 +245,7 @@ public void addPainlessClass(ClassLoader classLoader, String javaClassName, bool
         else if ("float".equals(javaClassName))   clazz = float.class;
         else if ("double".equals(javaClassName))  clazz = double.class;
         else {
-            try {
-                clazz = Class.forName(javaClassName, true, classLoader);
-            } catch (ClassNotFoundException cnfe) {
-                throw new IllegalArgumentException("class [" + javaClassName + "] not found", cnfe);
-            }
+            clazz = loadClass(classLoader, javaClassName, () -> "class [" + javaClassName + "] not found");
         }
 
         addPainlessClass(clazz, importClassName);
@@ -425,12 +437,9 @@ public void addPainlessMethod(ClassLoader classLoader, String targetCanonicalCla
         Class<?> augmentedClass = null;
 
         if (augmentedCanonicalClassName != null) {
-            try {
-                augmentedClass = Class.forName(augmentedCanonicalClassName, true, classLoader);
-            } catch (ClassNotFoundException cnfe) {
-                throw new IllegalArgumentException("augmented class [" + augmentedCanonicalClassName + "] not found for method " +
-                        "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]", cnfe);
-            }
+            augmentedClass = loadClass(classLoader, augmentedCanonicalClassName,
+                () -> "augmented class [" + augmentedCanonicalClassName + "] not found for method " +
+                "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]");
         }
 
         List<Class<?>> typeParameters = new ArrayList<>(canonicalTypeNameParameters.size());
@@ -735,14 +744,7 @@ public void addImportedPainlessMethod(ClassLoader classLoader, String targetJava
         Objects.requireNonNull(returnCanonicalTypeName);
         Objects.requireNonNull(canonicalTypeNameParameters);
 
-        Class<?> targetClass;
-
-        try {
-            targetClass = Class.forName(targetJavaClassName, true, classLoader);
-        } catch (ClassNotFoundException cnfe) {
-            throw new IllegalArgumentException("class [" + targetJavaClassName + "] not found", cnfe);
-        }
-
+        Class<?> targetClass = loadClass(classLoader, targetJavaClassName, () -> "class [" + targetJavaClassName + "] not found");
         String targetCanonicalClassName = typeToCanonicalTypeName(targetClass);
 
         if (targetClass == null) {
@@ -888,14 +890,7 @@ public void addPainlessClassBinding(ClassLoader classLoader, String targetJavaCl
         Objects.requireNonNull(returnCanonicalTypeName);
         Objects.requireNonNull(canonicalTypeNameParameters);
 
-        Class<?> targetClass;
-
-        try {
-            targetClass = Class.forName(targetJavaClassName, true, classLoader);
-        } catch (ClassNotFoundException cnfe) {
-            throw new IllegalArgumentException("class [" + targetJavaClassName + "] not found", cnfe);
-        }
-
+        Class<?> targetClass = loadClass(classLoader, targetJavaClassName, () -> "class [" + targetJavaClassName + "] not found");
         String targetCanonicalClassName = typeToCanonicalTypeName(targetClass);
         List<Class<?>> typeParameters = new ArrayList<>(canonicalTypeNameParameters.size());
 

modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java

@@ -25,8 +25,8 @@ public class AugmentationTests extends ScriptTestCase {
     @Override
     protected Map<ScriptContext<?>, List<Whitelist>> scriptContexts() {
         Map<ScriptContext<?>, List<Whitelist>> contexts = super.scriptContexts();
-        List<Whitelist> digestWhitelist = new ArrayList<>(Whitelist.BASE_WHITELISTS);
-        digestWhitelist.add(WhitelistLoader.loadFromResourceFiles(Whitelist.class, "org.elasticsearch.ingest.txt"));
+        List<Whitelist> digestWhitelist = new ArrayList<>(PainlessPlugin.BASE_WHITELISTS);
+        digestWhitelist.add(WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, "org.elasticsearch.ingest.txt"));
         contexts.put(DigestTestScript.CONTEXT, digestWhitelist);
 
         return contexts;