Create a lossy index of resource -> ClassPathElement mapping

The size of ClassLoaderState can be extremely problematic so the idea of this patch is to have a lossy index instead and try to find a good compromise between speed and memory usage.
quarkusio · Aug 29, 2024 · a5c83d0 · a5c83d0
1 parent 844a541
commit a5c83d0
Show file tree

Hide file tree

Showing 27 changed files with 595 additions and 269 deletions.
diff --git a/core/deployment/src/main/java/io/quarkus/deployment/CodeGenerator.java b/core/deployment/src/main/java/io/quarkus/deployment/CodeGenerator.java
@@ -340,7 +340,7 @@ public static <T> T readConfig(ApplicationModel appModel, LaunchMode launchMode,
             final QuarkusClassLoader.Builder configClBuilder = QuarkusClassLoader.builder("CodeGenerator Config ClassLoader",
                     deploymentClassLoader, false);
             if (!allowedConfigServices.isEmpty()) {
-                configClBuilder.addElement(new MemoryClassPathElement(allowedConfigServices, true));
+                configClBuilder.addNormalPriorityElement(new MemoryClassPathElement(allowedConfigServices, true));
             }
             if (!bannedConfigServices.isEmpty()) {
                 configClBuilder.addBannedElement(new MemoryClassPathElement(bannedConfigServices, true));

diff --git a/core/deployment/src/main/java/io/quarkus/deployment/GeneratedClassGizmoAdaptor.java b/core/deployment/src/main/java/io/quarkus/deployment/GeneratedClassGizmoAdaptor.java
@@ -1,17 +1,13 @@
 package io.quarkus.deployment;
 
-import static io.quarkus.commons.classloading.ClassLoaderHelper.fromClassNameToResourceName;
-
 import java.io.StringWriter;
 import java.io.Writer;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Function;
 import java.util.function.Predicate;
 
 import io.quarkus.bootstrap.BootstrapDebug;
-import io.quarkus.bootstrap.classloading.ClassPathElement;
 import io.quarkus.bootstrap.classloading.QuarkusClassLoader;
 import io.quarkus.deployment.annotations.BuildProducer;
 import io.quarkus.deployment.builditem.GeneratedClassBuildItem;
@@ -75,12 +71,7 @@ public Writer getSourceWriter(String className) {
     }
 
     public static boolean isApplicationClass(String className) {
-        QuarkusClassLoader cl = (QuarkusClassLoader) Thread.currentThread()
-                .getContextClassLoader();
-        //if the class file is present in this (and not the parent) CL then it is an application class
-        List<ClassPathElement> res = cl
-                .getElementsWithResource(fromClassNameToResourceName(className), true);
-        return !res.isEmpty();
+        return QuarkusClassLoader.isApplicationClass(className);
     }
 
 }
diff --git a/core/deployment/src/main/java/io/quarkus/deployment/dev/IsolatedDevModeMain.java b/core/deployment/src/main/java/io/quarkus/deployment/dev/IsolatedDevModeMain.java
@@ -30,14 +30,12 @@
 import io.quarkus.bootstrap.app.CuratedApplication;
 import io.quarkus.bootstrap.app.RunningQuarkusApplication;
 import io.quarkus.bootstrap.app.StartupAction;
-import io.quarkus.bootstrap.classloading.ClassPathElement;
 import io.quarkus.bootstrap.classloading.QuarkusClassLoader;
 import io.quarkus.bootstrap.logging.InitialConfigurator;
 import io.quarkus.bootstrap.runner.Timing;
 import io.quarkus.builder.BuildChainBuilder;
 import io.quarkus.builder.BuildContext;
 import io.quarkus.builder.BuildStep;
-import io.quarkus.commons.classloading.ClassLoaderHelper;
 import io.quarkus.deployment.builditem.ApplicationClassPredicateBuildItem;
 import io.quarkus.deployment.console.ConsoleCommand;
 import io.quarkus.deployment.console.ConsoleStateManager;
@@ -478,14 +476,8 @@ public void execute(BuildContext context) {
                     //we need to make sure all hot reloadable classes are application classes
                     context.produce(new ApplicationClassPredicateBuildItem(new Predicate<String>() {
                         @Override
-                        public boolean test(String s) {
-                            QuarkusClassLoader cl = (QuarkusClassLoader) Thread.currentThread()
-                                    .getContextClassLoader();
-                            String resourceName = ClassLoaderHelper.fromClassNameToResourceName(s);
-                            //if the class file is present in this (and not the parent) CL then it is an application class
-                            List<ClassPathElement> res = cl
-                                    .getElementsWithResource(resourceName, true);
-                            return !res.isEmpty();
+                        public boolean test(String className) {
+                            return QuarkusClassLoader.isApplicationClass(className);
                         }
                     }));
                 }

diff --git a/core/deployment/src/main/java/io/quarkus/deployment/dev/testing/TestSupport.java b/core/deployment/src/main/java/io/quarkus/deployment/dev/testing/TestSupport.java
@@ -221,7 +221,7 @@ public void init() {
                                             + curatedApplication.getClassLoaderNameSuffix(),
                                             getClass().getClassLoader().getParent(), false);
                                 }
-                                clBuilder.addElement(ClassPathElement.fromDependency(d));
+                                clBuilder.addNormalPriorityElement(ClassPathElement.fromDependency(d));
                             }
                         }
 

diff --git a/core/deployment/src/main/java/io/quarkus/deployment/dev/testing/TestTracingProcessor.java b/core/deployment/src/main/java/io/quarkus/deployment/dev/testing/TestTracingProcessor.java
@@ -25,7 +25,6 @@
 import org.objectweb.asm.MethodVisitor;
 import org.objectweb.asm.Opcodes;
 
-import io.quarkus.bootstrap.classloading.ClassPathElement;
 import io.quarkus.bootstrap.classloading.QuarkusClassLoader;
 import io.quarkus.deployment.IsDevelopment;
 import io.quarkus.deployment.IsNormal;
@@ -155,13 +154,8 @@ public ServiceStartBuildItem searchForTags(CombinedIndexBuildItem combinedIndexB
         return null;
     }
 
-    public boolean isAppClass(String theClassName) {
-        QuarkusClassLoader cl = (QuarkusClassLoader) Thread.currentThread()
-                .getContextClassLoader();
-        //if the class file is present in this (and not the parent) CL then it is an application class
-        List<ClassPathElement> res = cl
-                .getElementsWithResource(theClassName.replace(".", "/") + ".class", true);
-        return !res.isEmpty();
+    public boolean isAppClass(String className) {
+        return QuarkusClassLoader.isApplicationClass(className);
     }
 
     public static class TracingClassVisitor extends ClassVisitor {

diff --git a/independent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/ArchivePathTree.java b/independent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/ArchivePathTree.java
@@ -86,6 +86,11 @@ static ArchivePathTree forPath(Path path, PathFilter filter, boolean manifestEna
         this.pathFilter = pathFilter;
     }
 
+    @Override
+    public boolean providesLocalResources() {
+        return false;
+    }
+
     @Override
     public Collection<Path> getRoots() {
         return List.of(archive);
@@ -240,6 +245,11 @@ protected OpenArchivePathTree(FileSystem fs) {
             this.rootPath = fs.getPath("/");
         }
 
+        @Override
+        public boolean providesLocalResources() {
+            return false;
+        }
+
         @Override
         protected Path getContainerPath() {
             return ArchivePathTree.this.archive;

diff --git a/...endent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/DirectoryPathTree.java b/...endent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/DirectoryPathTree.java
@@ -36,6 +36,11 @@ protected DirectoryPathTree(Path dir, PathFilter pathFilter, PathTreeWithManifes
         this.dir = dir;
     }
 
+    @Override
+    public boolean providesLocalResources() {
+        return true;
+    }
+
     @Override
     protected Path getRootPath() {
         return dir;

diff --git a/independent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/EmptyPathTree.java b/independent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/EmptyPathTree.java
@@ -15,6 +15,11 @@ public static EmptyPathTree getInstance() {
         return INSTANCE;
     }
 
+    @Override
+    public boolean providesLocalResources() {
+        return false;
+    }
+
     @Override
     public Collection<Path> getRoots() {
         return Collections.emptyList();

diff --git a/independent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/FilePathTree.java b/independent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/FilePathTree.java
@@ -22,6 +22,11 @@ class FilePathTree implements OpenPathTree {
         this.pathFilter = pathFilter;
     }
 
+    @Override
+    public boolean providesLocalResources() {
+        return true;
+    }
+
     @Override
     public Collection<Path> getRoots() {
         return Collections.singletonList(file);

diff --git a/...pendent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/FilteredPathTree.java b/...pendent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/FilteredPathTree.java
@@ -17,6 +17,11 @@ public FilteredPathTree(PathTree tree, PathFilter filter) {
         this.filter = Objects.requireNonNull(filter, "filter is null");
     }
 
+    @Override
+    public boolean providesLocalResources() {
+        return original.providesLocalResources();
+    }
+
     @Override
     public Collection<Path> getRoots() {
         return original.getRoots();

diff --git a/...endent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/MultiRootPathTree.java b/...endent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/MultiRootPathTree.java
@@ -25,6 +25,20 @@ public MultiRootPathTree(PathTree... trees) {
         roots = tmp;
     }
 
+    /**
+     * If at least one of the PathTrees contains local resources, we return true.
+     */
+    @Override
+    public boolean providesLocalResources() {
+        for (PathTree tree : trees) {
+            if (tree.providesLocalResources()) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     @Override
     public Collection<Path> getRoots() {
         return roots;

diff --git a/independent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/PathTree.java b/independent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/PathTree.java
@@ -80,6 +80,13 @@ static PathTree ofArchive(Path archive, PathFilter filter) {
         return ArchivePathTree.forPath(archive, filter);
     }
 
+    /**
+     * Whether the PathTree provides local resources.
+     * <p>
+     * For instance a directory or a file provides local resources. A jar does not.
+     */
+    boolean providesLocalResources();
+
     /**
      * The roots of the path tree.
      * <p>

diff --git a/...nt-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/SharedArchivePathTree.java b/...nt-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/SharedArchivePathTree.java
@@ -122,6 +122,11 @@ private CallerOpenPathTree(SharedOpenArchivePathTree delegate) {
             this.delegate = delegate;
         }
 
+        @Override
+        public boolean providesLocalResources() {
+            return delegate.providesLocalResources();
+        }
+
         @Override
         public PathTree getOriginalTree() {
             return delegate.getOriginalTree();

diff --git a/.../classloader-commons/src/main/java/io/quarkus/commons/classloading/ClassLoaderHelper.java b/.../classloader-commons/src/main/java/io/quarkus/commons/classloading/ClassLoaderHelper.java
@@ -6,6 +6,8 @@ public final class ClassLoaderHelper {
     private static final String JDK_INTERNAL = "jdk.internal.";
     private static final String SUN_MISC = "sun.misc.";
 
+    private static final String CLASS_SUFFIX = ".class";
+
     private ClassLoaderHelper() {
         //Not meant to be instantiated
     }
@@ -19,7 +21,23 @@ private ClassLoaderHelper() {
      */
     public static String fromClassNameToResourceName(final String className) {
         //Important: avoid indy!
-        return className.replace('.', '/').concat(".class");
+        return className.replace('.', '/').concat(CLASS_SUFFIX);
+    }
+
+    /**
+     * Helper method to convert a resource name into the corresponding class name:
+     * replace all "/" with "." and remove the ".class" postfix.
+     *
+     * @param resourceName
+     * @return the name of the respective class
+     */
+    public static String fromResourceNameToClassName(final String resourceName) {
+        if (!resourceName.endsWith(CLASS_SUFFIX)) {
+            throw new IllegalArgumentException(
+                    String.format("%s is not a valid resource name as it doesn't end with .class", resourceName));
+        }
+
+        return resourceName.substring(0, resourceName.length() - CLASS_SUFFIX.length()).replace('/', '.');
     }
 
     public static boolean isInJdkPackage(String name) {

diff --git a/...nt-projects/bootstrap/core/src/main/java/io/quarkus/bootstrap/app/CuratedApplication.java b/...nt-projects/bootstrap/core/src/main/java/io/quarkus/bootstrap/app/CuratedApplication.java
@@ -183,10 +183,12 @@ private void addCpElement(QuarkusClassLoader.Builder builder, ResolvedDependency
             //we always load this from the parent if it is available, as this acts as a bridge between the running
             //app and the dev mode code
             builder.addParentFirstElement(element);
+            builder.addNormalPriorityElement(element);
         } else if (dep.isFlagSet(DependencyFlags.CLASSLOADER_LESSER_PRIORITY)) {
             builder.addLesserPriorityElement(element);
+        } else {
+            builder.addNormalPriorityElement(element);
         }
-        builder.addElement(element);
     }
 
     public synchronized QuarkusClassLoader getOrCreateAugmentClassLoader() {
@@ -212,7 +214,7 @@ public synchronized QuarkusClassLoader getOrCreateAugmentClassLoader() {
             }
 
             for (Path i : quarkusBootstrap.getAdditionalDeploymentArchives()) {
-                builder.addElement(ClassPathElement.fromPath(i, false));
+                builder.addNormalPriorityElement(ClassPathElement.fromPath(i, false));
             }
             Map<String, byte[]> banned = new HashMap<>();
             for (Collection<String> i : configuredClassLoading.getRemovedResources().values()) {
@@ -256,7 +258,7 @@ public synchronized QuarkusClassLoader getOrCreateBaseRuntimeClassLoader() {
                 //there is no need to restart so there is no need for an additional CL
 
                 for (Path root : quarkusBootstrap.getApplicationRoot()) {
-                    builder.addElement(ClassPathElement.fromPath(root, true));
+                    builder.addNormalPriorityElement(ClassPathElement.fromPath(root, true));
                 }
             } else {
                 for (Path root : quarkusBootstrap.getApplicationRoot()) {
@@ -269,7 +271,7 @@ public synchronized QuarkusClassLoader getOrCreateBaseRuntimeClassLoader() {
             for (AdditionalDependency i : quarkusBootstrap.getAdditionalApplicationArchives()) {
                 if (!i.isHotReloadable()) {
                     for (Path root : i.getResolvedPaths()) {
-                        builder.addElement(ClassPathElement.fromPath(root, true));
+                        builder.addNormalPriorityElement(ClassPathElement.fromPath(root, true));
                     }
                 } else {
                     for (Path root : i.getResolvedPaths()) {
@@ -339,15 +341,15 @@ public QuarkusClassLoader createDeploymentClassLoader() {
                 .setAggregateParentResources(true);
 
         for (Path root : quarkusBootstrap.getApplicationRoot()) {
-            builder.addElement(ClassPathElement.fromPath(root, true));
+            builder.addNormalPriorityElement(ClassPathElement.fromPath(root, true));
         }
 
         builder.setResettableElement(new MemoryClassPathElement(Collections.emptyMap(), false));
 
         //additional user class path elements first
         for (AdditionalDependency i : quarkusBootstrap.getAdditionalApplicationArchives()) {
             for (Path root : i.getResolvedPaths()) {
-                builder.addElement(ClassPathElement.fromPath(root, true));
+                builder.addNormalPriorityElement(ClassPathElement.fromPath(root, true));
             }
         }
         for (ResolvedDependency dependency : appModel.getDependencies()) {
@@ -361,7 +363,7 @@ public QuarkusClassLoader createDeploymentClassLoader() {
             }
         }
         for (Path root : configuredClassLoading.getAdditionalClasspathElements()) {
-            builder.addElement(ClassPathElement.fromPath(root, true));
+            builder.addNormalPriorityElement(ClassPathElement.fromPath(root, true));
         }
         return builder.build();
     }
@@ -387,15 +389,15 @@ public QuarkusClassLoader createRuntimeClassLoader(ClassLoader base, Map<String,
                 .setAggregateParentResources(true);
         builder.setTransformedClasses(transformedClasses);
 
-        builder.addElement(new MemoryClassPathElement(resources, true));
+        builder.addNormalPriorityElement(new MemoryClassPathElement(resources, true));
         for (Path root : quarkusBootstrap.getApplicationRoot()) {
-            builder.addElement(ClassPathElement.fromPath(root, true));
+            builder.addNormalPriorityElement(ClassPathElement.fromPath(root, true));
         }
 
         for (AdditionalDependency i : getQuarkusBootstrap().getAdditionalApplicationArchives()) {
             if (i.isHotReloadable()) {
                 for (Path root : i.getResolvedPaths()) {
-                    builder.addElement(ClassPathElement.fromPath(root, true));
+                    builder.addNormalPriorityElement(ClassPathElement.fromPath(root, true));
                 }
             }
         }
@@ -410,7 +412,7 @@ public QuarkusClassLoader createRuntimeClassLoader(ClassLoader base, Map<String,
             }
         }
         for (Path root : configuredClassLoading.getAdditionalClasspathElements()) {
-            builder.addElement(ClassPathElement.fromPath(root, true));
+            builder.addNormalPriorityElement(ClassPathElement.fromPath(root, true));
         }
         return builder.build();
     }
@@ -493,6 +495,11 @@ public Set<String> getProvidedResources() {
             return delegate.getProvidedResources().stream().filter(s -> s.endsWith(".class")).collect(Collectors.toSet());
         }
 
+        @Override
+        public boolean providesLocalResources() {
+            return delegate.providesLocalResources();
+        }
+
         @Override
         public ProtectionDomain getProtectionDomain() {
             return delegate.getProtectionDomain();