Skip to content

Commit

Permalink
Merge pull request #13383 from geoand/#13256-v2
Browse files Browse the repository at this point in the history 
Introduce full indexing of specific directories of the classpath
  • Loading branch information
@geoand
geoand authored Nov 25, 2020
2 parents 4f1fab5 + e8638a8 commit 8b8bba7
Show file tree
Hide file tree
Showing 3 changed files with 100 additions and 28 deletions.
14 changes: 8 additions & 6 deletions ...rojects/bootstrap/runner/src/main/java/io/quarkus/bootstrap/runner/QuarkusEntryPoint.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.quarkus.bootstrap.runner;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
Expand Down Expand Up @@ -35,17 +36,18 @@ private static void doRun(Object args) throws IOException, ClassNotFoundExceptio
} else if (Boolean.getBoolean("quarkus.launch.rebuild")) {
doReaugment(appRoot);
} else {
SerializedApplication app = null;
try (InputStream in = Files.newInputStream(appRoot.resolve(QUARKUS_APPLICATION_DAT))) {
SerializedApplication app;
// the magic number here is close to the smallest possible dat file
try (InputStream in = new BufferedInputStream(Files.newInputStream(appRoot.resolve(QUARKUS_APPLICATION_DAT)),
24_576)) {
app = SerializedApplication.read(in, appRoot);
}
try {
Thread.currentThread().setContextClassLoader(app.getRunnerClassLoader());
Class<?> mainClass = app.getRunnerClassLoader().loadClass(app.getMainClass());
mainClass.getMethod("main", String[].class).invoke(null, args);

} finally {
if (app != null) {
app.getRunnerClassLoader().close();
}
app.getRunnerClassLoader().close();
}
}
}
Expand Down
24 changes: 19 additions & 5 deletions ...rojects/bootstrap/runner/src/main/java/io/quarkus/bootstrap/runner/RunnerClassLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ public class RunnerClassLoader extends ClassLoader {

private final Set<String> parentFirstPackages;
private final Set<String> nonExistentResources;
// the following two fields go hand in hand - they need to both be populated from the same data
// in order for the resource loading to work properly
private final Set<String> fullyIndexedDirectories;
private final Map<String, ClassLoadingResource[]> directlyIndexedResourcesIndexMap;

private final ConcurrentMap<ClassLoadingResource, ProtectionDomain> protectionDomains = new ConcurrentHashMap<>();

Expand All @@ -33,11 +37,14 @@ public class RunnerClassLoader extends ClassLoader {
}

RunnerClassLoader(ClassLoader parent, Map<String, ClassLoadingResource[]> resourceDirectoryMap,
Set<String> parentFirstPackages, Set<String> nonExistentResources) {
Set<String> parentFirstPackages, Set<String> nonExistentResources,
Set<String> fullyIndexedDirectories, Map<String, ClassLoadingResource[]> directlyIndexedResourcesIndexMap) {
super(parent);
this.resourceDirectoryMap = resourceDirectoryMap;
this.parentFirstPackages = parentFirstPackages;
this.nonExistentResources = nonExistentResources;
this.fullyIndexedDirectories = fullyIndexedDirectories;
this.directlyIndexedResourcesIndexMap = directlyIndexedResourcesIndexMap;
}

@Override
Expand Down Expand Up @@ -121,13 +128,20 @@ private String sanitizeName(String name) {
}

private ClassLoadingResource[] getClassLoadingResources(String name) {
ClassLoadingResource[] resources = directlyIndexedResourcesIndexMap.get(name);
if (resources != null) {
return resources;
}
String dirName = getDirNameFromResourceName(name);
ClassLoadingResource[] resources;
if (dirName == null) {
resources = resourceDirectoryMap.get("");
} else {
resources = resourceDirectoryMap.get(dirName);
dirName = "";
}
if (!dirName.equals(name) && fullyIndexedDirectories.contains(dirName)) {
// If we arrive here, we know that resource being queried belongs to one of the fully indexed directories
// Had that resource existed however, it would have been present in directlyIndexedResourcesIndexMap
return null;
}
resources = resourceDirectoryMap.get(dirName);
if (resources == null) {
// the resource could itself be a directory
resources = resourceDirectoryMap.get(name);
Expand Down
90 changes: 73 additions & 17 deletions ...cts/bootstrap/runner/src/main/java/io/quarkus/bootstrap/runner/SerializedApplication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,13 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand All @@ -30,9 +34,11 @@
public class SerializedApplication {

public static final String META_INF_VERSIONS = "META-INF/versions/";
// the files immediately (i.e. not recursively) under these paths should all be indexed
private static final Set<String> FULLY_INDEXED_PATHS = new LinkedHashSet<>(Arrays.asList("", "META-INF/services"));

private static final int MAGIC = 0XF0315432;
private static final int VERSION = 1;
private static final int VERSION = 2;

private final RunnerClassLoader runnerClassLoader;
private final String mainClass;
Expand All @@ -57,25 +63,38 @@ public static void write(OutputStream outputStream, String mainClass, Path appli
data.writeInt(MAGIC);
data.writeInt(VERSION);
data.writeUTF(mainClass);
data.writeInt(classPath.size());
for (Path jar : classPath) {
data.writeShort(classPath.size());
Map<String, List<Integer>> directlyIndexedResourcesToCPJarIndex = new HashMap<>();
for (int i = 0; i < classPath.size(); i++) {
Path jar = classPath.get(i);
String relativePath = applicationRoot.relativize(jar).toString().replace("\\", "/");
data.writeUTF(relativePath);
writeJar(data, jar);
Collection<String> resources = writeJar(data, jar);
for (String resource : resources) {
directlyIndexedResourcesToCPJarIndex.computeIfAbsent(resource, s -> new ArrayList<>()).add(i);
}
}
Set<String> parentFirstPackages = new HashSet<>();

for (Path jar : parentFirst) {
collectPackages(jar, parentFirstPackages);
}
data.writeInt(parentFirstPackages.size());
data.writeShort(parentFirstPackages.size());
for (String p : parentFirstPackages) {
data.writeUTF(p.replace("/", ".").replace("\\", "."));
}
data.writeInt(nonExistentResources.size());
data.writeShort(nonExistentResources.size());
for (String nonExistentResource : nonExistentResources) {
data.writeUTF(nonExistentResource);
}
data.writeShort(directlyIndexedResourcesToCPJarIndex.size());
for (Map.Entry<String, List<Integer>> entry : directlyIndexedResourcesToCPJarIndex.entrySet()) {
data.writeUTF(entry.getKey());
data.writeShort(entry.getValue().size());
for (Integer index : entry.getValue()) {
data.writeShort(index);
}
}
data.flush();
}
}
Expand All @@ -91,8 +110,9 @@ public static SerializedApplication read(InputStream inputStream, Path appRoot)
String mainClass = in.readUTF();
Map<String, ClassLoadingResource[]> resourceDirectoryMap = new HashMap<>();
Set<String> parentFirstPackages = new HashSet<>();
int numPaths = in.readInt();
for (int pathCount = 0; pathCount < numPaths; ++pathCount) {
int numPaths = in.readUnsignedShort();
ClassLoadingResource[] allClassLoadingResources = new ClassLoadingResource[numPaths];
for (int pathCount = 0; pathCount < numPaths; pathCount++) {
String path = in.readUTF();
boolean hasManifest = in.readBoolean();
ManifestInfo info = null;
Expand All @@ -101,7 +121,8 @@ public static SerializedApplication read(InputStream inputStream, Path appRoot)
readNullableString(in), readNullableString(in), readNullableString(in));
}
JarResource resource = new JarResource(info, appRoot.resolve(path));
int numDirs = in.readInt();
allClassLoadingResources[pathCount] = resource;
int numDirs = in.readUnsignedShort();
for (int i = 0; i < numDirs; ++i) {
String dir = in.readUTF();
ClassLoadingResource[] existing = resourceDirectoryMap.get(dir);
Expand All @@ -115,18 +136,31 @@ public static SerializedApplication read(InputStream inputStream, Path appRoot)
}
}
}
int packages = in.readInt();
int packages = in.readUnsignedShort();
for (int i = 0; i < packages; ++i) {
parentFirstPackages.add(in.readUTF());
}
Set<String> nonExistentResources = new HashSet<>();
int nonExistentResourcesSize = in.readInt();
int nonExistentResourcesSize = in.readUnsignedShort();
for (int i = 0; i < nonExistentResourcesSize; i++) {
nonExistentResources.add(in.readUTF());
}
// this map is populated correctly because the JarResource entries are added to allClassLoadingResources
// in the same order as the classpath was written during the writing of the index
Map<String, ClassLoadingResource[]> directlyIndexedResourcesIndexMap = new HashMap<>();
int directlyIndexedSize = in.readUnsignedShort();
for (int i = 0; i < directlyIndexedSize; i++) {
String resource = in.readUTF();
int indexesSize = in.readUnsignedShort();
ClassLoadingResource[] matchingResources = new ClassLoadingResource[indexesSize];
for (int j = 0; j < indexesSize; j++) {
matchingResources[j] = allClassLoadingResources[in.readUnsignedShort()];
}
directlyIndexedResourcesIndexMap.put(resource, matchingResources);
}
return new SerializedApplication(
new RunnerClassLoader(ClassLoader.getSystemClassLoader(), resourceDirectoryMap, parentFirstPackages,
nonExistentResources),
nonExistentResources, FULLY_INDEXED_PATHS, directlyIndexedResourcesIndexMap),
mainClass);
}
}
Expand All @@ -138,7 +172,11 @@ private static String readNullableString(DataInputStream in) throws IOException
return null;
}

private static void writeJar(DataOutputStream out, Path jar) throws IOException {
/**
* @return a List of all resources that exist in the paths that we desire to have fully indexed
* (configured via {@code FULLY_INDEXED_PATHS})
*/
private static List<String> writeJar(DataOutputStream out, Path jar) throws IOException {
try (JarFile zip = new JarFile(jar.toFile())) {
Manifest manifest = zip.getManifest();
if (manifest == null) {
Expand All @@ -160,12 +198,16 @@ private static void writeJar(DataOutputStream out, Path jar) throws IOException
}

Set<String> dirs = new HashSet<>();
Map<String, List<String>> fullyIndexedPaths = new HashMap<>();
Enumeration<? extends ZipEntry> entries = zip.entries();
boolean hasDefaultPackge = false;
boolean hasDefaultPackage = false;
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
if (!entry.getName().contains("/")) {
hasDefaultPackge = true;
hasDefaultPackage = true;
if (!entry.getName().isEmpty() && FULLY_INDEXED_PATHS.contains("")) {
fullyIndexedPaths.computeIfAbsent("", s -> new ArrayList<>(10)).add(entry.getName());
}
} else if (!entry.isDirectory()) {
//some jars don't have correct directory entries
//so we look at the file paths instead
Expand All @@ -186,15 +228,29 @@ private static void writeJar(DataOutputStream out, Path jar) throws IOException
}
}
}

for (String path : FULLY_INDEXED_PATHS) {
if (path.isEmpty()) {
continue;
}
if (entry.getName().startsWith(path)) {
fullyIndexedPaths.computeIfAbsent(path, s -> new ArrayList<>(10)).add(entry.getName());
}
}
}
}
if (hasDefaultPackge) {
if (hasDefaultPackage) {
dirs.add("");
}
out.writeInt(dirs.size());
out.writeShort(dirs.size());
for (String i : dirs) {
out.writeUTF(i);
}
List<String> result = new ArrayList<>();
for (List<String> values : fullyIndexedPaths.values()) {
result.addAll(values);
}
return result;
}
}

Expand Down

0 comments on commit 8b8bba7

Please sign in to comment.