Skip to content

Commit

Permalink
Merge pull request #27159 from mkouba/reuse-build-executor
Browse files Browse the repository at this point in the history 
Reuse the build executor thread pool for parallel execution
  • Loading branch information
@mkouba
mkouba authored Aug 8, 2022
2 parents 74b1e1c + efe6397 commit 565dc63
Show file tree
Hide file tree
Showing 4 changed files with 103 additions and 114 deletions.
4 changes: 2 additions & 2 deletions core/builder/src/main/java/io/quarkus/builder/BuildContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

Expand Down Expand Up @@ -224,7 +224,7 @@ public void error(Location location, String format, Object... args) {
*
* @return an executor which can be used for asynchronous tasks
*/
public Executor getExecutor() {
public ExecutorService getExecutor() {
return execution.getExecutor();
}

Expand Down
4 changes: 3 additions & 1 deletion core/deployment/src/main/java/io/quarkus/deployment/ExtensionLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.BooleanSupplier;
Expand Down Expand Up @@ -620,7 +621,8 @@ private static Consumer<BuildChainBuilder> loadStepsFromClass(Class<?> clazz,
.andThen(bsb -> bsb.consumes(buildItemClass, ConsumeFlags.of(ConsumeFlag.OPTIONAL)));
methodParamFns.add((bc, bri) -> (Supplier<Optional<? extends SimpleBuildItem>>) () -> Optional
.ofNullable(bc.consume(buildItemClass)));
} else if (rawTypeOf(parameterType) == Executor.class) {
} else if (rawTypeOf(parameterType) == Executor.class
|| rawTypeOf(parameterType) == ExecutorService.class) {
methodParamFns.add((bc, bri) -> bc.getExecutor());
} else if (parameterClass.isAnnotationPresent(ConfigRoot.class)) {
final ConfigRoot annotation = parameterClass.getAnnotation(ConfigRoot.class);
Expand Down
149 changes: 72 additions & 77 deletions core/deployment/src/main/java/io/quarkus/deployment/steps/ClassTransformingBuildStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.BiFunction;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -76,7 +75,8 @@ TransformedClassesBuildItem handleClassTransformation(List<BytecodeTransformerBu
ApplicationArchivesBuildItem appArchives, LiveReloadBuildItem liveReloadBuildItem,
LaunchModeBuildItem launchModeBuildItem, ClassLoadingConfig classLoadingConfig,
CurateOutcomeBuildItem curateOutcomeBuildItem, List<RemovedResourceBuildItem> removedResourceBuildItems,
ArchiveRootBuildItem archiveRoot, LaunchModeBuildItem launchMode, PackageConfig packageConfig)
ArchiveRootBuildItem archiveRoot, LaunchModeBuildItem launchMode, PackageConfig packageConfig,
ExecutorService buildExecutor)
throws ExecutionException, InterruptedException {
if (bytecodeTransformerBuildItems.isEmpty() && classLoadingConfig.removedResources.isEmpty()
&& removedResourceBuildItems.isEmpty()) {
Expand Down Expand Up @@ -113,7 +113,6 @@ TransformedClassesBuildItem handleClassTransformation(List<BytecodeTransformerBu
// now copy all the contents to the runner jar
// we also record if any additional archives needed transformation
// when we copy these archives we will remove the problematic classes
final ExecutorService executorPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
final ConcurrentLinkedDeque<Future<TransformedClassesBuildItem.TransformedClass>> transformed = new ConcurrentLinkedDeque<>();
final Map<Path, Set<TransformedClassesBuildItem.TransformedClass>> transformedClassesByJar = new HashMap<>();
ClassLoader transformCl = Thread.currentThread().getContextClassLoader();
Expand Down Expand Up @@ -173,90 +172,86 @@ public byte[] apply(String className, byte[] originalBytes) {
}
}
};
try {
for (Map.Entry<String, List<BytecodeTransformerBuildItem>> entry : bytecodeTransformers
.entrySet()) {
String className = entry.getKey();
boolean cacheable = !nonCacheable.contains(className);
if (cacheable && transformedClassesCache.containsKey(className)) {
if (liveReloadBuildItem.getChangeInformation() != null) {
if (!liveReloadBuildItem.getChangeInformation().getChangedClasses().contains(className)) {
//we can use the cached transformation
handleTransformedClass(transformedToArchive, transformedClassesByJar,
transformedClassesCache.get(className));
continue;
}
}
}
String classFileName = className.replace('.', '/') + ".class";
List<ClassPathElement> archives = cl.getElementsWithResource(classFileName);
if (!archives.isEmpty()) {
ClassPathElement classPathElement = archives.get(0);
Path jar = classPathElement.getRoot();
if (jar == null) {
log.warnf("Cannot transform %s as its containing application archive could not be found.",
entry.getKey());
for (Map.Entry<String, List<BytecodeTransformerBuildItem>> entry : bytecodeTransformers
.entrySet()) {
String className = entry.getKey();
boolean cacheable = !nonCacheable.contains(className);
if (cacheable && transformedClassesCache.containsKey(className)) {
if (liveReloadBuildItem.getChangeInformation() != null) {
if (!liveReloadBuildItem.getChangeInformation().getChangedClasses().contains(className)) {
//we can use the cached transformation
handleTransformedClass(transformedToArchive, transformedClassesByJar,
transformedClassesCache.get(className));
continue;
}
}
}
String classFileName = className.replace('.', '/') + ".class";
List<ClassPathElement> archives = cl.getElementsWithResource(classFileName);
if (!archives.isEmpty()) {
ClassPathElement classPathElement = archives.get(0);
Path jar = classPathElement.getRoot();
if (jar == null) {
log.warnf("Cannot transform %s as its containing application archive could not be found.",
entry.getKey());
continue;
}

boolean continueOnFailure = entry.getValue().stream()
.filter(a -> !a.isContinueOnFailure())
.findAny().isEmpty();
List<BiFunction<String, ClassVisitor, ClassVisitor>> visitors = entry.getValue().stream()
.map(BytecodeTransformerBuildItem::getVisitorFunction).filter(Objects::nonNull)
.collect(Collectors.toList());
List<BiFunction<String, byte[], byte[]>> preVisitFunctions = entry.getValue().stream()
.map(BytecodeTransformerBuildItem::getInputTransformer).filter(Objects::nonNull)
.collect(Collectors.toList());
transformedToArchive.put(classFileName, jar);
transformed.add(executorPool.submit(new Callable<TransformedClassesBuildItem.TransformedClass>() {
@Override
public TransformedClassesBuildItem.TransformedClass call() throws Exception {
ClassLoader old = Thread.currentThread().getContextClassLoader();
try {
byte[] classData = classPathElement.getResource(classFileName).getData();
Thread.currentThread().setContextClassLoader(transformCl);
Set<String> constValues = constScanning.get(className);
if (constValues != null && !noConstScanning.contains(className)) {
if (!ConstPoolScanner.constPoolEntryPresent(classData, constValues)) {
return null;
}
}
byte[] data = transformClass(className, visitors, classData, preVisitFunctions,
classReaderOptions.getOrDefault(className, 0));
TransformedClassesBuildItem.TransformedClass transformedClass = new TransformedClassesBuildItem.TransformedClass(
className, data,
classFileName, eager.contains(className));
if (cacheable && launchModeBuildItem.getLaunchMode() == LaunchMode.DEVELOPMENT
&& classData != null) {
transformedClassesCache.put(className, transformedClass);
}
return transformedClass;
} catch (Throwable e) {
if (continueOnFailure) {
if (log.isDebugEnabled()) {
log.errorf(e, "Failed to transform %s", className);
} else {
log.errorf("Failed to transform %s", className);
}
boolean continueOnFailure = entry.getValue().stream()
.filter(a -> !a.isContinueOnFailure())
.findAny().isEmpty();
List<BiFunction<String, ClassVisitor, ClassVisitor>> visitors = entry.getValue().stream()
.map(BytecodeTransformerBuildItem::getVisitorFunction).filter(Objects::nonNull)
.collect(Collectors.toList());
List<BiFunction<String, byte[], byte[]>> preVisitFunctions = entry.getValue().stream()
.map(BytecodeTransformerBuildItem::getInputTransformer).filter(Objects::nonNull)
.collect(Collectors.toList());
transformedToArchive.put(classFileName, jar);
transformed.add(buildExecutor.submit(new Callable<TransformedClassesBuildItem.TransformedClass>() {
@Override
public TransformedClassesBuildItem.TransformedClass call() throws Exception {
ClassLoader old = Thread.currentThread().getContextClassLoader();
try {
byte[] classData = classPathElement.getResource(classFileName).getData();
Thread.currentThread().setContextClassLoader(transformCl);
Set<String> constValues = constScanning.get(className);
if (constValues != null && !noConstScanning.contains(className)) {
if (!ConstPoolScanner.constPoolEntryPresent(classData, constValues)) {
return null;
}
}
byte[] data = transformClass(className, visitors, classData, preVisitFunctions,
classReaderOptions.getOrDefault(className, 0));
TransformedClassesBuildItem.TransformedClass transformedClass = new TransformedClassesBuildItem.TransformedClass(
className, data,
classFileName, eager.contains(className));
if (cacheable && launchModeBuildItem.getLaunchMode() == LaunchMode.DEVELOPMENT
&& classData != null) {
transformedClassesCache.put(className, transformedClass);
}
return transformedClass;
} catch (Throwable e) {
if (continueOnFailure) {
if (log.isDebugEnabled()) {
log.errorf(e, "Failed to transform %s", className);
} else {
throw e;
log.errorf("Failed to transform %s", className);
}
} finally {
Thread.currentThread().setContextClassLoader(old);
return null;
} else {
throw e;
}
} finally {
Thread.currentThread().setContextClassLoader(old);
}
}));
} else {
log.warnf("Cannot transform %s as its containing application archive could not be found.",
entry.getKey());
}
}
}));
} else {
log.warnf("Cannot transform %s as its containing application archive could not be found.",
entry.getKey());
}

} finally {
executorPool.shutdown();
}

handleRemovedResources(classLoadingConfig, curateOutcomeBuildItem, transformedClassesByJar, removedResourceBuildItems);
if (!transformed.isEmpty()) {
for (Future<TransformedClassesBuildItem.TransformedClass> i : transformed) {
Expand Down
60 changes: 26 additions & 34 deletions extensions/arc/deployment/src/main/java/io/quarkus/arc/deployment/ArcProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Predicate;
Expand Down Expand Up @@ -516,7 +515,8 @@ public BeanContainerBuildItem generateResources(ArcConfig config, ArcRecorder re
BuildProducer<GeneratedResourceBuildItem> generatedResource,
BuildProducer<BytecodeTransformerBuildItem> bytecodeTransformer,
List<ReflectiveBeanClassBuildItem> reflectiveBeanClasses,
Optional<CurrentContextFactoryBuildItem> currentContextFactory) throws Exception {
Optional<CurrentContextFactoryBuildItem> currentContextFactory,
ExecutorService buildExecutor) throws Exception {

for (ValidationErrorBuildItem validationError : validationErrors) {
for (Throwable error : validationError.getValues()) {
Expand All @@ -539,45 +539,37 @@ public BeanContainerBuildItem generateResources(ArcConfig config, ArcRecorder re
boolean parallelResourceGeneration = Boolean
.parseBoolean(System.getProperty("quarkus.arc.parallel-resource-generation", "true"));
long start = System.nanoTime();
ExecutorService executor = parallelResourceGeneration
? Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())
: null;
ExecutorService executor = parallelResourceGeneration ? buildExecutor : null;
List<ResourceOutput.Resource> resources;
try {
resources = beanProcessor.generateResources(new ReflectionRegistration() {
@Override
public void registerMethod(MethodInfo methodInfo) {
reflectiveMethods.produce(new ReflectiveMethodBuildItem(methodInfo));
}
resources = beanProcessor.generateResources(new ReflectionRegistration() {
@Override
public void registerMethod(MethodInfo methodInfo) {
reflectiveMethods.produce(new ReflectiveMethodBuildItem(methodInfo));
}

@Override
public void registerField(FieldInfo fieldInfo) {
reflectiveFields.produce(new ReflectiveFieldBuildItem(fieldInfo));
}
@Override
public void registerField(FieldInfo fieldInfo) {
reflectiveFields.produce(new ReflectiveFieldBuildItem(fieldInfo));
}

@Override
public void registerClientProxy(DotName beanClassName, String clientProxyName) {
if (reflectiveBeanClassesNames.contains(beanClassName)) {
// Fields should never be registered for client proxies
reflectiveClasses.produce(new ReflectiveClassBuildItem(true, false, clientProxyName));
}
@Override
public void registerClientProxy(DotName beanClassName, String clientProxyName) {
if (reflectiveBeanClassesNames.contains(beanClassName)) {
// Fields should never be registered for client proxies
reflectiveClasses.produce(new ReflectiveClassBuildItem(true, false, clientProxyName));
}
}

@Override
public void registerSubclass(DotName beanClassName, String subclassName) {
if (reflectiveBeanClassesNames.contains(beanClassName)) {
// Fields should never be registered for subclasses
reflectiveClasses.produce(new ReflectiveClassBuildItem(true, false, subclassName));
}
@Override
public void registerSubclass(DotName beanClassName, String subclassName) {
if (reflectiveBeanClassesNames.contains(beanClassName)) {
// Fields should never be registered for subclasses
reflectiveClasses.produce(new ReflectiveClassBuildItem(true, false, subclassName));
}

}, existingClasses.existingClasses, bytecodeTransformerConsumer,
config.shouldEnableBeanRemoval() && config.detectUnusedFalsePositives, executor);
} finally {
if (executor != null) {
executor.shutdown();
}
}

}, existingClasses.existingClasses, bytecodeTransformerConsumer,
config.shouldEnableBeanRemoval() && config.detectUnusedFalsePositives, executor);

for (ResourceOutput.Resource resource : resources) {
switch (resource.getType()) {
Expand Down

0 comments on commit 565dc63

Please sign in to comment.