From e4f1299e51ac2142176cee34917ed0a23d7f4d9d Mon Sep 17 00:00:00 2001 From: Luca Di Grazia Date: Sun, 4 Sep 2022 16:26:44 +0200 Subject: [PATCH] Propagate tags for the AndroidBinary actions Tags are not propagated from targets to actions for Android rules. https://github.com/bazelbuild/bazel/issues/8830 This PR adds basic propagation of tags from the android_binary target it's actions. Testing with aquery with the Android repo: ``` bazel aquery 'mnemonic(RClassGenerator, //apps/foo)' bazel aquery 'mnemonic(JavaDeployJar, //apps/foo)' bazel aquery 'mnemonic(ApkBuilder, //apps/foo)' ``` Closes #13093. PiperOrigin-RevId: 360836954 --- .../lib/rules/android/AndroidBinary.java | 7 +- .../lib/rules/android/AndroidDataContext.java | 6 +- .../lib/rules/android/ApkActionsBuilder.java | 18 - .../rules/android/BusyBoxActionBuilder.java | 264 ++++++++++++- .../lib/rules/android/DexArchiveAspect.java | 12 +- .../lib/rules/java/DeployArchiveBuilder.java | 362 +++++++++++++----- 6 files changed, 519 insertions(+), 150 deletions(-) diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/AndroidBinary.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/AndroidBinary.java index 3799714ea24..1ed8538c60e 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/AndroidBinary.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/AndroidBinary.java @@ -1815,7 +1815,7 @@ static Artifact createMainDexListAction( Artifact jar, @Nullable Artifact mainDexProguardSpec, @Nullable Artifact proguardOutputMap) - throws InterruptedException, RuleErrorException { + throws InterruptedException { AndroidSdkProvider sdk = AndroidSdkProvider.fromRuleContext(ruleContext); // Create the main dex classes list. Artifact mainDexList = AndroidBinary.getDxArtifact(ruleContext, "main_dex_list.txt"); @@ -1841,11 +1841,6 @@ static Artifact createMainDexListAction( // legacy_main_dex_list_generator is provided, use that tool instead. // TODO(b/147692286): Remove the old main-dex list generation that relied on ProGuard. if (legacyMainDexListGenerator == null) { - if (sdk.getShrinkedAndroidJar() == null) { - ruleContext.throwWithRuleError( - "In \"legacy\" multidex mode, either legacy_main_dex_list_generator or " - + "shrinked_android_jar must be set in the android_sdk."); - } // Process the input jar through Proguard into an intermediate, streamlined jar. Artifact strippedJar = AndroidBinary.getDxArtifact(ruleContext, "main_dex_intermediate.jar"); SpawnAction.Builder streamlinedBuilder = diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/AndroidDataContext.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/AndroidDataContext.java index dc55f49f287..03160217e9e 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/AndroidDataContext.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/AndroidDataContext.java @@ -169,9 +169,9 @@ public void registerAction(SpawnAction.Builder spawnActionBuilder) { registerAction(spawnActionBuilder.build(ruleContext)); } - /** Registers an action. */ - public void registerAction(ActionAnalysisMetadata action) { - ruleContext.registerAction(action); + /** Registers one or more actions. */ + public void registerAction(ActionAnalysisMetadata... actions) { + ruleContext.registerAction(actions); } public Artifact createOutputArtifact(SafeImplicitOutputsFunction function) diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/ApkActionsBuilder.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/ApkActionsBuilder.java index 6092864e860..e12b1abd1dd 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/ApkActionsBuilder.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/ApkActionsBuilder.java @@ -49,7 +49,6 @@ public class ApkActionsBuilder { private Artifact signingLineage; private String artifactLocation; private Artifact v4SignatureFile; - private boolean deterministicSigning; private final String apkName; @@ -154,11 +153,6 @@ public ApkActionsBuilder setArtifactLocationDirectory(String artifactLocation) { return this; } - public ApkActionsBuilder setDeterministicSigning(boolean deterministicSigning) { - this.deterministicSigning = deterministicSigning; - return this; - } - /** Registers the actions needed to build the requested APKs in the rule context. */ public void registerActions(RuleContext ruleContext) { // If the caller did not request an unsigned APK, we still need to construct one so that @@ -347,18 +341,6 @@ private void signApk( actionBuilder.addInput(signingLineage); commandLine.add("--lineage").addExecPath(signingLineage); } - - if (deterministicSigning) { - // Enable deterministic DSA signing to keep the output of apksigner deterministic. - // This requires including BouncyCastleProvider as a Security provider, since the standard - // JDK Security providers do not include support for deterministic DSA signing. - // Since this adds BouncyCastleProvider to the end of the Provider list, any non-DSA signing - // algorithms (such as RSA) invoked by apksigner will still use the standard JDK - // implementations and not Bouncy Castle. - commandLine.add("--deterministic-dsa-signing", "true"); - commandLine.add("--provider-class", "org.bouncycastle.jce.provider.BouncyCastleProvider"); - } - for (int i = 0; i < signingKeys.size(); i++) { if (i > 0) { commandLine.add("--next-signer"); diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/BusyBoxActionBuilder.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/BusyBoxActionBuilder.java index 463022c2952..73629e8ba09 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/BusyBoxActionBuilder.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/BusyBoxActionBuilder.java @@ -13,18 +13,24 @@ // limitations under the License. package com.google.devtools.build.lib.rules.android; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ExecutionRequirements; import com.google.devtools.build.lib.actions.ParamFileInfo; import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; -import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.VectorArg; import com.google.devtools.build.lib.analysis.actions.SpawnAction; -import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget.Mode; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.util.OS; import com.google.errorprone.annotations.CompileTimeConstant; +import java.util.Collection; +import java.util.List; +import javax.annotation.Nullable; /** Builder for actions that invoke the Android BusyBox. */ public final class BusyBoxActionBuilder { @@ -42,32 +48,52 @@ public final class BusyBoxActionBuilder { .setUseAlways(OS.getCurrent() == OS.WINDOWS) .build(); - private final RuleContext ruleContext; + private static final ParamFileInfo WORKERS_FORCED_PARAM_FILE_INFO = + ParamFileInfo.builder(ParameterFileType.UNQUOTED) + .setUseAlways(true) + .build(); + + private final AndroidDataContext dataContext; private final NestedSetBuilder inputs = NestedSetBuilder.naiveLinkOrder(); private final ImmutableList.Builder outputs = ImmutableList.builder(); + private final SpawnAction.Builder spawnActionBuilder = new SpawnAction.Builder(); private final CustomCommandLine.Builder commandLine = CustomCommandLine.builder(); public static BusyBoxActionBuilder create( - RuleContext ruleContext, @CompileTimeConstant String toolName) { - BusyBoxActionBuilder builder = new BusyBoxActionBuilder(ruleContext); + AndroidDataContext dataContext, @CompileTimeConstant String toolName) { + BusyBoxActionBuilder builder = new BusyBoxActionBuilder(dataContext); builder.commandLine.add("--tool").add(toolName).add("--"); return builder; } - private BusyBoxActionBuilder(RuleContext ruleContext) { - this.ruleContext = ruleContext; + private BusyBoxActionBuilder(AndroidDataContext dataContext) { + this.dataContext = dataContext; } + /** Adds a direct input artifact. */ + public BusyBoxActionBuilder addInput(Artifact value) { + Preconditions.checkNotNull(value); + commandLine.addExecPath(value); + inputs.add(value); + return this; + } + + /** Adds a direct input artifact. */ public BusyBoxActionBuilder addInput(@CompileTimeConstant String arg, Artifact value) { + Preconditions.checkNotNull(value); commandLine.addExecPath(arg, value); inputs.add(value); return this; } /** - * Adds a series of input artifacts. For efficiency, when adding a NestedSet of artifacts, use - * {@link #addTransitiveFlag(String, NestedSet, AndroidDataConverter)} and {@link - * #addTransitiveInputValues(NestedSet)} instead. + * Adds a series of direct input artifacts. + * + *

For efficiency, when adding a NestedSet of artifacts, use one of the transitive methods, + * such as {@link #addTransitiveFlag(String, NestedSet, AndroidDataConverter)} and {@link + * #addTransitiveInputValues(NestedSet)}, instead. + * + * @param value a string representation of the value artifacts */ public BusyBoxActionBuilder addInput( @CompileTimeConstant String arg, String value, Iterable valueArtifacts) { @@ -76,12 +102,87 @@ public BusyBoxActionBuilder addInput( return this; } + /** Adds the given input artifacts without any command line options. */ + public BusyBoxActionBuilder addInputs(Iterable inputs) { + this.inputs.addAll(inputs); + return this; + } + + /** Adds an input artifact if it is non-null */ + public BusyBoxActionBuilder maybeAddInput( + @CompileTimeConstant String arg, @Nullable Artifact value) { + if (value != null) { + addInput(arg, value); + } + return this; + } + + /** Adds an input artifact if it is non-null */ + public BusyBoxActionBuilder maybeAddInput(@Nullable Artifact value) { + if (value != null) { + this.inputs.add(value); + } + return this; + } + + /** + * Adds a series of direct input artifacts if the list containing them is not null or empty. + * + *

For efficiency, when adding a NestedSet of artifacts, use one of the transitive methods, + * such as {@link #addTransitiveFlag(String, NestedSet, AndroidDataConverter)} and {@link + * #addTransitiveInputValues(NestedSet)}, instead. + */ + public BusyBoxActionBuilder maybeAddInput( + @CompileTimeConstant String arg, @Nullable Collection values) { + if (values != null && !values.isEmpty()) { + commandLine.addExecPaths(arg, values); + inputs.addAll(values); + } + return this; + } + + /** + * Adds a series of direct input artifacts if the list containing them is not null or empty. + * + *

For efficiency, when adding a NestedSet of artifacts, use one of the transitive methods, + * such as {@link #addTransitiveFlag(String, NestedSet, AndroidDataConverter)} and {@link + * #addTransitiveInputValues(NestedSet)}, instead. + * + * @param value a string representation of the value artifacts + */ + public BusyBoxActionBuilder maybeAddInput( + @CompileTimeConstant String arg, + String value, + @Nullable Collection valueArtifacts) { + if (valueArtifacts != null && !valueArtifacts.isEmpty()) { + addInput(arg, value, valueArtifacts); + } + return this; + } + + /** Adds an output artifact */ public BusyBoxActionBuilder addOutput(@CompileTimeConstant String arg, Artifact value) { + Preconditions.checkNotNull(value); commandLine.addExecPath(arg, value); outputs.add(value); return this; } + /** Adds the given output artifacts without adding any command line options. */ + public BusyBoxActionBuilder addOutputs(Iterable outputs) { + this.outputs.addAll(outputs); + return this; + } + + /** Adds an output artifact if it is non-null */ + public BusyBoxActionBuilder maybeAddOutput( + @CompileTimeConstant String arg, @Nullable Artifact value) { + if (value != null) { + return addOutput(arg, value); + } + return this; + } + /** * Adds a series of transitive input artifacts. * @@ -96,6 +197,9 @@ public BusyBoxActionBuilder addTransitiveInputValues(NestedSet values) /** * Adds an efficient flag based on transitive values. * + *

The flag will only be specified once, followed by the joined values specified by the + * converter, for example: --flag value1,value2 + * *

The values will only be collapsed and turned into a flag at execution time. * *

The values will not be added as inputs - use {@link #addTransitiveInputValues(NestedSet)} @@ -109,11 +213,116 @@ public BusyBoxActionBuilder addTransitiveFlag( return this; } + /** + * Adds an efficient flag based on transitive values. + * + *

Each transitive value, as created using the converter, will be proceeded by the flag, for + * example: --flag value1 --flag value2 + * + *

The values will only be collapsed and turned into a flag at execution time. + * + *

The values will not be added as inputs - use {@link #addTransitiveInputValues(NestedSet)} + * for that. + */ + public BusyBoxActionBuilder addTransitiveFlagForEach( + @CompileTimeConstant String arg, + NestedSet transitiveValues, + AndroidDataConverter converter) { + commandLine.addAll(converter.getVectorArgForEach(arg, transitiveValues)); + return this; + } + + /** + * Adds an efficient flag and inputs based on transitive values. + * + *

Each value will be separated on the command line by the ':' character, the option parser's + * PathListConverter delimiter. + * + *

Unlike other transitive input methods in this class, this method adds the values to both the + * command line and the list of inputs. + */ + public BusyBoxActionBuilder addTransitiveVectoredInput( + @CompileTimeConstant String arg, NestedSet values) { + commandLine.addExecPaths(arg, VectorArg.join(":").each(values)); + inputs.addTransitive(values); + return this; + } + + /** Adds a flag with a value set to the current target's label */ + public BusyBoxActionBuilder addLabelFlag(@CompileTimeConstant String arg) { + commandLine.addLabel(arg, dataContext.getLabel()); + return this; + } + + /** Adds a flag with no arguments to the command line. */ public BusyBoxActionBuilder addFlag(@CompileTimeConstant String value) { commandLine.add(value); return this; } + /** Adds a flag with a String value to the command line. */ + public BusyBoxActionBuilder addFlag(@CompileTimeConstant String arg, String value) { + Preconditions.checkNotNull(value); + commandLine.add(arg, value); + return this; + } + + /** If the condition is true, adds a flag with no arguments to the command line. */ + public BusyBoxActionBuilder maybeAddFlag(@CompileTimeConstant String arg, boolean condition) { + if (condition) { + commandLine.add(arg); + } + return this; + } + + /** If the flag is a non-null, non-empty String, adds the flag and value to the command line. */ + public BusyBoxActionBuilder maybeAddFlag( + @CompileTimeConstant String arg, @Nullable String value) { + if (value != null && !value.isEmpty()) { + addFlag(arg, value); + } + return this; + } + + /** + * Efficiently adds a flag and a list of values to the command line. + * + *

The values will be joined in execution and separated by commas. + */ + public BusyBoxActionBuilder addVectoredFlag( + @CompileTimeConstant String arg, List values) { + Preconditions.checkNotNull(values); + commandLine.addAll(arg, VectorArg.join(",").each(values)); + + return this; + } + + /** + * If the values are not null or empty, efficiently adds a flag with them to the command line. + * + *

The values will be joined in execution and separated by commas. + */ + public BusyBoxActionBuilder maybeAddVectoredFlag( + @CompileTimeConstant String arg, @Nullable List values) { + if (values != null && !values.isEmpty()) { + addVectoredFlag(arg, values); + } + return this; + } + + /** Adds aapt to the command line and inputs. */ + public BusyBoxActionBuilder addAapt() { + FilesToRunProvider aapt2 = dataContext.getSdk().getAapt2(); + commandLine.addExecPath("--aapt2", aapt2.getExecutable()); + spawnActionBuilder.addTool(aapt2); + return this; + } + + /** Adds the Android JAR from the SDK to the command line and inputs */ + public BusyBoxActionBuilder addAndroidJar() { + return addInput("--androidJar", dataContext.getSdk().getAndroidJar()); + } + /** * Builds and registers this action. * @@ -122,16 +331,29 @@ public BusyBoxActionBuilder addFlag(@CompileTimeConstant String value) { * @param mnemonic a mnemonic used to indicate the tool being run, for example, "BusyBoxTool". */ public void buildAndRegister(String message, String mnemonic) { - ruleContext.registerAction( - new SpawnAction.Builder() - .useDefaultShellEnvironment() - .addTransitiveInputs(inputs.build()) - .addOutputs(outputs.build()) - .addCommandLine(commandLine.build(), FORCED_PARAM_FILE_INFO) - .setExecutable( - ruleContext.getExecutablePrerequisite("$android_resources_busybox", Mode.HOST)) - .setProgressMessage("%s for %s", message, ruleContext.getLabel()) - .setMnemonic(mnemonic) - .build(ruleContext)); + spawnActionBuilder + .useDefaultShellEnvironment() + .addTransitiveInputs(inputs.build()) + .addOutputs(outputs.build()) + .setExecutable(dataContext.getBusybox()) + .setProgressMessage("%s for %s", message, dataContext.getLabel()) + .setMnemonic(mnemonic); + + ImmutableMap.Builder executionInfo = ImmutableMap.builder(); + executionInfo.putAll(dataContext.getExecutionInfo()); + + if (dataContext.isPersistentBusyboxToolsEnabled()) { + commandLine.add("--logWarnings=false"); + spawnActionBuilder + .addCommandLine(commandLine.build(), WORKERS_FORCED_PARAM_FILE_INFO); + + executionInfo.putAll(ExecutionRequirements.WORKER_MODE_ENABLED); + } else { + spawnActionBuilder.addCommandLine(commandLine.build(), FORCED_PARAM_FILE_INFO); + } + + spawnActionBuilder.setExecutionInfo(executionInfo.build()); + + dataContext.registerAction(spawnActionBuilder); } } diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/DexArchiveAspect.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/DexArchiveAspect.java index ef0b276c9b9..ac3f6ad4ad5 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/DexArchiveAspect.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/android/DexArchiveAspect.java @@ -62,7 +62,7 @@ import com.google.devtools.build.lib.rules.java.JavaCompilationInfoProvider; import com.google.devtools.build.lib.rules.java.JavaInfo; import com.google.devtools.build.lib.rules.java.JavaRuleOutputJarsProvider; -import com.google.devtools.build.lib.rules.java.JavaRuleOutputJarsProvider.JavaOutput; +import com.google.devtools.build.lib.rules.java.JavaRuleOutputJarsProvider.OutputJar; import com.google.devtools.build.lib.rules.java.proto.JavaProtoAspectCommon; import com.google.devtools.build.lib.rules.java.proto.JavaProtoLibraryAspectProvider; import com.google.devtools.build.lib.rules.proto.ProtoInfo; @@ -303,8 +303,10 @@ private static Iterable getProducedRuntimeJars( JavaRuleOutputJarsProvider outputJarsProvider = base.getProvider(JavaRuleOutputJarsProvider.class); if (outputJarsProvider != null) { - return outputJarsProvider.getJavaOutputs().stream() - .map(JavaOutput::getClassJar) + return outputJarsProvider + .getOutputJars() + .stream() + .map(OutputJar::getClassJar) .collect(toImmutableList()); } else { JavaInfo javaInfo = JavaInfo.getJavaInfo(base); @@ -352,8 +354,8 @@ private static boolean isProtoLibrary(RuleContext ruleContext) { private static Artifact getAndroidLibraryRJar(ConfiguredTarget base) { AndroidIdeInfoProvider provider = (AndroidIdeInfoProvider) base.get(AndroidIdeInfoProvider.PROVIDER.getKey()); - if (provider != null && provider.getResourceJarJavaOutput() != null) { - return provider.getResourceJarJavaOutput().getClassJar(); + if (provider != null && provider.getResourceJar() != null) { + return provider.getResourceJar().getClassJar(); } return null; } diff --git a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java index b9fe186ebdb..024ba37dc5c 100644 --- a/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java +++ b/dataset/GitHub_Java/bazelbuild.bazel/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. All rights reserved. +// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,39 +13,50 @@ // limitations under the License. package com.google.devtools.build.lib.rules.java; +import static com.google.common.collect.ImmutableList.toImmutableList; + +import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.CommandLine; +import com.google.devtools.build.lib.actions.ExecutionRequirements; +import com.google.devtools.build.lib.actions.ParamFileInfo; import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; import com.google.devtools.build.lib.actions.ResourceSet; -import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; import com.google.devtools.build.lib.analysis.RuleContext; -import com.google.devtools.build.lib.analysis.actions.CommandLine; import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; import com.google.devtools.build.lib.analysis.actions.SpawnAction; -import com.google.devtools.build.lib.collect.IterablesChain; - +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.rules.cpp.CppHelper; +import com.google.devtools.build.lib.rules.java.JavaConfiguration.OneVersionEnforcementLevel; import java.util.HashSet; import java.util.List; import java.util.Set; - import javax.annotation.Nullable; -/** - * Utility for configuring an action to generate a deploy archive. - */ +/** Utility for configuring an action to generate a deploy archive. */ public class DeployArchiveBuilder { /** * Memory consumption of SingleJar is about 250 bytes per entry in the output file. Unfortunately, * the JVM tends to kill the process with an OOM long before we're at the limit. In the most * recent example, 400 MB of memory was enough for about 500,000 entries. */ - private static final String SINGLEJAR_MAX_MEMORY = "-Xmx1600m"; + private static final int SINGLEJAR_MEMORY_MB = 1600; + + private static final String SINGLEJAR_MAX_MEMORY = "-Xmx" + SINGLEJAR_MEMORY_MB + "m"; + + private static final ResourceSet DEPLOY_ACTION_RESOURCE_SET = + ResourceSet.createWithRamCpu(/*memoryMb = */ SINGLEJAR_MEMORY_MB, /*cpuUsage = */ 1); private final RuleContext ruleContext; - private final IterablesChain.Builder runtimeJarsBuilder = IterablesChain.builder(); + private final NestedSetBuilder runtimeJarsBuilder = NestedSetBuilder.stableOrder(); private final JavaSemantics semantics; @@ -57,10 +68,13 @@ public class DeployArchiveBuilder { @Nullable private String javaStartClass; private ImmutableList deployManifestLines = ImmutableList.of(); @Nullable private Artifact launcher; + @Nullable private Function derivedJars = null; + private boolean checkDesugarDeps; + private OneVersionEnforcementLevel oneVersionEnforcementLevel = OneVersionEnforcementLevel.OFF; + @Nullable private Artifact oneVersionAllowlistArtifact; + @Nullable private Artifact sharedArchive; - /** - * Type of compression to apply to output archive. - */ + /** Type of compression to apply to output archive. */ public enum Compression { /** Output should be compressed */ @@ -70,94 +84,139 @@ public enum Compression { UNCOMPRESSED; } - /** - * Creates a builder using the configuration of the rule as the action configuration. - */ + /** Creates a builder using the configuration of the rule as the action configuration. */ public DeployArchiveBuilder(JavaSemantics semantics, RuleContext ruleContext) { this.ruleContext = ruleContext; this.semantics = semantics; } - /** - * Sets the processed attributes of the rule generating the deploy archive. - */ + /** Sets the processed attributes of the rule generating the deploy archive. */ public DeployArchiveBuilder setAttributes(JavaTargetAttributes attributes) { this.attributes = attributes; return this; } - /** - * Sets whether to include build-data.properties in the deploy archive. - */ + /** Sets whether to include build-data.properties in the deploy archive. */ public DeployArchiveBuilder setIncludeBuildData(boolean includeBuildData) { this.includeBuildData = includeBuildData; return this; } - /** - * Sets whether to enable compression of the output deploy archive. - */ + /** Sets whether to enable compression of the output deploy archive. */ public DeployArchiveBuilder setCompression(Compression compress) { this.compression = Preconditions.checkNotNull(compress); return this; } /** - * Sets additional dependencies to be added to the action that creates the - * deploy jar so that we force the runtime dependencies to be built. + * Sets additional dependencies to be added to the action that creates the deploy jar so that we + * force the runtime dependencies to be built. */ public DeployArchiveBuilder setRunfilesMiddleman(@Nullable Artifact runfilesMiddleman) { this.runfilesMiddleman = runfilesMiddleman; return this; } - /** - * Sets the artifact to create with the action. - */ + /** Sets the artifact to create with the action. */ public DeployArchiveBuilder setOutputJar(Artifact outputJar) { this.outputJar = Preconditions.checkNotNull(outputJar); return this; } - /** - * Sets the class to launch the Java application. - */ + /** Sets the class to launch the Java application. */ public DeployArchiveBuilder setJavaStartClass(@Nullable String javaStartClass) { this.javaStartClass = javaStartClass; return this; } - /** - * Adds additional jars that should be on the classpath at runtime. - */ + /** Adds additional jars that should be on the classpath at runtime. */ + public DeployArchiveBuilder addRuntimeJars(NestedSet jars) { + this.runtimeJarsBuilder.addTransitive(jars); + return this; + } + + /** Adds additional jars that should be on the classpath at runtime. */ public DeployArchiveBuilder addRuntimeJars(Iterable jars) { - this.runtimeJarsBuilder.add(jars); + this.runtimeJarsBuilder.addAll(jars); return this; } - /** - * Sets the list of extra lines to add to the archive's MANIFEST.MF file. - */ - public DeployArchiveBuilder setDeployManifestLines(Iterable deployManifestLines) { - this.deployManifestLines = ImmutableList.copyOf(deployManifestLines); + /** Sets the list of extra lines to add to the archive's MANIFEST.MF file. */ + public DeployArchiveBuilder setDeployManifestLines(ImmutableList deployManifestLines) { + this.deployManifestLines = Preconditions.checkNotNull(deployManifestLines); return this; } - /** - * Sets the optional launcher to be used as the executable for this deploy - * JAR - */ + /** Sets the optional launcher to be used as the executable for this deploy JAR */ public DeployArchiveBuilder setLauncher(@Nullable Artifact launcher) { this.launcher = launcher; return this; } - public static CustomCommandLine.Builder defaultSingleJarCommandLine(Artifact outputJar, + public DeployArchiveBuilder setDerivedJarFunction(Function derivedJars) { + this.derivedJars = derivedJars; + return this; + } + + /** Whether singlejar should process META-INF/desugar_deps files and fail upon inconsistencies. */ + public DeployArchiveBuilder setCheckDesugarDeps(boolean checkDesugarDeps) { + this.checkDesugarDeps = checkDesugarDeps; + return this; + } + + /** Whether or not singlejar would attempt to enforce one version of java classes in the jar */ + public DeployArchiveBuilder setOneVersionEnforcementLevel( + OneVersionEnforcementLevel oneVersionEnforcementLevel, + @Nullable Artifact oneVersionAllowlistArtifact) { + this.oneVersionEnforcementLevel = oneVersionEnforcementLevel; + this.oneVersionAllowlistArtifact = oneVersionAllowlistArtifact; + return this; + } + + public DeployArchiveBuilder setSharedArchive(@Nullable Artifact sharedArchive) { + this.sharedArchive = sharedArchive; + return this; + } + + public static CustomCommandLine.Builder defaultSingleJarCommandLineWithoutOneVersion( + Artifact outputJar, + String javaMainClass, + ImmutableList deployManifestLines, + Iterable buildInfoFiles, + ImmutableList classpathResources, + NestedSet runtimeClasspath, + boolean includeBuildData, + Compression compress, + Artifact launcher, + boolean usingNativeSinglejar) { + return defaultSingleJarCommandLine( + outputJar, + javaMainClass, + deployManifestLines, + buildInfoFiles, + classpathResources, + runtimeClasspath, + includeBuildData, + compress, + launcher, + usingNativeSinglejar, + OneVersionEnforcementLevel.OFF, + null); + } + + public static CustomCommandLine.Builder defaultSingleJarCommandLine( + Artifact outputJar, String javaMainClass, - ImmutableList deployManifestLines, Iterable buildInfoFiles, + ImmutableList deployManifestLines, + Iterable buildInfoFiles, ImmutableList classpathResources, - Iterable runtimeClasspath, boolean includeBuildData, - Compression compress, Artifact launcher) { + NestedSet runtimeClasspath, + boolean includeBuildData, + Compression compress, + Artifact launcher, + boolean usingNativeSinglejar, + OneVersionEnforcementLevel oneVersionEnforcementLevel, + @Nullable Artifact oneVersionAllowlistArtifact) { CustomCommandLine.Builder args = CustomCommandLine.builder(); args.addExecPath("--output", outputJar); @@ -166,13 +225,12 @@ public static CustomCommandLine.Builder defaultSingleJarCommandLine(Artifact out } args.add("--normalize"); if (javaMainClass != null) { - args.add("--main_class"); - args.add(javaMainClass); + args.add("--main_class", javaMainClass); } if (!deployManifestLines.isEmpty()) { args.add("--deploy_manifest_lines"); - args.add(deployManifestLines); + args.addAll(deployManifestLines); } if (buildInfoFiles != null) { @@ -184,73 +242,183 @@ public static CustomCommandLine.Builder defaultSingleJarCommandLine(Artifact out args.add("--exclude_build_data"); } if (launcher != null) { - args.add("--java_launcher"); - args.add(launcher.getExecPathString()); + args.addExecPath("--java_launcher", launcher); } args.addExecPaths("--classpath_resources", classpathResources); - args.addExecPaths("--sources", runtimeClasspath); + if (runtimeClasspath != null) { + if (usingNativeSinglejar) { + args.addAll( + "--sources", OneVersionCheckActionBuilder.jarAndTargetVectorArg(runtimeClasspath)); + } else { + args.addExecPaths("--sources", runtimeClasspath); + } + } + if (oneVersionEnforcementLevel != OneVersionEnforcementLevel.OFF && usingNativeSinglejar) { + args.add("--enforce_one_version"); + // RuleErrors should have been added in Builder.build() before this command + // line is invoked. + Preconditions.checkNotNull(oneVersionAllowlistArtifact); + args.addExecPath("--one_version_whitelist", oneVersionAllowlistArtifact); + if (oneVersionEnforcementLevel == OneVersionEnforcementLevel.WARNING) { + args.add("--succeed_on_found_violations"); + } + } return args; } - /** - * Builds the action as configured. - */ - public void build() { + private static NestedSet getArchiveInputs( + JavaTargetAttributes attributes, + NestedSet runtimeClasspathForArchive, + @Nullable Function derivedJarFunction) { + NestedSetBuilder inputs = NestedSetBuilder.stableOrder(); + if (derivedJarFunction != null) { + inputs.addAll( + runtimeClasspathForArchive.toList().stream() + .map(derivedJarFunction) + .collect(toImmutableList())); + } else { + inputs.addTransitive(runtimeClasspathForArchive); + } + // TODO(bazel-team): Remove? Resources not used as input to singlejar action + inputs.addAll(attributes.getResources().values()); + inputs.addAll(attributes.getClassPathResources()); + return inputs.build(); + } + + /** Builds the action as configured. */ + public void build() throws InterruptedException { ImmutableList classpathResources = attributes.getClassPathResources(); Set classPathResourceNames = new HashSet<>(); for (Artifact artifact : classpathResources) { String name = artifact.getExecPath().getBaseName(); if (!classPathResourceNames.add(name)) { - ruleContext.attributeError("classpath_resources", + ruleContext.attributeError( + "classpath_resources", "entries must have different file names (duplicate: " + name + ")"); return; } } - IterablesChain runtimeJars = runtimeJarsBuilder.build(); + NestedSet runtimeJars = runtimeJarsBuilder.build(); - IterablesChain.Builder inputs = IterablesChain.builder(); - inputs.add(attributes.getArchiveInputs(true)); + NestedSet runtimeClasspathForArchive = attributes.getRuntimeClassPathForArchive(); - inputs.add(ImmutableList.copyOf(runtimeJars)); + // TODO(kmb): Consider not using getArchiveInputs, specifically because we don't want/need to + // transform anything but the runtimeClasspath and b/c we currently do it twice here and below + NestedSetBuilder inputs = NestedSetBuilder.stableOrder(); + inputs.addTransitive(getArchiveInputs(attributes, runtimeClasspathForArchive, derivedJars)); + + if (derivedJars != null) { + inputs.addAll(Iterables.transform(runtimeJars.toList(), derivedJars)); + } else { + inputs.addTransitive(runtimeJars); + } if (runfilesMiddleman != null) { - inputs.addElement(runfilesMiddleman); + inputs.add(runfilesMiddleman); } - final ImmutableList buildInfoArtifacts = - ruleContext.getAnalysisEnvironment().getBuildInfo(ruleContext, JavaBuildInfoFactory.KEY); - inputs.add(buildInfoArtifacts); - - Iterable runtimeClasspath = Iterables.concat( - runtimeJars, - attributes.getRuntimeClassPathForArchive()); + ImmutableList buildInfoArtifacts = ruleContext.getBuildInfo(JavaBuildInfoFactory.KEY); + inputs.addAll(buildInfoArtifacts); + + NestedSetBuilder runtimeClasspath = NestedSetBuilder.stableOrder(); + if (derivedJars != null) { + runtimeClasspath.addAll(Iterables.transform(runtimeJars.toList(), derivedJars)); + runtimeClasspath.addAll( + Iterables.transform(runtimeClasspathForArchive.toList(), derivedJars)); + } else { + runtimeClasspath.addTransitive(runtimeJars); + runtimeClasspath.addTransitive(runtimeClasspathForArchive); + } if (launcher != null) { - inputs.addElement(launcher); + inputs.add(launcher); } - CommandLine commandLine = semantics.buildSingleJarCommandLine(ruleContext.getConfiguration(), - outputJar, javaStartClass, deployManifestLines, buildInfoArtifacts, classpathResources, - runtimeClasspath, includeBuildData, compression, launcher); - - List jvmArgs = ImmutableList.of("-client", SINGLEJAR_MAX_MEMORY); - ResourceSet resourceSet = - new ResourceSet(/*memoryMb = */200.0, /*cpuUsage = */.2, /*ioUsage=*/.2); - - ruleContext.registerAction(new SpawnAction.Builder() - .addInputs(inputs.build()) - .addTransitiveInputs(JavaCompilationHelper.getHostJavabaseInputs(ruleContext)) - .addOutput(outputJar) - .setResources(resourceSet) - .setJarExecutable( - ruleContext.getHostConfiguration().getFragment(Jvm.class).getJavaExecutable(), - ruleContext.getPrerequisiteArtifact("$singlejar", Mode.HOST), - jvmArgs) - .setCommandLine(commandLine) - .useParameterFile(ParameterFileType.SHELL_QUOTED) - .setProgressMessage("Building deploy jar " + outputJar.prettyPrint()) - .setMnemonic("JavaDeployJar") - .build(ruleContext)); + if (oneVersionEnforcementLevel != OneVersionEnforcementLevel.OFF) { + if (oneVersionAllowlistArtifact == null) { + OneVersionCheckActionBuilder.addRuleErrorForMissingArtifacts( + ruleContext, JavaToolchainProvider.from(ruleContext)); + return; + } + inputs.add(oneVersionAllowlistArtifact); + } + if (sharedArchive != null) { + inputs.add(sharedArchive); + } + // If singlejar's name ends with .jar, it is Java application, otherwise it is native. + // TODO(asmundak): once https://github.com/bazelbuild/bazel/issues/2241 is fixed (that is, + // the native singlejar is used on windows) remove support for the Java implementation + Artifact singlejar = JavaToolchainProvider.from(ruleContext).getSingleJar(); + boolean usingNativeSinglejar = !singlejar.getFilename().endsWith(".jar"); + + String toolchainIdentifier = null; + try { + toolchainIdentifier = + CppHelper.getToolchainUsingDefaultCcToolchainAttribute(ruleContext) + .getToolchainIdentifier(); + } catch (RuleErrorException e) { + // Something went wrong loading the toolchain, which is an exceptional condition. + throw new IllegalStateException("Unable to load cc toolchain", e); + } + CommandLine commandLine = + semantics.buildSingleJarCommandLine( + toolchainIdentifier, + outputJar, + javaStartClass, + deployManifestLines, + buildInfoArtifacts, + classpathResources, + runtimeClasspath.build(), + includeBuildData, + compression, + launcher, + usingNativeSinglejar, + oneVersionEnforcementLevel, + oneVersionAllowlistArtifact, + sharedArchive); + if (checkDesugarDeps) { + commandLine = CommandLine.concat(commandLine, ImmutableList.of("--check_desugar_deps")); + } + + List jvmArgs = ImmutableList.of(SINGLEJAR_MAX_MEMORY); + + ImmutableMap.Builder executionInfo = ImmutableMap.builder(); + executionInfo.putAll( + TargetUtils.getExecutionInfo(ruleContext.getRule(), ruleContext.isAllowTagsPropagation())); + + if (!usingNativeSinglejar) { + executionInfo.putAll(ExecutionRequirements.WORKER_MODE_ENABLED); + ruleContext.registerAction( + new SpawnAction.Builder() + .useDefaultShellEnvironment() + .addTransitiveInputs(inputs.build()) + .addTransitiveInputs(JavaRuntimeInfo.forHost(ruleContext).javaBaseInputsMiddleman()) + .addOutput(outputJar) + .setResources(DEPLOY_ACTION_RESOURCE_SET) + .setJarExecutable(JavaCommon.getHostJavaExecutable(ruleContext), singlejar, jvmArgs) + .addCommandLine( + commandLine, + ParamFileInfo.builder(ParameterFileType.SHELL_QUOTED).setUseAlways(true).build()) + .setProgressMessage("Building deploy jar %s", outputJar.prettyPrint()) + .setMnemonic("JavaDeployJar") + .setExecutionInfo(executionInfo.build()) + .build(ruleContext)); + } else { + ruleContext.registerAction( + new SpawnAction.Builder() + .useDefaultShellEnvironment() + .addTransitiveInputs(inputs.build()) + .addOutput(outputJar) + .setResources(DEPLOY_ACTION_RESOURCE_SET) + .setExecutable(singlejar) + .addCommandLine( + commandLine, + ParamFileInfo.builder(ParameterFileType.SHELL_QUOTED).setUseAlways(true).build()) + .setProgressMessage("Building deploy jar %s", outputJar.prettyPrint()) + .setMnemonic("JavaDeployJar") + .setExecutionInfo(executionInfo.build()) + .build(ruleContext)); + } } }