Enable persistent worker mode in Android desugar actions

RELNOTES: The Android desugaring actions now support a persistent worker mode for faster local build performance. Enable it with `--strategy=Desugar=worker`.

Fixes bazelbuild#8342

Closes bazelbuild#8343.

Change-Id: Ib87575ab6c4b65d5927f6f5b7295332fcdb08c0e
PiperOrigin-RevId: 249495154

WORKSPACE

@@ -116,7 +116,7 @@ distdir_tar(
         "2d9566b21fbe405acf5f7bf77eda30df72a4744c.tar.gz",
         "8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz",
         "0.16.2.zip",
-        "android_tools_pkg-0.3.tar.gz",
+        "android_tools_pkg-0.4.tar.gz",
     ],
     dirname = "derived/distdir",
     sha256 = {
@@ -129,7 +129,7 @@ distdir_tar(
         "2d9566b21fbe405acf5f7bf77eda30df72a4744c.tar.gz": "4a1318fed4831697b83ce879b3ab70ae09592b167e5bda8edaff45132d1c3b3f",
         "8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz": "d868ce50d592ef4aad7dec4dd32ae68d2151261913450fac8390b3fd474bb898",
         "0.16.2.zip": "9b72bb0aea72d7cbcfc82a01b1e25bf3d85f791e790ddec16c65e2d906382ee0",
-        "android_tools_pkg-0.3.tar.gz": "04d25892234cdf8ae49d9c275a25b3371bcc26518179619dc785718db2812b5f", # built at 00ff4dbbf339d5f120a983e002815e37a3e8248e
+        "android_tools_pkg-0.4.tar.gz": "331e7706f2bcae8a68057d8ddd3e3f1574bca26c67c65802fc4a8ac6164fa912", # built at 0c7c89d43256217cce2a3aa4335efaa8eefcf5c4
     },
     urls = {
         "e0b0291b2c51fbe5a7cfa14473a1ae850f94f021.zip": [
@@ -164,8 +164,8 @@ distdir_tar(
             "https://mirror.bazel.build/github.com/bazelbuild/rules_nodejs/archive/0.16.2.zip",
             "https://github.com/bazelbuild/rules_nodejs/archive/0.16.2.zip",
         ],
-        "android_tools_pkg-0.3.tar.gz": [
-            "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.3.tar.gz",
+        "android_tools_pkg-0.4.tar.gz": [
+            "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.4.tar.gz",
         ],
     },
 )
@@ -390,7 +390,7 @@ distdir_tar(
         "zulu11.29.3-ca-jdk11.0.2-linux_x64.tar.gz",
         "zulu11.29.3-ca-jdk11.0.2-macosx_x64.zip",
         "zulu11.29.3-ca-jdk11.0.2-win_x64.zip",
-        "android_tools_pkg-0.3.tar.gz",
+        "android_tools_pkg-0.4.tar.gz",
     ],
     dirname = "test_WORKSPACE/distdir",
     sha256 = {
@@ -412,7 +412,7 @@ distdir_tar(
         "zulu11.29.3-ca-jdk11.0.2-linux_x64.tar.gz": "f3f44b6235508e87b760bf37a49e186cc1fa4e9cd28384c4dbf5a33991921e08",
         "zulu11.29.3-ca-jdk11.0.2-macosx_x64.zip": "059f8e3484bf07b63a8f2820d5f528f473eff1befdb1896ee4f8ff06be3b8d8f",
         "zulu11.29.3-ca-jdk11.0.2-win_x64.zip": "e1f5b4ce1b9148140fae2fcfb8a96d1c9b7eac5b8df0e13fbcad9b8561284880",
-        "android_tools_pkg-0.3.tar.gz": "04d25892234cdf8ae49d9c275a25b3371bcc26518179619dc785718db2812b5f", # built at 00ff4dbbf339d5f120a983e002815e37a3e8248e
+        "android_tools_pkg-0.4.tar.gz": "331e7706f2bcae8a68057d8ddd3e3f1574bca26c67c65802fc4a8ac6164fa912", # built at 0c7c89d43256217cce2a3aa4335efaa8eefcf5c4
     },
     urls = {
         "zulu9.0.7.1-jdk9.0.7-linux_x64-allmodules.tar.gz": ["https://mirror.bazel.build/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-linux_x64-allmodules.tar.gz"],
@@ -433,8 +433,8 @@ distdir_tar(
         "zulu11.29.3-ca-jdk11.0.2-linux_x64.tar.gz": ["https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-linux_x64.tar.gz"],
         "zulu11.29.3-ca-jdk11.0.2-macosx_x64.zip": ["https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-macosx_x64.zip"],
         "zulu11.29.3-ca-jdk11.0.2-win_x64.zip": ["https://mirror.bazel.build/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-win_x64.zip"],
-        "android_tools_pkg-0.3.tar.gz": [
-            "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.3.tar.gz",
+        "android_tools_pkg-0.4.tar.gz": [
+            "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.4.tar.gz",
         ],
     },
 )

src/main/java/com/google/devtools/build/lib/bazel/rules/android/android_remote_tools.WORKSPACE

@@ -2,7 +2,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 http_archive(
     name = "android_tools",
-    url = "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.3.tar.gz",
-    sha256 = "04d25892234cdf8ae49d9c275a25b3371bcc26518179619dc785718db2812b5f", # built at 00ff4dbbf339d5f120a983e002815e37a3e8248e
+    url = "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.4.tar.gz",
+    sha256 = "331e7706f2bcae8a68057d8ddd3e3f1574bca26c67c65802fc4a8ac6164fa912", # built at 0c7c89d43256217cce2a3aa4335efaa8eefcf5c4
 )
 

src/main/java/com/google/devtools/build/lib/rules/android/DexArchiveAspect.java

@@ -461,6 +461,7 @@ private static Artifact createDesugarAction(
             .addOutput(result)
             .setMnemonic("Desugar")
             .setProgressMessage("Desugaring %s for Android", jar.prettyPrint())
+            .setExecutionInfo(ExecutionRequirements.WORKER_MODE_ENABLED)
             .addCommandLine(
                 // Always use params file, so we don't need to compute command line length first
                 args.build(), ParamFileInfo.builder(UNQUOTED).setUseAlways(true).build())

src/tools/android/java/com/google/devtools/build/android/desugar/BUILD

@@ -35,6 +35,7 @@ java_library(
     deps = [
         ":deps_collector_api",
         "//src/main/java/com/google/devtools/common/options",
+        "//src/main/protobuf:worker_protocol_java_proto",
         "//src/tools/android/java/com/google/devtools/build/android:android_builder_lib",
         "//src/tools/android/java/com/google/devtools/build/android/desugar/io",
         "//third_party:asm",

src/tools/android/java/com/google/devtools/build/android/desugar/Desugar.java

@@ -34,9 +34,12 @@
 import com.google.devtools.build.android.desugar.io.InputFileProvider;
 import com.google.devtools.build.android.desugar.io.OutputFileProvider;
 import com.google.devtools.build.android.desugar.io.ThrowingClassLoader;
+import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest;
+import com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse;
 import com.google.devtools.common.options.Option;
 import com.google.devtools.common.options.OptionDocumentationCategory;
 import com.google.devtools.common.options.OptionEffectTag;
+import com.google.devtools.common.options.OptionMetadataTag;
 import com.google.devtools.common.options.OptionsBase;
 import com.google.devtools.common.options.OptionsParser;
 import com.google.devtools.common.options.ShellQuotedParamsFilePreProcessor;
@@ -331,6 +334,15 @@ public static class DesugarOptions extends OptionsBase {
           + "This flag may be removed when no longer needed."
     )
     public boolean legacyJacocoFix;
+
+    @Option(
+        name = "persistent_worker",
+        defaultValue = "false",
+        documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+        effectTags = {OptionEffectTag.UNKNOWN},
+        metadataTags = {OptionMetadataTag.HIDDEN},
+        help = "Run as a Bazel persistent worker.")
+    public boolean persistentWorker;
   }
 
   private static final String RUNTIME_LIB_PACKAGE =
@@ -968,10 +980,57 @@ public static void main(String[] args) throws Exception {
     verifyLambdaDumpDirectoryRegistered(dumpDirectory);
 
     DesugarOptions options = parseCommandLineOptions(args);
+    if (options.persistentWorker) {
+      runPersistentWorker(dumpDirectory);
+    } else {
+      processRequest(options, dumpDirectory);
+    }
+  }
+
+  private static void runPersistentWorker(Path dumpDirectory) throws Exception {
+    while (true) {
+      WorkRequest request = WorkRequest.parseDelimitedFrom(System.in);
+
+      if (request == null) {
+        break;
+      }
+
+      String[] argList = new String[request.getArgumentsCount()];
+      argList = request.getArgumentsList().toArray(argList);
+
+      DesugarOptions options = parseCommandLineOptions(argList);
+
+      int exitCode = processRequest(options, dumpDirectory);
+      WorkResponse.newBuilder().setExitCode(exitCode).build().writeDelimitedTo(System.out);
+      System.out.flush();
+    }
+  }
+
+  private static int processRequest(DesugarOptions options, Path dumpDirectory) throws Exception {
+    checkArgument(!options.inputJars.isEmpty(), "--input is required");
+    checkArgument(
+        options.inputJars.size() == options.outputJars.size(),
+        "Desugar requires the same number of inputs and outputs to pair them. #input=%s,#output=%s",
+        options.inputJars.size(),
+        options.outputJars.size());
+    checkArgument(
+        !options.bootclasspath.isEmpty() || options.allowEmptyBootclasspath,
+        "At least one --bootclasspath_entry is required");
+    for (Path path : options.bootclasspath) {
+      checkArgument(!Files.isDirectory(path), "Bootclasspath entry must be a jar file: %s", path);
+    }
+    checkArgument(
+        !options.desugarCoreLibs
+            || !options.rewriteCoreLibraryPrefixes.isEmpty()
+            || !options.emulateCoreLibraryInterfaces.isEmpty(),
+        "--desugar_supported_core_libs requires specifying renamed and/or emulated core libraries");
+
     if (options.verbose) {
       System.out.printf("Lambda classes will be written under %s%n", dumpDirectory);
     }
     new Desugar(options, dumpDirectory).desugar();
+
+    return 0;
   }
 
   static void verifyLambdaDumpDirectoryRegistered(Path dumpDirectory) throws IOException {
@@ -1030,22 +1089,6 @@ private static DesugarOptions parseCommandLineOptions(String[] args) {
     parser.parseAndExitUponError(args);
     DesugarOptions options = parser.getOptions(DesugarOptions.class);
 
-    checkArgument(!options.inputJars.isEmpty(), "--input is required");
-    checkArgument(
-        options.inputJars.size() == options.outputJars.size(),
-        "Desugar requires the same number of inputs and outputs to pair them. #input=%s,#output=%s",
-        options.inputJars.size(),
-        options.outputJars.size());
-    checkArgument(
-        !options.bootclasspath.isEmpty() || options.allowEmptyBootclasspath,
-        "At least one --bootclasspath_entry is required");
-    for (Path path : options.bootclasspath) {
-      checkArgument(!Files.isDirectory(path), "Bootclasspath entry must be a jar file: %s", path);
-    }
-    checkArgument(!options.desugarCoreLibs
-        || !options.rewriteCoreLibraryPrefixes.isEmpty()
-        || !options.emulateCoreLibraryInterfaces.isEmpty(),
-        "--desugar_supported_core_libs requires specifying renamed and/or emulated core libraries");
     return options;
   }
 

tools/android/runtime_deps/upload_android_tools.sh

@@ -30,7 +30,7 @@
 set -euo pipefail
 
 # The version of android_tools.tar.gz
-VERSION="0.3"
+VERSION="0.4"
 VERSIONED_FILENAME="android_tools_pkg-$VERSION.tar.gz"
 
 # Create a temp directory to hold the versioned tarball, and clean it up when the script exits.