Improve efficiency of nested Gradle builds (elastic#92801)

danielmitterdorfer · Jan 12, 2023 · fb6d2ec · fb6d2ec
1 parent 4f680dc
commit fb6d2ec
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 6 deletions.
diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh
@@ -44,3 +44,9 @@ fi
 ## therefore we run main _AFTER_ we run 6.8 which uses an earlier gradle version
 export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA}
 ./gradlew --parallel clean -s resolveAllDependencies -Dorg.gradle.warning.mode=none
+
+## Copy all dependencies into a "read-only" location to be used by nested Gradle builds
+mkdir -p ${HOME}/gradle_ro_cache
+cp -R ${HOME}/.gradle/caches/modules-2 ${HOME}/gradle_ro_cache
+rm ${HOME}/gradle_ro_cache/modules-2/gc.properties
+rm ${HOME}/gradle_ro_cache/modules-2/*.lock
diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle
@@ -336,6 +336,10 @@ tasks.register("bootstrapPerformanceTests", Copy) {
           branchWrapper:"${-> gradle.gradleVersion}".toString()])
 }
 
+tasks.named("jar") {
+  exclude("classpath.index")
+}
+
 def resolveMainWrapperVersion() {
   new URL("https://raw.githubusercontent.com/elastic/elasticsearch/main/build-tools-internal/src/main/resources/minimumGradleVersion").text.trim()
 }

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java
@@ -12,6 +12,7 @@
 import org.apache.tools.ant.taskdefs.condition.Os;
 import org.elasticsearch.gradle.LoggedExec;
 import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.internal.info.BuildParams;
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
@@ -56,14 +57,18 @@ TaskProvider<LoggedExec> bwcTask(String name, Action<LoggedExec> configuration)
 
     private TaskProvider<LoggedExec> createRunBwcGradleTask(Project project, String name, Action<LoggedExec> configAction) {
         return project.getTasks().register(name, LoggedExec.class, loggedExec -> {
-            loggedExec.dependsOn("checkoutBwcBranch");
+            loggedExec.dependsOn("checkoutBwcBranch", "setupGradleUserHome");
             loggedExec.getWorkingDir().set(checkoutDir.get());
 
             loggedExec.getEnvironment().put("JAVA_HOME", unreleasedVersionInfo.zip(checkoutDir, (version, checkoutDir) -> {
                 String minimumCompilerVersion = readFromFile(new File(checkoutDir, minimumCompilerVersionPath(version.version())));
                 return getJavaHome(Integer.parseInt(minimumCompilerVersion));
             }));
 
+            if (BuildParams.isCi() && Os.isFamily(Os.FAMILY_WINDOWS) == false) {
+                loggedExec.getEnvironment().put("GRADLE_RO_DEP_CACHE", "$HOME/gradle_ro_cache");
+            }
+
             if (Os.isFamily(Os.FAMILY_WINDOWS)) {
                 loggedExec.getExecutable().set("cmd");
                 loggedExec.args("/C", "call", new File(checkoutDir.get(), "gradlew").toString());

diff --git a/...l/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/...l/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
@@ -18,6 +18,7 @@
 import org.gradle.api.Task;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.tasks.Copy;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
@@ -77,6 +78,15 @@ private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionI
         TaskProvider<Task> buildBwcTaskProvider = project.getTasks().register("buildBwc");
         List<DistributionProject> distributionProjects = resolveArchiveProjects(checkoutDir.get(), bwcVersion.get());
 
+        // Setup gradle user home directory
+        project.getTasks().register("setupGradleUserHome", Copy.class, copy -> {
+            copy.into(project.getGradle().getGradleUserHomeDir().getAbsolutePath() + "-" + project.getName());
+            copy.from(project.getGradle().getGradleUserHomeDir().getAbsolutePath(), copySpec -> {
+                copySpec.include("gradle.properties");
+                copySpec.include("init.d/*");
+            });
+        });
+
         for (DistributionProject distributionProject : distributionProjects) {
             createBuildBwcTask(
                 bwcSetupExtension,
@@ -227,11 +237,7 @@ static void createBuildBwcTask(
             } else {
                 c.getOutputs().files(expectedOutputFile);
             }
-            c.getOutputs().cacheIf("BWC distribution caching is disabled on 'main' branch", task -> {
-                String gitBranch = System.getenv("GIT_BRANCH");
-                return BuildParams.isCi()
-                    && (gitBranch == null || gitBranch.endsWith("master") == false || gitBranch.endsWith("main") == false);
-            });
+            c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> BuildParams.isCi() == false);
             c.getArgs().add(projectPath.replace('/', ':') + ":" + assembleTaskName);
             if (project.getGradle().getStartParameter().isBuildCacheEnabled()) {
                 c.getArgs().add("--build-cache");