From 5dd23249f599acac39d846b3b8523e2b91d90554 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 11 Jan 2023 12:51:48 -0800 Subject: [PATCH] Improve efficiency of nested Gradle builds (#92801) --- .ci/packer_cache.sh | 6 ++++++ build-tools-internal/build.gradle | 4 ++++ .../gradle/internal/BwcSetupExtension.java | 7 ++++++- .../InternalDistributionBwcSetupPlugin.java | 16 +++++++++++----- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh index 5a7b3c792d7fc..4f0775610ba15 100755 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -44,3 +44,9 @@ fi ## therefore we run main _AFTER_ we run 6.8 which uses an earlier gradle version export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA} ./gradlew --parallel clean -s resolveAllDependencies -Dorg.gradle.warning.mode=none + +## Copy all dependencies into a "read-only" location to be used by nested Gradle builds +mkdir -p ${HOME}/gradle_ro_cache +cp -R ${HOME}/.gradle/caches/modules-2 ${HOME}/gradle_ro_cache +rm ${HOME}/gradle_ro_cache/modules-2/gc.properties +rm ${HOME}/gradle_ro_cache/modules-2/*.lock diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 92aed9cc94290..8f1ecf041819b 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -336,6 +336,10 @@ tasks.register("bootstrapPerformanceTests", Copy) { branchWrapper:"${-> gradle.gradleVersion}".toString()]) } +tasks.named("jar") { + exclude("classpath.index") +} + def resolveMainWrapperVersion() { new URL("https://raw.githubusercontent.com/elastic/elasticsearch/main/build-tools-internal/src/main/resources/minimumGradleVersion").text.trim() } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index b9566a8624a3f..888ab83036129 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -12,6 +12,7 @@ import org.apache.tools.ant.taskdefs.condition.Os; import org.elasticsearch.gradle.LoggedExec; import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.Action; import org.gradle.api.GradleException; import org.gradle.api.Project; @@ -56,7 +57,7 @@ TaskProvider bwcTask(String name, Action configuration) private TaskProvider createRunBwcGradleTask(Project project, String name, Action configAction) { return project.getTasks().register(name, LoggedExec.class, loggedExec -> { - loggedExec.dependsOn("checkoutBwcBranch"); + loggedExec.dependsOn("checkoutBwcBranch", "setupGradleUserHome"); loggedExec.getWorkingDir().set(checkoutDir.get()); loggedExec.getEnvironment().put("JAVA_HOME", unreleasedVersionInfo.zip(checkoutDir, (version, checkoutDir) -> { @@ -64,6 +65,10 @@ private TaskProvider createRunBwcGradleTask(Project project, String return getJavaHome(Integer.parseInt(minimumCompilerVersion)); })); + if (BuildParams.isCi() && Os.isFamily(Os.FAMILY_WINDOWS) == false) { + loggedExec.getEnvironment().put("GRADLE_RO_DEP_CACHE", "$HOME/gradle_ro_cache"); + } + if (Os.isFamily(Os.FAMILY_WINDOWS)) { loggedExec.getExecutable().set("cmd"); loggedExec.args("/C", "call", new File(checkoutDir.get(), "gradlew").toString()); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java index 373e5ab6bbae2..1f26b8e31ebcf 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java @@ -18,6 +18,7 @@ import org.gradle.api.Task; import org.gradle.api.provider.Provider; import org.gradle.api.provider.ProviderFactory; +import org.gradle.api.tasks.Copy; import org.gradle.api.tasks.TaskProvider; import org.gradle.language.base.plugins.LifecycleBasePlugin; @@ -77,6 +78,15 @@ private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionI TaskProvider buildBwcTaskProvider = project.getTasks().register("buildBwc"); List distributionProjects = resolveArchiveProjects(checkoutDir.get(), bwcVersion.get()); + // Setup gradle user home directory + project.getTasks().register("setupGradleUserHome", Copy.class, copy -> { + copy.into(project.getGradle().getGradleUserHomeDir().getAbsolutePath() + "-" + project.getName()); + copy.from(project.getGradle().getGradleUserHomeDir().getAbsolutePath(), copySpec -> { + copySpec.include("gradle.properties"); + copySpec.include("init.d/*"); + }); + }); + for (DistributionProject distributionProject : distributionProjects) { createBuildBwcTask( bwcSetupExtension, @@ -227,11 +237,7 @@ static void createBuildBwcTask( } else { c.getOutputs().files(expectedOutputFile); } - c.getOutputs().cacheIf("BWC distribution caching is disabled on 'main' branch", task -> { - String gitBranch = System.getenv("GIT_BRANCH"); - return BuildParams.isCi() - && (gitBranch == null || gitBranch.endsWith("master") == false || gitBranch.endsWith("main") == false); - }); + c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> BuildParams.isCi() == false); c.getArgs().add(projectPath.replace('/', ':') + ":" + assembleTaskName); if (project.getGradle().getStartParameter().isBuildCacheEnabled()) { c.getArgs().add("--build-cache");