Allow overriding all external repositories used by our integration te…

…sts.

Downloading and extracting external repositories over and over again is the biggest performance and SSD killer in our integration tests.

This PR introduces a mechanism where the integration tests can reuse the repositories downloaded by the "outer" Bazel. It also means we can disable network access for most tests, increasing hermeticity.

RELNOTES: None.
PiperOrigin-RevId: 291901125

src/test/java/com/google/devtools/build/lib/blackbox/bazel/DefaultToolsSetup.java

@@ -14,20 +14,63 @@
 
 package com.google.devtools.build.lib.blackbox.bazel;
 
+import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.blackbox.framework.BlackBoxTestContext;
 import com.google.devtools.build.lib.blackbox.framework.ToolsSetup;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
 
 /** Setup for Bazel default tools */
 public class DefaultToolsSetup implements ToolsSetup {
+
+  private static ImmutableList<String> repos =
+      ImmutableList.<String>builder().add("rules_cc").add("rules_proto").add("rules_java").build();
+
+  private ImmutableList<String> getRepositoryOverrides() {
+    String sharedRepoHome = System.getenv("TEST_REPOSITORY_HOME");
+    if (sharedRepoHome == null) {
+      return ImmutableList.of();
+    }
+
+    Path sharedRepoHomePath = Paths.get(sharedRepoHome);
+    if (!Files.exists(sharedRepoHomePath)) {
+      return ImmutableList.of();
+    }
+
+    ImmutableList.Builder<String> lines = ImmutableList.builder();
+    for (String repo : repos) {
+      Path sharedRepoPath = sharedRepoHomePath.resolve(repo);
+      lines.add(
+          "common --override_repository="
+              + repo
+              + "="
+              + sharedRepoPath.toString().replace('\\', '/'));
+    }
+
+    return lines.build();
+  }
+
   @Override
   public void setup(BlackBoxTestContext context) throws IOException {
     Path outputRoot = Files.createTempDirectory(context.getTmpDir(), "root").toAbsolutePath();
-    context.write(
-        ".bazelrc",
-        "startup --output_user_root=" + outputRoot.toString().replace('\\', '/'),
-        "build -j 8");
+    ArrayList<String> lines = new ArrayList<>();
+    lines.add("startup --output_user_root=" + outputRoot.toString().replace('\\', '/'));
+    lines.addAll(getRepositoryOverrides());
+
+    String sharedInstallBase = System.getenv("TEST_INSTALL_BASE");
+    if (sharedInstallBase != null) {
+      lines.add("startup --install_base=" + sharedInstallBase);
+    }
+
+    String sharedRepoCache = System.getenv("REPOSITORY_CACHE");
+    if (sharedRepoCache != null) {
+      lines.add("common --repository_cache=" + sharedRepoCache);
+      lines.add("common --experimental_repository_cache_hardlinks");
+    }
+
+    context.write(".bazelrc", lines);
   }
 }

src/test/java/com/google/devtools/build/lib/blackbox/framework/BlackBoxTestContext.java

@@ -102,6 +102,21 @@ public Path write(String subPath, String... lines) throws IOException {
     return PathUtils.writeFileInDir(workDir, subPath, lines);
   }
 
+  /**
+   * Writes <code>lines</code> using ISO_8859_1 into the file, specified by the <code>subPath</code>
+   * relative to the working directory. Overrides the file if it exists, creates the file if it does
+   * not exist.
+   *
+   * @param subPath path to file relative to working directory
+   * @param lines lines of text to write. Newlines are added by the method.
+   * @return Path to the file
+   * @throws IOException in case if the file can not be created/overridden, or can not be open for
+   *     writing
+   */
+  public Path write(String subPath, List<String> lines) throws IOException {
+    return PathUtils.writeFileInDir(workDir, subPath, lines);
+  }
+
   /**
    * Writes <code>lines</code> using ISO_8859_1 into the BUILD file in the directory, specified by
    * the <code>subPath</code> relative to the working directory. Overrides the file if it exists,

src/test/java/com/google/devtools/build/lib/blackbox/framework/PathUtils.java

@@ -244,6 +244,21 @@ public static Path writeFileInDir(Path directory, String subPath, String... line
     return writeFile(resolve(directory, subPath), lines);
   }
 
+  /**
+   * Writes the file in the <code>directory/subPath</code> location using ISO_8859_1. Overrides the
+   * file if it exists, creates the file if it does not exist.
+   *
+   * @param directory root directory, under which the subtree with the file is created
+   * @param subPath path under <code>directory</code>, under which the file is created
+   * @param lines lines to be written
+   * @return Path to created file
+   * @throws IOException in case file can not be written
+   */
+  public static Path writeFileInDir(Path directory, String subPath, List<String> lines)
+      throws IOException {
+    return writeFile(resolve(directory, subPath), lines);
+  }
+
   /**
    * Writes the file in the <code>path</code> location using ISO_8859_1. Overrides the file if it
    * exists, creates the file if it does not exist.
@@ -257,6 +272,19 @@ public static Path writeFile(Path path, String... lines) throws IOException {
     return Files.write(path, Lists.newArrayList(lines), StandardCharsets.ISO_8859_1);
   }
 
+  /**
+   * Writes the file in the <code>path</code> location using ISO_8859_1. Overrides the file if it
+   * exists, creates the file if it does not exist.
+   *
+   * @param path location where to write the file
+   * @param lines lines to be written
+   * @throws IOException in case file can not be written
+   */
+  public static Path writeFile(Path path, List<String> lines) throws IOException {
+    Files.createDirectories(path.getParent());
+    return Files.write(path, lines, StandardCharsets.ISO_8859_1);
+  }
+
   /**
    * Writes the BUILD file under <code>directory</code> using ISO_8859_1. Overrides the file if it
    * exists, creates the file if it does not exist.

src/test/py/bazel/test_base.py

@@ -52,6 +52,20 @@ class TestBase(unittest.TestCase):
   _worker_proc = None
   _cas_path = None
 
+  _SHARED_REPOS = (
+      'rules_cc',
+      'rules_java',
+      'rules_proto',
+      'remotejdk11_linux_for_testing',
+      'remotejdk11_linux_aarch64_for_testing',
+      'remotejdk11_macos_for_testing',
+      'remotejdk11_win_for_testing',
+      'remote_java_tools_darwin_for_testing',
+      'remote_java_tools_linux_for_testing',
+      'remote_java_tools_windows_for_testing',
+      'remote_coverage_tools_for_testing',
+  )
+
   def setUp(self):
     unittest.TestCase.setUp(self)
     if self._runfiles is None:
@@ -63,7 +77,19 @@ def setUp(self):
     self._test_cwd = tempfile.mkdtemp(dir=self._tests_root)
     self._test_bazelrc = os.path.join(self._temp, 'test_bazelrc')
     with open(self._test_bazelrc, 'wt') as f:
-      f.write('build --jobs=8\n')
+      shared_repo_home = os.environ.get('TEST_REPOSITORY_HOME')
+      if shared_repo_home and os.path.exists(shared_repo_home):
+        for repo in self._SHARED_REPOS:
+          f.write('common --override_repository={}={}\n'.format(
+              repo.replace('_for_testing', ''),
+              os.path.join(shared_repo_home, repo).replace('\\', '/')))
+      shared_install_base = os.environ.get('TEST_INSTALL_BASE')
+      if shared_install_base:
+        f.write('startup --install_base={}\n'.format(shared_install_base))
+      shared_repo_cache = os.environ.get('REPOSITORY_CACHE')
+      if shared_repo_cache:
+        f.write('common --repository_cache={}\n'.format(shared_repo_cache))
+        f.write('common --experimental_repository_cache_hardlinks\n')
     os.chdir(self._test_cwd)
 
   def tearDown(self):

src/test/shell/testenv.sh

@@ -282,19 +282,79 @@ build --incompatible_skip_genfiles_symlink=false
 ${EXTRA_BAZELRC:-}
 EOF
 
+  if [[ -n ${TEST_REPOSITORY_HOME:-} ]]; then
+    echo "testenv.sh: Using shared repositories from $TEST_REPOSITORY_HOME."
+
+    repos=(
+        "android_tools_for_testing"
+        "bazel_toolchains"
+        "com_google_protobuf"
+        "openjdk10_darwin_archive"
+        "openjdk10_linux_archive"
+        "openjdk10_windows_archive"
+        "openjdk11_darwin_archive"
+        "openjdk11_linux_archive"
+        "openjdk11_windows_archive"
+        "openjdk12_darwin_archive"
+        "openjdk12_linux_archive"
+        "openjdk12_windows_archive"
+        "openjdk9_darwin_archive"
+        "openjdk9_linux_archive"
+        "openjdk9_windows_archive"
+        "openjdk_linux_aarch64_minimal"
+        "openjdk_linux_minimal"
+        "openjdk_macos_minimal"
+        "openjdk_win_minimal"
+        "remote_coverage_tools_for_testing"
+        "remote_java_tools_darwin_for_testing"
+        "remote_java_tools_javac10_test_darwin"
+        "remote_java_tools_javac10_test_linux"
+        "remote_java_tools_javac10_test_windows"
+        "remote_java_tools_javac11_test_darwin"
+        "remote_java_tools_javac11_test_linux"
+        "remote_java_tools_javac11_test_windows"
+        "remote_java_tools_javac12_test_darwin"
+        "remote_java_tools_javac12_test_linux"
+        "remote_java_tools_javac12_test_windows"
+        "remote_java_tools_javac9_test_darwin"
+        "remote_java_tools_javac9_test_linux"
+        "remote_java_tools_javac9_test_windows"
+        "remote_java_tools_linux_for_testing"
+        "remote_java_tools_windows_for_testing"
+        "remotejdk10_linux_for_testing"
+        "remotejdk10_linux_aarch64_for_testing"
+        "remotejdk10_macos_for_testing"
+        "remotejdk10_win_for_testing"
+        "remotejdk11_linux_for_testing"
+        "remotejdk11_linux_aarch64_for_testing"
+        "remotejdk11_macos_for_testing"
+        "remotejdk11_win_for_testing"
+        "remotejdk_linux_for_testing"
+        "remotejdk_linux_aarch64_for_testing"
+        "remotejdk_macos_for_testing"
+        "remotejdk_win_for_testing"
+        "rules_cc"
+        "rules_java"
+        "rules_pkg"
+        "rules_proto"
+        "rules_python"
+    )
+    for repo in "${repos[@]}"; do
+      reponame="${repo%"_for_testing"}"
+      echo "common --override_repository=$reponame=$TEST_REPOSITORY_HOME/$repo" >> $TEST_TMPDIR/bazelrc
+    done
+  fi
+
   if [[ -n ${REPOSITORY_CACHE:-} ]]; then
     echo "testenv.sh: Using repository cache at $REPOSITORY_CACHE."
     cat >>$TEST_TMPDIR/bazelrc <<EOF
-sync --repository_cache=$REPOSITORY_CACHE --experimental_repository_cache_hardlinks
-fetch --repository_cache=$REPOSITORY_CACHE --experimental_repository_cache_hardlinks
-build --repository_cache=$REPOSITORY_CACHE --experimental_repository_cache_hardlinks
-query --repository_cache=$REPOSITORY_CACHE --experimental_repository_cache_hardlinks
+common --repository_cache=$REPOSITORY_CACHE --experimental_repository_cache_hardlinks
 EOF
   fi
 
-  if [[ -n ${INSTALL_BASE:-} ]]; then
-    echo "testenv.sh: Using shared install base at $INSTALL_BASE."
-    echo "startup --install_base=$INSTALL_BASE" >> $TEST_TMPDIR/bazelrc
+  if [[ -n ${TEST_INSTALL_BASE:-} ]]; then
+    echo "testenv.sh: Using shared install base at $TEST_INSTALL_BASE."
+    echo "startup --install_base=$TEST_INSTALL_BASE" >> $TEST_TMPDIR/bazelrc
   fi
 }