Vendor registry files

- Vendor registry files needed for Bazel module resolution to achieve offline build with vendor mode. - Also refactored bazel_vendor_test to avoid vendoring dependencies of bazel_tools, which speeds up the test significantly. Fixes bazelbuild#22554 Closes bazelbuild#22566. PiperOrigin-RevId: 641246903 Change-Id: I01a78612ad7ca454df2c70dc5dcc38ec2fbfb71c
meteorcloudy · Jun 18, 2024 · f2fd3a8 · f2fd3a8
1 parent db0d0fc
commit f2fd3a8
Show file tree

Hide file tree

Showing 21 changed files with 570 additions and 179 deletions.
diff --git a/site/en/external/vendor.md b/site/en/external/vendor.md
@@ -67,6 +67,23 @@ Under the hood, it's doing a `bazel build --nobuild` command to analyze the
 target patterns, therefore build flags could be applied to this command and
 affect the result.
 
+### Build the target offline {:#build-the-target-offline}
+
+With the external dependencies vendored, you can build the target offline by
+
+```none
+bazel build --vendor_dir=vendor_src //src/main:hello-world //src/test/...
+```
+
+The build should work in a clean build environment without network access and
+repository cache.
+
+Therefore, you should be able to check in the vendored source and build the same
+targets offline on another machine.
+
+Note: If you build different targets or change the external dependencies, build
+configuration, or Bazel version, you may need to re-vendor.
+
 ## Vendor all external dependencies {:#vendor-all-dependencies}
 
 To vendor all repos in your transitive external dependencies graph, you can

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -165,7 +165,7 @@ public class BazelRepositoryModule extends BlazeModule {
   private Clock clock;
   private Instant lastRegistryInvalidation = Instant.EPOCH;
 
-  private Optional<Path> vendorDirectory;
+  private Optional<Path> vendorDirectory = Optional.empty();
   private List<String> allowedYankedVersions = ImmutableList.of();
   private boolean disableNativeRepoRules;
   private SingleExtensionEvalFunction singleExtensionEvalFunction;
@@ -224,7 +224,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde
     builder.addCommands(new FetchCommand());
     builder.addCommands(new ModCommand());
     builder.addCommands(new SyncCommand());
-    builder.addCommands(new VendorCommand());
+    builder.addCommands(new VendorCommand(downloadManager, clientEnvironmentSupplier));
     builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache));
   }
 
@@ -503,15 +503,10 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
       bazelCompatibilityMode = repoOptions.bazelCompatibilityMode;
       bazelLockfileMode = repoOptions.lockfileMode;
       allowedYankedVersions = repoOptions.allowedYankedVersions;
-
-      if (repoOptions.vendorDirectory != null) {
+      if (env.getWorkspace() != null) {
         vendorDirectory =
-            Optional.of(
-                repoOptions.vendorDirectory.isAbsolute()
-                    ? filesystem.getPath(repoOptions.vendorDirectory)
-                    : env.getWorkspace().getRelative(repoOptions.vendorDirectory));
-      } else {
-        vendorDirectory = Optional.empty();
+            Optional.ofNullable(repoOptions.vendorDirectory)
+                .map(vendorDirectory -> env.getWorkspace().getRelative(vendorDirectory));
       }
 
       if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) {

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
@@ -46,6 +46,17 @@ java_library(
     ],
 )
 
+java_library(
+    name = "vendor",
+    srcs = ["VendorManager.java"],
+    deps = [
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
+        "//src/main/java/com/google/devtools/build/lib/cmdline",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//third_party:guava",
+    ],
+)
+
 java_library(
     name = "module_extension",
     srcs = [
@@ -80,17 +91,20 @@ java_library(
     ],
     deps = [
         ":common",
+        ":vendor",
         ":yanked_versions_value",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/profiler",
         "//src/main/java/com/google/devtools/build/lib/util:os",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
         "//third_party:gson",
         "//third_party:guava",
+        "//third_party:jsr305",
     ],
 )
 

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java
@@ -48,7 +48,7 @@ public abstract class BazelModuleResolutionValue implements SkyValue {
   /**
    * Hashes of files obtained (or known to be missing) from registries while performing resolution.
    */
-  abstract ImmutableMap<String, Optional<Checksum>> getRegistryFileHashes();
+  public abstract ImmutableMap<String, Optional<Checksum>> getRegistryFileHashes();
 
   /**
    * Selected module versions that are known to be yanked (and hence must have been explicitly

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
@@ -32,6 +32,7 @@
 import com.google.devtools.build.lib.profiler.ProfilerTask;
 import com.google.devtools.build.lib.profiler.SilentCloseable;
 import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.gson.FieldNamingPolicy;
 import com.google.gson.Gson;
@@ -45,6 +46,7 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
+import javax.annotation.Nullable;
 
 /**
  * Represents a Bazel module registry that serves a list of module metadata from a static HTTP
@@ -88,6 +90,7 @@ public enum KnownFileHashesMode {
   private final Gson gson;
   private final ImmutableMap<String, Optional<Checksum>> knownFileHashes;
   private final ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions;
+  @Nullable private final VendorManager vendorManager;
   private final KnownFileHashesMode knownFileHashesMode;
   private volatile Optional<BazelRegistryJson> bazelRegistryJson;
   private volatile StoredEventHandler bazelRegistryJsonEvents;
@@ -100,7 +103,8 @@ public IndexRegistry(
       Map<String, String> clientEnv,
       ImmutableMap<String, Optional<Checksum>> knownFileHashes,
       KnownFileHashesMode knownFileHashesMode,
-      ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions) {
+      ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions,
+      Optional<Path> vendorDir) {
     this.uri = uri;
     this.downloadManager = downloadManager;
     this.clientEnv = clientEnv;
@@ -111,6 +115,7 @@ public IndexRegistry(
     this.knownFileHashes = knownFileHashes;
     this.knownFileHashesMode = knownFileHashesMode;
     this.previouslySelectedYankedVersions = previouslySelectedYankedVersions;
+    this.vendorManager = vendorDir.map(VendorManager::new).orElse(null);
   }
 
   @Override
@@ -143,19 +148,19 @@ private Optional<byte[]> grabFile(
   }
 
   private Optional<byte[]> doGrabFile(
-      String url, ExtendedEventHandler eventHandler, boolean useChecksum)
+      String rawUrl, ExtendedEventHandler eventHandler, boolean useChecksum)
       throws IOException, InterruptedException {
     Optional<Checksum> checksum;
     if (knownFileHashesMode != KnownFileHashesMode.IGNORE && useChecksum) {
-      Optional<Checksum> knownChecksum = knownFileHashes.get(url);
+      Optional<Checksum> knownChecksum = knownFileHashes.get(rawUrl);
       if (knownChecksum == null) {
         if (knownFileHashesMode == KnownFileHashesMode.ENFORCE) {
           throw new MissingChecksumException(
               String.format(
                   "Missing checksum for registry file %s not permitted with --lockfile_mode=error."
                       + " Please run `bazel mod deps --lockfile_mode=update` to update your"
                       + " lockfile.",
-                  url));
+                  rawUrl));
         }
         // This is a new file, download without providing a checksum.
         checksum = Optional.empty();
@@ -182,17 +187,40 @@ private Optional<byte[]> doGrabFile(
           "Cannot fetch a file without a checksum in ENFORCE mode. This is a bug in Bazel, please "
               + "report at https://github.com/bazelbuild/bazel/issues/new/choose.");
     }
+
+    URL url = URI.create(rawUrl).toURL();
+    // Don't read the registry URL from the vendor directory in the following cases:
+    // 1. vendorUtil is null, which means vendor mode is disabled.
+    // 2. The checksum is not present, which means the URL is not vendored or the vendored content
+    // is out-dated.
+    // 3. The URL starts with "file:", which means it's a local file and isn't vendored.
+    // 4. The vendor path doesn't exist, which means the URL is not vendored.
+    if (vendorManager != null
+        && checksum.isPresent()
+        && !url.getProtocol().equals("file")
+        && vendorManager.isUrlVendored(url)) {
+      try {
+        return Optional.of(vendorManager.readRegistryUrl(url, checksum.get()));
+      } catch (IOException e) {
+        throw new IOException(
+            String.format(
+                "Failed to read vendored registry file %s at %s: %s. Please rerun the bazel"
+                    + " vendor command.",
+                rawUrl, vendorManager.getVendorPathForUrl(url), e.getMessage()),
+            e);
+      }
+    }
+
     try (SilentCloseable c =
-        Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + url)) {
+        Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + rawUrl)) {
       return Optional.of(
-          downloadManager.downloadAndReadOneUrlForBzlmod(
-              new URL(url), eventHandler, clientEnv, checksum));
+          downloadManager.downloadAndReadOneUrlForBzlmod(url, eventHandler, clientEnv, checksum));
     } catch (FileNotFoundException e) {
       return Optional.empty();
     } catch (IOException e) {
       // Include the URL in the exception message for easier debugging.
       throw new IOException(
-          "Failed to fetch registry file %s: %s".formatted(url, e.getMessage()), e);
+          "Failed to fetch registry file %s: %s".formatted(rawUrl, e.getMessage()), e);
     }
   }
 

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java
@@ -18,6 +18,7 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
 import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
+import com.google.devtools.build.lib.vfs.Path;
 import java.net.URISyntaxException;
 import java.util.Optional;
 
@@ -33,6 +34,7 @@ Registry createRegistry(
       String url,
       RepositoryOptions.LockfileMode lockfileMode,
       ImmutableMap<String, Optional<Checksum>> fileHashes,
-      ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions)
+      ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions,
+      Optional<Path> vendorDir)
       throws URISyntaxException;
 }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java
@@ -20,6 +20,7 @@
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
 import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
 import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
+import com.google.devtools.build.lib.vfs.Path;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Map;
@@ -42,7 +43,8 @@ public Registry createRegistry(
       String url,
       LockfileMode lockfileMode,
       ImmutableMap<String, Optional<Checksum>> knownFileHashes,
-      ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions)
+      ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions,
+      Optional<Path> vendorDir)
       throws URISyntaxException {
     URI uri = new URI(url);
     if (uri.getScheme() == null) {
@@ -75,6 +77,7 @@ public Registry createRegistry(
         clientEnvironmentSupplier.get(),
         knownFileHashes,
         knownFileHashesMode,
-        previouslySelectedYankedVersions);
+        previouslySelectedYankedVersions,
+        vendorDir);
   }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java
@@ -16,6 +16,7 @@
 package com.google.devtools.build.lib.bazel.bzlmod;
 
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
+import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction;
 import com.google.devtools.build.lib.server.FailureDetails;
 import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed;
 import com.google.devtools.build.lib.vfs.Path;
@@ -26,6 +27,7 @@
 import java.net.URISyntaxException;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.Optional;
 import javax.annotation.Nullable;
 
 /** A simple SkyFunction that creates a {@link Registry} with a given URL. */
@@ -56,6 +58,7 @@ public RegistryFunction(RegistryFactory registryFactory, Path workspaceRoot) {
   public SkyValue compute(SkyKey skyKey, Environment env)
       throws InterruptedException, RegistryException {
     LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env);
+    Optional<Path> vendorDir = RepositoryDelegatorFunction.VENDOR_DIRECTORY.get(env);
 
     if (lockfileMode == LockfileMode.REFRESH) {
       RegistryFunction.LAST_INVALIDATION.get(env);
@@ -72,7 +75,8 @@ public SkyValue compute(SkyKey skyKey, Environment env)
           key.getUrl().replace("%workspace%", workspaceRoot.getPathString()),
           lockfileMode,
           lockfile.getRegistryFileHashes(),
-          lockfile.getSelectedYankedVersions());
+          lockfile.getSelectedYankedVersions(),
+          vendorDir);
     } catch (URISyntaxException e) {
       throw new RegistryException(
           ExternalDepsException.withCauseAndMessage(