Vendoring (Related #19563)

Vendor command fetches all external repositories into a folder specified by the flag --vendor_dir.
When building using the same flag & value the vendored repos are used as long as they are up to date.

For the initial vendoring or updating existing vendor repositories the new command vendor command is:
`bazel vendor --vendor_dir={vendordir}`

The {vendordir} parameter can be either a full path or just a folder name. Vendored repositories will be located under {workspace}/vendordir.

To perform a build using vendored repositories:
`bazel build {target} --vendor_dir={vendordir}`

In this scenario, the vendor directory will be prioritized for repository retrieval. If the repository is not found or is outdated, the standard process will be followed, which involves checking cached repositories and fetching if not found.

To exclude repositories from the vendoring process:
A file named ".vendorignore" is generated under the vendor directory. By adding the repository name to this file, the vendoring process will ignore it.
Also, any local or configure repos are ignored.

PiperOrigin-RevId: 601099450
Change-Id: I08196c7ad924c1d22918ba09dba2ae774802a691

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -56,6 +56,7 @@
 import com.google.devtools.build.lib.bazel.commands.FetchCommand;
 import com.google.devtools.build.lib.bazel.commands.ModCommand;
 import com.google.devtools.build.lib.bazel.commands.SyncCommand;
+import com.google.devtools.build.lib.bazel.commands.VendorCommand;
 import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformFunction;
 import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformRule;
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
@@ -156,6 +157,8 @@ public class BazelRepositoryModule extends BlazeModule {
   private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING;
   private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR;
   private LockfileMode bazelLockfileMode = LockfileMode.UPDATE;
+
+  private Optional<Path> vendorDirectory;
   private List<String> allowedYankedVersions = ImmutableList.of();
   private SingleExtensionEvalFunction singleExtensionEvalFunction;
   private final ExecutorService repoFetchingWorkerThreadPool =
@@ -216,6 +219,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde
     builder.addCommands(new FetchCommand());
     builder.addCommands(new ModCommand());
     builder.addCommands(new SyncCommand());
+    builder.addCommands(new VendorCommand());
     builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache));
   }
 
@@ -516,6 +520,16 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
       bazelLockfileMode = repoOptions.lockfileMode;
       allowedYankedVersions = repoOptions.allowedYankedVersions;
 
+      if (repoOptions.vendorDirectory != null) {
+        vendorDirectory =
+            Optional.of(
+                repoOptions.vendorDirectory.isAbsolute()
+                    ? filesystem.getPath(repoOptions.vendorDirectory)
+                    : env.getWorkspace().getRelative(repoOptions.vendorDirectory));
+      } else {
+        vendorDirectory = Optional.empty();
+      }
+
       if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) {
         registries = repoOptions.registries;
       } else {
@@ -587,6 +601,8 @@ public ImmutableList<Injected> getPrecomputedValues() {
         PrecomputedValue.injected(
             BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, bazelCompatibilityMode),
         PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, bazelLockfileMode),
+        PrecomputedValue.injected(RepositoryDelegatorFunction.IS_VENDOR_COMMAND, false),
+        PrecomputedValue.injected(RepositoryDelegatorFunction.VENDOR_DIRECTORY, vendorDirectory),
         PrecomputedValue.injected(
             YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, allowedYankedVersions));
   }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllFunction.java

@@ -18,6 +18,7 @@
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryFunction;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
@@ -31,8 +32,8 @@
 import javax.annotation.Nullable;
 
 /**
- * Void function designed to gather and fetch all the repositories without returning any specific
- * result (empty value is returned).
+ * Gather and fetch all the repositories from MODULE.bazel resolution and extensions evaluation. If
+ * this is fetch configure, only configure repos will be fetched and returned
  */
 public class BazelFetchAllFunction implements SkyFunction {
 
@@ -53,14 +54,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept
             .filter(repo -> !repo.isMain())
             .collect(toImmutableList()));
 
-    // 2. Run every extension found in the modules
+    // 2. Run every extension found in the modules & collect its generated repos
     ImmutableSet<ModuleExtensionId> extensionIds =
         depGraphValue.getExtensionUsagesTable().rowKeySet();
     ImmutableSet<SkyKey> singleEvalKeys =
         extensionIds.stream().map(SingleExtensionEvalValue::key).collect(toImmutableSet());
     SkyframeLookupResult singleEvalValues = env.getValuesAndExceptions(singleEvalKeys);
-
-    // 3. For each extension, collect its generated repos
     for (SkyKey singleEvalKey : singleEvalKeys) {
       SingleExtensionEvalValue singleEvalValue =
           (SingleExtensionEvalValue) singleEvalValues.get(singleEvalKey);
@@ -70,7 +69,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept
       reposToFetch.addAll(singleEvalValue.getCanonicalRepoNameToInternalNames().keySet());
     }
 
-    // 4. If this is fetch configure, get repo rules and only collect repos marked as configure
+    // 3. If this is fetch configure, get repo rules and only collect repos marked as configure
     Boolean fetchConfigure = (Boolean) skyKey.argument();
     if (fetchConfigure) {
       ImmutableSet<SkyKey> repoRuleKeys =
@@ -83,12 +82,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept
           return null;
         }
         if (StarlarkRepositoryFunction.isConfigureRule(repoRuleValue.getRule())) {
-          reposToFetch.add(RepositoryName.createUnvalidated(repoRuleValue.getRule().getName()));
+          reposToFetch.add((RepositoryName) repoRuleKey.argument());
         }
       }
     }
 
-    // 5. Fetch all the collected repos
+    // 4. Fetch all the collected repos
+    List<RepositoryName> shouldVendor = new ArrayList<>();
     ImmutableSet<SkyKey> repoDelegatorKeys =
         reposToFetch.stream().map(RepositoryDirectoryValue::key).collect(toImmutableSet());
     SkyframeLookupResult repoDirValues = env.getValuesAndExceptions(repoDelegatorKeys);
@@ -98,9 +98,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept
       if (repoDirValue == null) {
         return null;
       }
+      if (!repoDirValue.excludeFromVendoring()) {
+        shouldVendor.add((RepositoryName) repoDelegatorKey.argument());
+      }
     }
 
-    return BazelFetchAllValue.create();
+    return BazelFetchAllValue.create(
+        ImmutableList.copyOf(reposToFetch), ImmutableList.copyOf(shouldVendor));
   }
 
 }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllValue.java

@@ -16,6 +16,8 @@
 package com.google.devtools.build.lib.bazel.bzlmod;
 
 import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.skyframe.SkyFunctions;
 import com.google.devtools.build.lib.skyframe.serialization.VisibleForSerialization;
 import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
@@ -36,8 +38,13 @@ public static BazelFetchAllValue.Key key(Boolean configureEnabled) {
     return BazelFetchAllValue.Key.create(configureEnabled);
   }
 
-  public static BazelFetchAllValue create() {
-    return new AutoValue_BazelFetchAllValue();
+  public abstract ImmutableList<RepositoryName> getFetchedRepos();
+
+  public abstract ImmutableList<RepositoryName> getReposToVendor();
+
+  public static BazelFetchAllValue create(
+      ImmutableList<RepositoryName> fetchedRepos, ImmutableList<RepositoryName> reposToVendor) {
+    return new AutoValue_BazelFetchAllValue(fetchedRepos, reposToVendor);
   }
 
   /** Key type for BazelFetchAllValue. */

src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD

@@ -20,6 +20,7 @@ java_library(
         "fetch.txt",
         "mod.txt",
         "sync.txt",
+        "vendor.txt",
     ],
     deps = [
         "//src/main/java/com/google/devtools/build/lib:keep-going-option",
@@ -36,6 +37,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/events",
@@ -60,6 +62,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/util:interrupted_failure_details",
         "//src/main/java/com/google/devtools/build/lib/util:maybe_complete_set",
         "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/com/google/devtools/build/skyframe",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
         "//src/main/java/com/google/devtools/common/options",