bzlmod: Load repositories from Bzlmod

With --experimental_enable_bzlmod=true, Bazel can now load external repositories
from the new dependency management system -- Bzlmod.

Related: #13316

RELNOTES: None
PiperOrigin-RevId: 384424361

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -129,6 +129,7 @@ public class BazelRepositoryModule extends BlazeModule {
   // We hold the precomputed value of the managed directories here, so that the dependency
   // on WorkspaceFileValue is not registered for each FileStateValue.
   private final ManagedDirectoriesKnowledgeImpl managedDirectoriesKnowledge;
+  private final AtomicBoolean enableBzlmod = new AtomicBoolean(false);
 
   public BazelRepositoryModule() {
     this.starlarkRepositoryFunction = new StarlarkRepositoryFunction(downloadManager);
@@ -353,6 +354,8 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
         overrides = ImmutableMap.of();
       }
 
+      enableBzlmod.set(repoOptions.enableBzlmod);
+
       if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) {
         registries = repoOptions.registries;
       } else {
@@ -419,6 +422,7 @@ public ImmutableList<Injected> getPrecomputedValues() {
         PrecomputedValue.injected(
             RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_CONFIGURING,
             RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY),
+        PrecomputedValue.injected(RepositoryDelegatorFunction.ENABLE_BZLMOD, enableBzlmod.get()),
         PrecomputedValue.injected(ModuleFileFunction.REGISTRIES, registries));
   }
 

src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java

@@ -168,6 +168,16 @@ public class RepositoryOptions extends OptionsBase {
               + "give, and in this case multiple URLs will be returned.")
   public String downloaderConfig;
 
+  @Option(
+      name = "experimental_enable_bzlmod",
+      defaultValue = "false",
+      documentationCategory = OptionDocumentationCategory.BAZEL_CLIENT_OPTIONS,
+      effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS},
+      help =
+          "If true, Bazel tries to load external repositories from the Bzlmod system before "
+              + "looking into the WORKSPACE file.")
+  public boolean enableBzlmod;
+
   /**
    * Converts from an equals-separated pair of strings into RepositoryName->PathFragment mapping.
    */

src/main/java/com/google/devtools/build/lib/rules/BUILD

@@ -417,6 +417,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
         "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
         "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_value",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/cmdline:LabelValidator",
         "//src/main/java/com/google/devtools/build/lib/concurrent",

src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java

@@ -25,6 +25,7 @@
 import com.google.common.collect.Ordering;
 import com.google.devtools.build.lib.actions.FileValue;
 import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.ExtendedEventHandler.FetchProgress;
@@ -88,6 +89,9 @@ public final class RepositoryDelegatorFunction implements SkyFunction {
   public static final Precomputed<Optional<RootedPath>> RESOLVED_FILE_INSTEAD_OF_WORKSPACE =
       new Precomputed<>("resolved_file_instead_of_workspace");
 
+  // This indicates whether we should load external repositories from the Bzlmod system.
+  public static final Precomputed<Boolean> ENABLE_BZLMOD = new Precomputed<>("enable_bzlmod");
+
   public static final String DONT_FETCH_UNCONDITIONALLY = "";
 
   // The marker file version is inject in the rule key digest so the rule key is always different
@@ -246,14 +250,32 @@ public SkyValue compute(SkyKey skyKey, Environment env)
           overrides.get(repositoryName), env, repoRoot, repositoryName.strippedName());
     }
 
-    Rule rule;
-    try {
-      rule = getRepository(repositoryName, env);
-      if (rule == null) {
+    Rule rule = null;
+
+    if (Preconditions.checkNotNull(ENABLE_BZLMOD.get(env))) {
+      // Trys to get a repository rule instance from Bzlmod generated repos.
+      SkyKey key = BzlmodRepoRuleValue.key(repositoryName.strippedName());
+      BzlmodRepoRuleValue value = (BzlmodRepoRuleValue) env.getValue(key);
+
+      if (env.valuesMissing()) {
         return null;
       }
-    } catch (NoSuchRepositoryException e) {
-      return RepositoryDirectoryValue.NO_SUCH_REPOSITORY_VALUE;
+
+      if (value != BzlmodRepoRuleValue.REPO_RULE_NOT_FOUND_VALUE) {
+        rule = value.getRule();
+      }
+    }
+
+    if (rule == null) {
+      // fallback to look up the repository in the WORKSPACE file.
+      try {
+        rule = getRepoRuleFromWorkspace(repositoryName, env);
+        if (env.valuesMissing()) {
+          return null;
+        }
+      } catch (NoSuchRepositoryException e) {
+        return RepositoryDirectoryValue.NO_SUCH_REPOSITORY_VALUE;
+      }
     }
 
     RepositoryFunction handler = getHandler(rule);
@@ -405,7 +427,7 @@ private RepositoryDirectoryValue.Builder fetchRepository(
    * returns null.
    */
   @Nullable
-  private Rule getRepository(RepositoryName repositoryName, Environment env)
+  private Rule getRepoRuleFromWorkspace(RepositoryName repositoryName, Environment env)
       throws InterruptedException, RepositoryFunctionException, NoSuchRepositoryException {
     try {
       return externalPackageHelper.getRuleByName(repositoryName.strippedName(), env);

src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java

@@ -132,6 +132,12 @@ private BzlmodRepoRuleValue createRuleFromSpec(
   private BzlmodRepoRuleValue createNativeRepoRule(
       RepoSpec repoSpec, StarlarkSemantics semantics, Environment env)
       throws InterruptedException, BzlmodRepoRuleFunctionException {
+    if (!ruleClassProvider.getRuleClassMap().containsKey(repoSpec.ruleClassName())) {
+      InvalidRuleException e =
+          new InvalidRuleException(
+              "Unrecognized native repository rule: " + repoSpec.getRuleClass());
+      throw new BzlmodRepoRuleFunctionException(e, Transience.PERSISTENT);
+    }
     RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(repoSpec.ruleClassName());
     BuildLangTypedAttributeValuesMap attributeValues =
         new BuildLangTypedAttributeValuesMap(repoSpec.attributes());

src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java

@@ -132,7 +132,8 @@ public BazelPackageLoader buildImpl() {
               RepositoryDelegatorFunction.RESOLVED_FILE_INSTEAD_OF_WORKSPACE, Optional.empty()),
           PrecomputedValue.injected(
               RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING,
-              RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY));
+              RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY),
+          PrecomputedValue.injected(RepositoryDelegatorFunction.ENABLE_BZLMOD, false));
 
       return new BazelPackageLoader(this);
     }

src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java

@@ -237,6 +237,7 @@ private void reinitializeSkyframeExecutor() {
             PrecomputedValue.injected(
                 RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING,
                 RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY),
+            PrecomputedValue.injected(RepositoryDelegatorFunction.ENABLE_BZLMOD, false),
             PrecomputedValue.injected(
                 BuildInfoCollectionFunction.BUILD_INFO_FACTORIES,
                 ruleClassProvider.getBuildInfoFactoriesAsMap())));

src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java

@@ -298,6 +298,7 @@ public void initializeSkyframeExecutor(boolean doPackageLoadingChecks) throws Ex
             PrecomputedValue.injected(
                 RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING,
                 RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY),
+            PrecomputedValue.injected(RepositoryDelegatorFunction.ENABLE_BZLMOD, false),
             PrecomputedValue.injected(
                 BuildInfoCollectionFunction.BUILD_INFO_FACTORIES,
                 ruleClassProvider.getBuildInfoFactoriesAsMap()));

src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java

@@ -161,6 +161,7 @@ public final void initializeSkyframeExecutor() throws Exception {
             PrecomputedValue.injected(
                 RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING,
                 RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY),
+            PrecomputedValue.injected(RepositoryDelegatorFunction.ENABLE_BZLMOD, false),
             PrecomputedValue.injected(
                 BuildInfoCollectionFunction.BUILD_INFO_FACTORIES,
                 ruleClassProvider.getBuildInfoFactoriesAsMap())));

src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD

@@ -16,8 +16,15 @@ filegroup(
 
 java_library(
     name = "BzlmodTests_lib",
-    srcs = glob(["*.java"]),
+    srcs = glob(
+        ["*.java"],
+        exclude = [
+            "FakeRegistry.java",
+            "BzlmodTestUtil.java",
+        ],
+    ),
     deps = [
+        ":util",
         "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
         "//src/main/java/com/google/devtools/build/lib/actions:thread_state_receiver",
         "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
@@ -30,7 +37,6 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark",
-        "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/packages",
         "//src/main/java/com/google/devtools/build/lib/pkgcache",
         "//src/main/java/com/google/devtools/build/lib/skyframe:bzl_compile",
@@ -68,3 +74,17 @@ java_test(
         "//src/test/java/com/google/devtools/build/lib:test_runner",
     ],
 )
+
+java_library(
+    name = "util",
+    srcs = [
+        "BzlmodTestUtil.java",
+        "FakeRegistry.java",
+    ],
+    deps = [
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:registry",
+        "//src/main/java/com/google/devtools/build/lib/events",
+        "//third_party:guava",
+    ],
+)

src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java

@@ -335,7 +335,8 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru
                 RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, ImmutableMap.of()),
             PrecomputedValue.injected(
                 RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING,
-                RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY)));
+                RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY),
+            PrecomputedValue.injected(RepositoryDelegatorFunction.ENABLE_BZLMOD, false)));
     SkyframeExecutorTestHelper.process(skyframeExecutor);
     return skyframeExecutor;
   }

src/test/java/com/google/devtools/build/lib/rules/repository/BUILD

@@ -21,6 +21,9 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
         "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
         "//src/main/java/com/google/devtools/build/lib/analysis:server_directories",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_helper",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_value",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
@@ -48,6 +51,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
         "//src/main/java/net/starlark/java/eval",
         "//src/test/java/com/google/devtools/build/lib/analysis/util",
+        "//src/test/java/com/google/devtools/build/lib/bazel/bzlmod:util",
         "//src/test/java/com/google/devtools/build/lib/testutil",
         "//src/test/java/com/google/devtools/build/lib/testutil:TestConstants",
         "//third_party:caffeine",

src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java

@@ -17,6 +17,7 @@
 
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.createModuleKey;
 
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.google.common.collect.ImmutableList;
@@ -30,20 +31,29 @@
 import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
 import com.google.devtools.build.lib.analysis.ServerDirectories;
 import com.google.devtools.build.lib.analysis.util.AnalysisMock;
+import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleHelperImpl;
+import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue;
+import com.google.devtools.build.lib.bazel.bzlmod.DiscoveryFunction;
+import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry;
+import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction;
+import com.google.devtools.build.lib.bazel.bzlmod.SelectionFunction;
 import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
 import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryFunction;
 import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.events.StoredEventHandler;
 import com.google.devtools.build.lib.packages.PackageFactory;
 import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleFactory.InvalidRuleException;
 import com.google.devtools.build.lib.packages.WorkspaceFileValue;
 import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
 import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue.SuccessfulRepositoryDirectoryValue;
 import com.google.devtools.build.lib.rules.repository.RepositoryFunction.AlreadyReportedRepositoryAccessException;
+import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
 import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants;
 import com.google.devtools.build.lib.skyframe.BzlCompileFunction;
 import com.google.devtools.build.lib.skyframe.BzlLoadFunction;
+import com.google.devtools.build.lib.skyframe.BzlmodRepoRuleFunction;
 import com.google.devtools.build.lib.skyframe.ContainingPackageLookupFunction;
 import com.google.devtools.build.lib.skyframe.ExternalFilesHelper;
 import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction;
@@ -106,10 +116,13 @@ public class RepositoryDelegatorTest extends FoundationTestCase {
   private RecordingDifferencer differencer;
   private TestStarlarkRepositoryFunction testStarlarkRepositoryFunction;
   private Path rootPath;
+  private FakeRegistry.Factory registryFactory;
 
   @Before
   public void setupDelegator() throws Exception {
     rootPath = scratch.dir("/outputbase");
+    scratch.file(
+        rootPath.getRelative("MODULE.bazel").getPathString(), "module(name='test',version='0.1')");
     BlazeDirectories directories =
         new BlazeDirectories(
             new ServerDirectories(rootPath, rootPath, rootPath),
@@ -127,8 +140,8 @@ public void setupDelegator() throws Exception {
         new RepositoryDelegatorFunction(
             repositoryHandlers,
             testStarlarkRepositoryFunction,
-            new AtomicBoolean(true),
-            ImmutableMap::of,
+            /* isFetch= */ new AtomicBoolean(true),
+            /* clientEnvironmentSupplier= */ ImmutableMap::of,
             directories,
             managedDirectoriesKnowledge,
             BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER);
@@ -157,6 +170,8 @@ public void setupDelegator() throws Exception {
             .getPackageFactoryBuilderForTesting(directories)
             .build(ruleClassProvider, fileSystem);
 
+    registryFactory = new FakeRegistry.Factory();
+
     HashFunction hashFunction = fileSystem.getDigestFunction().getHashFunction();
     MemoizingEvaluator evaluator =
         new InMemoryMemoizingEvaluator(
@@ -218,6 +233,13 @@ public void setupDelegator() throws Exception {
                     new IgnoredPackagePrefixesFunction(
                         /*ignoredPackagePrefixesFile=*/ PathFragment.EMPTY_FRAGMENT))
                 .put(SkyFunctions.RESOLVED_HASH_VALUES, new ResolvedHashesFunction())
+                .put(SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootPath))
+                .put(SkyFunctions.DISCOVERY, new DiscoveryFunction())
+                .put(SkyFunctions.SELECTION, new SelectionFunction())
+                .put(
+                    BzlmodRepoRuleValue.BZLMOD_REPO_RULE,
+                    new BzlmodRepoRuleFunction(
+                        pkgFactory, ruleClassProvider, directories, new BzlmodRepoRuleHelperImpl()))
                 .build(),
             differencer);
     driver = new SequentialBuildDriver(evaluator);
@@ -234,6 +256,7 @@ public void setupDelegator() throws Exception {
     RepositoryDelegatorFunction.OUTPUT_VERIFICATION_REPOSITORY_RULES.set(
         differencer, ImmutableSet.of());
     RepositoryDelegatorFunction.RESOLVED_FILE_FOR_VERIFICATION.set(differencer, Optional.empty());
+    RepositoryDelegatorFunction.ENABLE_BZLMOD.set(differencer, true);
   }
 
   @Test
@@ -417,6 +440,51 @@ public void testFetchRepositoryException_eventHandled() throws Exception {
     assertThat(eventHandler.getEvents()).hasSize(1);
   }
 
+  @Test
+  public void loadRepositoryFromBzlmod() throws Exception {
+    scratch.overwriteFile(
+        rootPath.getRelative("MODULE.bazel").getPathString(),
+        "module(name='A',version='0.1')",
+        "bazel_dep(name='B',version='1.0')");
+    FakeRegistry registry =
+        registryFactory
+            .newFakeRegistry()
+            .addModule(createModuleKey("B", "1.0"), "module(name='B', version='1.0');");
+    ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+    scratch.file(rootPath.getRelative("BUILD").getPathString());
+    scratch.file(
+        rootPath.getRelative("repo_rule.bzl").getPathString(),
+        "def _impl(rctx):",
+        " rctx.file('BUILD', '')",
+        "fictive_repo_rule = repository_rule(implementation = _impl)");
+    scratch.file(
+        rootPath.getRelative("WORKSPACE").getPathString(),
+        "load(':repo_rule.bzl', 'fictive_repo_rule')",
+        "fictive_repo_rule(name = 'B')",
+        "fictive_repo_rule(name = 'C')");
+
+    StoredEventHandler eventHandler = new StoredEventHandler();
+    SkyKey key = RepositoryDirectoryValue.key(RepositoryName.createFromValidStrippedName("B"));
+    EvaluationContext evaluationContext =
+        EvaluationContext.newBuilder()
+            .setKeepGoing(false)
+            .setNumThreads(8)
+            .setEventHandler(eventHandler)
+            .build();
+    EvaluationResult<SkyValue> result = driver.evaluate(ImmutableList.of(key), evaluationContext);
+
+    // B should be fetched from MODULE.bazel file instead of WORKSPACE file.
+    // Because FakeRegistry returns a fake_http_archive_rule, the fetch should fail as expected.
+    assertThat(result.hasError()).isTrue();
+    assertThat(result.getError().getException()).isInstanceOf(InvalidRuleException.class);
+    assertThat(result.getError().getException())
+        .hasMessageThat()
+        .isEqualTo("Unrecognized native repository rule: fake_http_archive_rule");
+
+    // C should still be fetched from WORKSPACE successfully.
+    loadRepo("C");
+  }
+
   private void loadRepo(String strippedRepoName) throws InterruptedException {
     StoredEventHandler eventHandler = new StoredEventHandler();
     SkyKey key =

src/test/java/com/google/devtools/build/lib/skyframe/AbstractCollectPackagesUnderDirectoryTest.java

@@ -305,7 +305,8 @@ private void initBuildDriver() throws AbruptExitException, InterruptedException,
                 RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, ImmutableMap.of()),
             PrecomputedValue.injected(
                 RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING,
-                RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY)));
+                RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY),
+            PrecomputedValue.injected(RepositoryDelegatorFunction.ENABLE_BZLMOD, false)));
     skyframeExecutor.sync(
         reporter,
         packageOptions,