[7.1.0] Add bazel mod dump_repo_mapping (#21023)

`bazel mod dump_repo_mapping` with no arguments is explicitly made an error so that a new mode that dumps all repository mappings with a single Bazel invocation can be added later if needed, e.g. to support IntelliJ's "sync" workflow. RELNOTES: `bazel mod dump_repo_mapping <canonical repo name>...` returns the repository mappings of the given repositories in NDJSON. This information can be used by IDEs and Starlark language servers to resolve labels with `--enable_bzlmod`. Work towards #20631 Closes #20686. Commit 59ac9ce PiperOrigin-RevId: 601332180 Change-Id: I828d7c88637bea175e11eccc52c6202f6da4c32c Co-authored-by: Fabian Meumertzheim <[email protected]>
bazelbuild · Jan 26, 2024 · 5202a05 · 5202a05
1 parent d0b4c39
commit 5202a05
Show file tree

Hide file tree

Showing 5 changed files with 178 additions and 24 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java
@@ -172,7 +172,8 @@ public enum ModSubcommand {
     PATH(true),
     EXPLAIN(true),
     SHOW_REPO(false),
-    SHOW_EXTENSION(false);
+    SHOW_EXTENSION(false),
+    DUMP_REPO_MAPPING(false);
 
     /** Whether this subcommand produces a graph output. */
     private final boolean isGraph;

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD
@@ -65,6 +65,7 @@ java_library(
         "//src/main/java/com/google/devtools/common/options",
         "//src/main/java/net/starlark/java/eval",
         "//src/main/protobuf:failure_details_java_proto",
+        "//third_party:gson",
         "//third_party:guava",
     ],
 )
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java
@@ -13,6 +13,7 @@
 // limitations under the License.
 package com.google.devtools.build.lib.bazel.commands;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.Charset.UTF8;
@@ -44,6 +45,7 @@
 import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.ModSubcommandConverter;
 import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg;
 import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg.ModuleArgConverter;
+import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
 import com.google.devtools.build.lib.cmdline.RepositoryMapping;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.events.Event;
@@ -57,6 +59,7 @@
 import com.google.devtools.build.lib.server.FailureDetails;
 import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
 import com.google.devtools.build.lib.server.FailureDetails.ModCommand.Code;
+import com.google.devtools.build.lib.skyframe.RepositoryMappingValue;
 import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
 import com.google.devtools.build.lib.util.AbruptExitException;
 import com.google.devtools.build.lib.util.DetailedExitCode;
@@ -70,11 +73,17 @@
 import com.google.devtools.common.options.OptionsParser;
 import com.google.devtools.common.options.OptionsParsingException;
 import com.google.devtools.common.options.OptionsParsingResult;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.stream.JsonWriter;
+import java.io.IOException;
 import java.io.OutputStreamWriter;
+import java.io.Writer;
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.stream.IntStream;
 
 /** Queries the Bzlmod external dependency graph. */
 @Command(
@@ -125,8 +134,51 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti
   }
 
   private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingResult options) {
+    ModOptions modOptions = options.getOptions(ModOptions.class);
+    Preconditions.checkArgument(modOptions != null);
+
+    if (options.getResidue().isEmpty()) {
+      String errorMessage =
+          String.format(
+              "No subcommand specified, choose one of : %s.", ModSubcommand.printValues());
+      return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN);
+    }
+
+    // The first element in the residue must be the subcommand, and then comes a list of arguments.
+    String subcommandStr = options.getResidue().get(0);
+    ModSubcommand subcommand;
+    try {
+      subcommand = new ModSubcommandConverter().convert(subcommandStr);
+    } catch (OptionsParsingException e) {
+      String errorMessage =
+          String.format("Invalid subcommand, choose one from : %s.", ModSubcommand.printValues());
+      return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN);
+    }
+    List<String> args = options.getResidue().subList(1, options.getResidue().size());
+
+    ImmutableList.Builder<RepositoryMappingValue.Key> repositoryMappingKeysBuilder =
+        ImmutableList.builder();
+    if (subcommand.equals(ModSubcommand.DUMP_REPO_MAPPING)) {
+      if (args.isEmpty()) {
+        // Make this case an error so that we are free to add a mode that emits all mappings in a
+        // single JSON object later.
+        return reportAndCreateFailureResult(
+            env, "No repository name(s) specified", Code.INVALID_ARGUMENTS);
+      }
+      for (String arg : args) {
+        try {
+          repositoryMappingKeysBuilder.add(RepositoryMappingValue.key(RepositoryName.create(arg)));
+        } catch (LabelSyntaxException e) {
+          return reportAndCreateFailureResult(env, e.getMessage(), Code.INVALID_ARGUMENTS);
+        }
+      }
+    }
+    ImmutableList<RepositoryMappingValue.Key> repoMappingKeys =
+        repositoryMappingKeysBuilder.build();
+
     BazelDepGraphValue depGraphValue;
     BazelModuleInspectorValue moduleInspector;
+    ImmutableList<RepositoryMappingValue> repoMappingValues;
 
     SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor();
     LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class);
@@ -140,10 +192,14 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe
     try {
       env.syncPackageLoading(options);
 
+      ImmutableSet.Builder<SkyKey> keys = ImmutableSet.builder();
+      if (subcommand.equals(ModSubcommand.DUMP_REPO_MAPPING)) {
+        keys.addAll(repoMappingKeys);
+      } else {
+        keys.add(BazelDepGraphValue.KEY, BazelModuleInspectorValue.KEY);
+      }
       EvaluationResult<SkyValue> evaluationResult =
-          skyframeExecutor.prepareAndGet(
-              ImmutableSet.of(BazelDepGraphValue.KEY, BazelModuleInspectorValue.KEY),
-              evaluationContext);
+          skyframeExecutor.prepareAndGet(keys.build(), evaluationContext);
 
       if (evaluationResult.hasError()) {
         Exception e = evaluationResult.getError().getException();
@@ -159,6 +215,11 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe
       moduleInspector =
           (BazelModuleInspectorValue) evaluationResult.get(BazelModuleInspectorValue.KEY);
 
+      repoMappingValues =
+          repoMappingKeys.stream()
+              .map(evaluationResult::get)
+              .map(RepositoryMappingValue.class::cast)
+              .collect(toImmutableList());
     } catch (InterruptedException e) {
       String errorMessage = "mod command interrupted: " + e.getMessage();
       env.getReporter().handle(Event.error(errorMessage));
@@ -169,27 +230,29 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe
       return BlazeCommandResult.detailedExitCode(e.getDetailedExitCode());
     }
 
-    ModOptions modOptions = options.getOptions(ModOptions.class);
-    Preconditions.checkArgument(modOptions != null);
-
-    if (options.getResidue().isEmpty()) {
-      String errorMessage =
-          String.format(
-              "No subcommand specified, choose one of : %s.", ModSubcommand.printValues());
-      return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN);
-    }
-
-    // The first element in the residue must be the subcommand, and then comes a list of arguments.
-    String subcommandStr = options.getResidue().get(0);
-    ModSubcommand subcommand;
-    try {
-      subcommand = new ModSubcommandConverter().convert(subcommandStr);
-    } catch (OptionsParsingException e) {
-      String errorMessage =
-          String.format("Invalid subcommand, choose one from : %s.", ModSubcommand.printValues());
-      return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN);
+    if (subcommand.equals(ModSubcommand.DUMP_REPO_MAPPING)) {
+      String missingRepos =
+          IntStream.range(0, repoMappingKeys.size())
+              .filter(i -> repoMappingValues.get(i) == RepositoryMappingValue.NOT_FOUND_VALUE)
+              .mapToObj(repoMappingKeys::get)
+              .map(RepositoryMappingValue.Key::repoName)
+              .map(RepositoryName::getName)
+              .collect(joining(", "));
+      if (!missingRepos.isEmpty()) {
+        return reportAndCreateFailureResult(
+            env, "Repositories not found: " + missingRepos, Code.INVALID_ARGUMENTS);
+      }
+      try {
+        dumpRepoMappings(
+            repoMappingValues,
+            new OutputStreamWriter(
+                env.getReporter().getOutErr().getOutputStream(),
+                modOptions.charset == UTF8 ? UTF_8 : US_ASCII));
+      } catch (IOException e) {
+        throw new IllegalStateException(e);
+      }
+      return BlazeCommandResult.success();
     }
-    List<String> args = options.getResidue().subList(1, options.getResidue().size());
 
     // Extract and check the --base_module argument first to use it when parsing the other args.
     // Can only be a TargetModule or a repoName relative to the ROOT.
@@ -453,6 +516,8 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe
       case SHOW_EXTENSION:
         modExecutor.showExtension(argsAsExtensions, usageKeys);
         break;
+      default:
+        throw new IllegalStateException("Unexpected subcommand: " + subcommand);
     }
 
     return BlazeCommandResult.success();
@@ -510,4 +575,21 @@ private static BlazeCommandResult createFailureResult(String message, Code detai
                 .setMessage(message)
                 .build()));
   }
+
+  public static void dumpRepoMappings(List<RepositoryMappingValue> repoMappings, Writer writer)
+      throws IOException {
+    Gson gson = new GsonBuilder().disableHtmlEscaping().create();
+    for (RepositoryMappingValue repoMapping : repoMappings) {
+      JsonWriter jsonWriter = gson.newJsonWriter(writer);
+      jsonWriter.beginObject();
+      for (Entry<String, RepositoryName> entry :
+          repoMapping.getRepositoryMapping().entries().entrySet()) {
+        jsonWriter.name(entry.getKey());
+        jsonWriter.value(entry.getValue().getName());
+      }
+      jsonWriter.endObject();
+      writer.write('\n');
+    }
+    writer.flush();
+  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt b/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt
@@ -12,6 +12,7 @@ The command will display the external dependency graph or parts thereof, structu
     - explain <module>...: Prints all the places where the module is (or was) requested as a direct dependency, along with the reason why the respective final version was selected. It will display a pruned version of the `all_paths <module>...` command which only contains the direct deps of the root, the <module(s)> leaves, along with their dependants (can be modified with --depth).
     - show_repo <module>...: Prints the rule that generated the specified repos (i.e. http_archive()). The arguments may refer to extension-generated repos.
     - show_extension <extension>...: Prints information about the given extension(s). Usages can be filtered down to only those from modules in --extension_usage.
+    - dump_repo_mapping <canonical_repo_name>...: Prints the mappings from apparent repo names to canonical repo names for the given repos in NDJSON format. The order of entries within each JSON object is unspecified. This command is intended for use by tools such as IDEs and Starlark language servers.
 
 
 <module> arguments must be one of the following:
@@ -25,4 +26,6 @@ The command will display the external dependency graph or parts thereof, structu
 
 <extension> arguments must be of the form <module><label_to_bzl_file>%<extension_name>. For example, both rules_java//java:extensions.bzl%toolchains and @rules_java//java:extensions.bzl%toolchains are valid specifications of extensions.
 
+<canonical_repo_name> arguments are canonical repo names without any leading @ characters. The canonical repo name of the root module repository is the empty string.
+
 %{options}
diff --git a/src/test/py/bazel/bzlmod/mod_command_test.py b/src/test/py/bazel/bzlmod/mod_command_test.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 """Tests the mod command."""
 
+import json
 import os
 import tempfile
 from absl.testing import absltest
@@ -454,6 +455,72 @@ def testShowRepoThrowsUnusedModule(self):
         stderr,
     )
 
+  def testDumpRepoMapping(self):
+    _, stdout, _ = self.RunBazel(
+        [
+            'mod',
+            'dump_repo_mapping',
+            '',
+            'foo~2.0',
+        ],
+    )
+    root_mapping, foo_mapping = [json.loads(l) for l in stdout]
+
+    self.assertContainsSubset(
+        {
+            'my_project': '',
+            'foo1': 'foo~1.0',
+            'foo2': 'foo~2.0',
+            'myrepo2': 'ext2~1.0~ext~repo1',
+            'bazel_tools': 'bazel_tools',
+        }.items(),
+        root_mapping.items(),
+    )
+
+    self.assertContainsSubset(
+        {
+            'foo': 'foo~2.0',
+            'ext_mod': 'ext~1.0',
+            'my_repo3': 'ext~1.0~ext~repo3',
+            'bazel_tools': 'bazel_tools',
+        }.items(),
+        foo_mapping.items(),
+    )
+
+  def testDumpRepoMappingThrowsNoRepos(self):
+    _, _, stderr = self.RunBazel(
+        ['mod', 'dump_repo_mapping'],
+        allow_failure=True,
+    )
+    self.assertIn(
+        "ERROR: No repository name(s) specified. Type 'bazel help mod' for"
+        ' syntax and help.',
+        stderr,
+    )
+
+  def testDumpRepoMappingThrowsInvalidRepoName(self):
+    _, _, stderr = self.RunBazel(
+        ['mod', 'dump_repo_mapping', '{}'],
+        allow_failure=True,
+    )
+    self.assertIn(
+        "ERROR: invalid repository name '{}': repo names may contain only A-Z,"
+        " a-z, 0-9, '-', '_', '.' and '~' and must not start with '~'. Type"
+        " 'bazel help mod' for syntax and help.",
+        stderr,
+    )
+
+  def testDumpRepoMappingThrowsUnknownRepoName(self):
+    _, _, stderr = self.RunBazel(
+        ['mod', 'dump_repo_mapping', 'does_not_exist'],
+        allow_failure=True,
+    )
+    self.assertIn(
+        "ERROR: Repositories not found: does_not_exist. Type 'bazel help mod'"
+        ' for syntax and help.',
+        stderr,
+    )
+
 
 if __name__ == '__main__':
   absltest.main()