Skip to content

Commit

Permalink
Add bazel mod dump_repo_mapping
Browse files Browse the repository at this point in the history
`bazel mod dump_repo_mapping` with no arguments is explicitly made an
error so that a new mode that dumps all repository mappings with a
single Bazel invocation can be added later if needed, e.g. to support
IntelliJ's "sync" workflow.

RELNOTES: `bazel mod dump_repo_mapping <canonical repo name>...` returns
the repository mappings of the given repositories in NDJSON. This
information can be used by IDEs and Starlark language servers to resolve
labels with `--enable_bzlmod`.

Work towards #20631

Closes #20686.

PiperOrigin-RevId: 601332180
Change-Id: I828d7c88637bea175e11eccc52c6202f6da4c32c
  • Loading branch information
fmeum authored and bazel-io committed Jan 25, 2024
1 parent fa0660a commit f15b13d
Show file tree
Hide file tree
Showing 5 changed files with 178 additions and 24 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,8 @@ public enum ModSubcommand {
PATH(true),
EXPLAIN(true),
SHOW_REPO(false),
SHOW_EXTENSION(false);
SHOW_EXTENSION(false),
DUMP_REPO_MAPPING(false);

/** Whether this subcommand produces a graph output. */
private final boolean isGraph;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ java_library(
"//src/main/java/com/google/devtools/common/options",
"//src/main/java/net/starlark/java/eval",
"//src/main/protobuf:failure_details_java_proto",
"//third_party:gson",
"//third_party:guava",
],
)
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
// limitations under the License.
package com.google.devtools.build.lib.bazel.commands;

import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.Charset.UTF8;
Expand Down Expand Up @@ -44,6 +45,7 @@
import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.ModSubcommandConverter;
import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg;
import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg.ModuleArgConverter;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.RepositoryMapping;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.events.Event;
Expand All @@ -57,6 +59,7 @@
import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.ModCommand.Code;
import com.google.devtools.build.lib.skyframe.RepositoryMappingValue;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.DetailedExitCode;
Expand All @@ -70,11 +73,17 @@
import com.google.devtools.common.options.OptionsParser;
import com.google.devtools.common.options.OptionsParsingException;
import com.google.devtools.common.options.OptionsParsingResult;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.stream.JsonWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.List;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.IntStream;

/** Queries the Bzlmod external dependency graph. */
@Command(
Expand Down Expand Up @@ -125,8 +134,51 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti
}

private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingResult options) {
ModOptions modOptions = options.getOptions(ModOptions.class);
Preconditions.checkArgument(modOptions != null);

if (options.getResidue().isEmpty()) {
String errorMessage =
String.format(
"No subcommand specified, choose one of : %s.", ModSubcommand.printValues());
return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN);
}

// The first element in the residue must be the subcommand, and then comes a list of arguments.
String subcommandStr = options.getResidue().get(0);
ModSubcommand subcommand;
try {
subcommand = new ModSubcommandConverter().convert(subcommandStr);
} catch (OptionsParsingException e) {
String errorMessage =
String.format("Invalid subcommand, choose one from : %s.", ModSubcommand.printValues());
return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN);
}
List<String> args = options.getResidue().subList(1, options.getResidue().size());

ImmutableList.Builder<RepositoryMappingValue.Key> repositoryMappingKeysBuilder =
ImmutableList.builder();
if (subcommand.equals(ModSubcommand.DUMP_REPO_MAPPING)) {
if (args.isEmpty()) {
// Make this case an error so that we are free to add a mode that emits all mappings in a
// single JSON object later.
return reportAndCreateFailureResult(
env, "No repository name(s) specified", Code.INVALID_ARGUMENTS);
}
for (String arg : args) {
try {
repositoryMappingKeysBuilder.add(RepositoryMappingValue.key(RepositoryName.create(arg)));
} catch (LabelSyntaxException e) {
return reportAndCreateFailureResult(env, e.getMessage(), Code.INVALID_ARGUMENTS);
}
}
}
ImmutableList<RepositoryMappingValue.Key> repoMappingKeys =
repositoryMappingKeysBuilder.build();

BazelDepGraphValue depGraphValue;
BazelModuleInspectorValue moduleInspector;
ImmutableList<RepositoryMappingValue> repoMappingValues;

SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor();
LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class);
Expand All @@ -140,10 +192,14 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe
try {
env.syncPackageLoading(options);

ImmutableSet.Builder<SkyKey> keys = ImmutableSet.builder();
if (subcommand.equals(ModSubcommand.DUMP_REPO_MAPPING)) {
keys.addAll(repoMappingKeys);
} else {
keys.add(BazelDepGraphValue.KEY, BazelModuleInspectorValue.KEY);
}
EvaluationResult<SkyValue> evaluationResult =
skyframeExecutor.prepareAndGet(
ImmutableSet.of(BazelDepGraphValue.KEY, BazelModuleInspectorValue.KEY),
evaluationContext);
skyframeExecutor.prepareAndGet(keys.build(), evaluationContext);

if (evaluationResult.hasError()) {
Exception e = evaluationResult.getError().getException();
Expand All @@ -159,6 +215,11 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe
moduleInspector =
(BazelModuleInspectorValue) evaluationResult.get(BazelModuleInspectorValue.KEY);

repoMappingValues =
repoMappingKeys.stream()
.map(evaluationResult::get)
.map(RepositoryMappingValue.class::cast)
.collect(toImmutableList());
} catch (InterruptedException e) {
String errorMessage = "mod command interrupted: " + e.getMessage();
env.getReporter().handle(Event.error(errorMessage));
Expand All @@ -169,27 +230,29 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe
return BlazeCommandResult.detailedExitCode(e.getDetailedExitCode());
}

ModOptions modOptions = options.getOptions(ModOptions.class);
Preconditions.checkArgument(modOptions != null);

if (options.getResidue().isEmpty()) {
String errorMessage =
String.format(
"No subcommand specified, choose one of : %s.", ModSubcommand.printValues());
return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN);
}

// The first element in the residue must be the subcommand, and then comes a list of arguments.
String subcommandStr = options.getResidue().get(0);
ModSubcommand subcommand;
try {
subcommand = new ModSubcommandConverter().convert(subcommandStr);
} catch (OptionsParsingException e) {
String errorMessage =
String.format("Invalid subcommand, choose one from : %s.", ModSubcommand.printValues());
return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN);
if (subcommand.equals(ModSubcommand.DUMP_REPO_MAPPING)) {
String missingRepos =
IntStream.range(0, repoMappingKeys.size())
.filter(i -> repoMappingValues.get(i) == RepositoryMappingValue.NOT_FOUND_VALUE)
.mapToObj(repoMappingKeys::get)
.map(RepositoryMappingValue.Key::repoName)
.map(RepositoryName::getName)
.collect(joining(", "));
if (!missingRepos.isEmpty()) {
return reportAndCreateFailureResult(
env, "Repositories not found: " + missingRepos, Code.INVALID_ARGUMENTS);
}
try {
dumpRepoMappings(
repoMappingValues,
new OutputStreamWriter(
env.getReporter().getOutErr().getOutputStream(),
modOptions.charset == UTF8 ? UTF_8 : US_ASCII));
} catch (IOException e) {
throw new IllegalStateException(e);
}
return BlazeCommandResult.success();
}
List<String> args = options.getResidue().subList(1, options.getResidue().size());

// Extract and check the --base_module argument first to use it when parsing the other args.
// Can only be a TargetModule or a repoName relative to the ROOT.
Expand Down Expand Up @@ -453,6 +516,8 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe
case SHOW_EXTENSION:
modExecutor.showExtension(argsAsExtensions, usageKeys);
break;
default:
throw new IllegalStateException("Unexpected subcommand: " + subcommand);
}

return BlazeCommandResult.success();
Expand Down Expand Up @@ -510,4 +575,21 @@ private static BlazeCommandResult createFailureResult(String message, Code detai
.setMessage(message)
.build()));
}

public static void dumpRepoMappings(List<RepositoryMappingValue> repoMappings, Writer writer)
throws IOException {
Gson gson = new GsonBuilder().disableHtmlEscaping().create();
for (RepositoryMappingValue repoMapping : repoMappings) {
JsonWriter jsonWriter = gson.newJsonWriter(writer);
jsonWriter.beginObject();
for (Entry<String, RepositoryName> entry :
repoMapping.getRepositoryMapping().entries().entrySet()) {
jsonWriter.name(entry.getKey());
jsonWriter.value(entry.getValue().getName());
}
jsonWriter.endObject();
writer.write('\n');
}
writer.flush();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ The command will display the external dependency graph or parts thereof, structu
- explain <module>...: Prints all the places where the module is (or was) requested as a direct dependency, along with the reason why the respective final version was selected. It will display a pruned version of the `all_paths <module>...` command which only contains the direct deps of the root, the <module(s)> leaves, along with their dependants (can be modified with --depth).
- show_repo <module>...: Prints the rule that generated the specified repos (i.e. http_archive()). The arguments may refer to extension-generated repos.
- show_extension <extension>...: Prints information about the given extension(s). Usages can be filtered down to only those from modules in --extension_usage.
- dump_repo_mapping <canonical_repo_name>...: Prints the mappings from apparent repo names to canonical repo names for the given repos in NDJSON format. The order of entries within each JSON object is unspecified. This command is intended for use by tools such as IDEs and Starlark language servers.


<module> arguments must be one of the following:
Expand All @@ -25,4 +26,6 @@ The command will display the external dependency graph or parts thereof, structu

<extension> arguments must be of the form <module><label_to_bzl_file>%<extension_name>. For example, both rules_java//java:extensions.bzl%toolchains and @rules_java//java:extensions.bzl%toolchains are valid specifications of extensions.

<canonical_repo_name> arguments are canonical repo names without any leading @ characters. The canonical repo name of the root module repository is the empty string.

%{options}
67 changes: 67 additions & 0 deletions src/test/py/bazel/bzlmod/mod_command_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# limitations under the License.
"""Tests the mod command."""

import json
import os
import tempfile
from absl.testing import absltest
Expand Down Expand Up @@ -454,6 +455,72 @@ def testShowRepoThrowsUnusedModule(self):
stderr,
)

def testDumpRepoMapping(self):
_, stdout, _ = self.RunBazel(
[
'mod',
'dump_repo_mapping',
'',
'foo~2.0',
],
)
root_mapping, foo_mapping = [json.loads(l) for l in stdout]

self.assertContainsSubset(
{
'my_project': '',
'foo1': 'foo~1.0',
'foo2': 'foo~2.0',
'myrepo2': 'ext2~1.0~ext~repo1',
'bazel_tools': 'bazel_tools',
}.items(),
root_mapping.items(),
)

self.assertContainsSubset(
{
'foo': 'foo~2.0',
'ext_mod': 'ext~1.0',
'my_repo3': 'ext~1.0~ext~repo3',
'bazel_tools': 'bazel_tools',
}.items(),
foo_mapping.items(),
)

def testDumpRepoMappingThrowsNoRepos(self):
_, _, stderr = self.RunBazel(
['mod', 'dump_repo_mapping'],
allow_failure=True,
)
self.assertIn(
"ERROR: No repository name(s) specified. Type 'bazel help mod' for"
' syntax and help.',
stderr,
)

def testDumpRepoMappingThrowsInvalidRepoName(self):
_, _, stderr = self.RunBazel(
['mod', 'dump_repo_mapping', '{}'],
allow_failure=True,
)
self.assertIn(
"ERROR: invalid repository name '{}': repo names may contain only A-Z,"
" a-z, 0-9, '-', '_', '.' and '~' and must not start with '~'. Type"
" 'bazel help mod' for syntax and help.",
stderr,
)

def testDumpRepoMappingThrowsUnknownRepoName(self):
_, _, stderr = self.RunBazel(
['mod', 'dump_repo_mapping', 'does_not_exist'],
allow_failure=True,
)
self.assertIn(
"ERROR: Repositories not found: does_not_exist. Type 'bazel help mod'"
' for syntax and help.',
stderr,
)


if __name__ == '__main__':
absltest.main()

0 comments on commit f15b13d

Please sign in to comment.