Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@fmeum
fmeum committed Nov 6, 2024
1 parent f8e2be5 commit 273012d
Show file tree
Hide file tree
Showing 9 changed files with 202 additions and 108 deletions.
20 changes: 20 additions & 0 deletions src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ public class BazelRepositoryModule extends BlazeModule {
private final MutableSupplier<Map<String, String>> clientEnvironmentSupplier =
new MutableSupplier<>();
private ImmutableMap<RepositoryName, PathFragment> overrides = ImmutableMap.of();
private ImmutableMap<String, PathFragment> injections = ImmutableMap.of();
private ImmutableMap<String, ModuleOverride> moduleOverrides = ImmutableMap.of();
private Optional<RootedPath> resolvedFileReplacingWorkspace = Optional.empty();
private FileSystem filesystem;
Expand Down Expand Up @@ -470,6 +471,24 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
overrides = ImmutableMap.of();
}

if (repoOptions.repositoryInjections != null) {
Map<String, PathFragment> injectionMap = new LinkedHashMap<>();
for (RepositoryOptions.RepositoryInjection injection : repoOptions.repositoryInjections) {
if (injection.path().isEmpty()) {
injectionMap.remove(injection.apparentName());
continue;
}
String repoPath = getAbsolutePath(injection.path(), env);
injectionMap.put(injection.apparentName(), PathFragment.create(repoPath));
}
ImmutableMap<String, PathFragment> newInjections = ImmutableMap.copyOf(injectionMap);
if (!Maps.difference(injections, newInjections).areEqual()) {
injections = newInjections;
}
} else {
injections = ImmutableMap.of();
}

if (repoOptions.moduleOverrides != null) {
Map<String, ModuleOverride> moduleOverrideMap = new LinkedHashMap<>();
for (RepositoryOptions.ModuleOverride override : repoOptions.moduleOverrides) {
Expand Down Expand Up @@ -605,6 +624,7 @@ public ImmutableList<Injected> getPrecomputedValues() {
}
return ImmutableList.of(
PrecomputedValue.injected(RepositoryMappingFunction.REPOSITORY_OVERRIDES, overrides),
PrecomputedValue.injected(ModuleFileFunction.INJECTED_REPOSITORIES, injections),
PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, moduleOverrides),
PrecomputedValue.injected(
RepositoryDelegatorFunction.RESOLVED_FILE_INSTEAD_OF_WORKSPACE,
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/skyframe:package_lookup_function",
"//src/main/java/com/google/devtools/build/lib/skyframe:package_lookup_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_function",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
Expand Down
35 changes: 35 additions & 0 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
import java.util.Optional;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import net.starlark.java.eval.Dict;
import net.starlark.java.eval.EvalException;
import net.starlark.java.eval.Mutability;
import net.starlark.java.eval.StarlarkSemantics;
Expand All @@ -95,6 +96,8 @@ public class ModuleFileFunction implements SkyFunction {

public static final Precomputed<Map<String, ModuleOverride>> MODULE_OVERRIDES =
new Precomputed<>("module_overrides");
public static final Precomputed<Map<String, PathFragment>> INJECTED_REPOSITORIES =
new Precomputed<>("repository_injections");

private final BazelStarlarkEnvironment starlarkEnv;
private final Path workspaceRoot;
Expand Down Expand Up @@ -194,6 +197,7 @@ public SkyValue compute(SkyKey skyKey, Environment env)
// Dev dependencies should always be ignored if the current module isn't the root module
/* ignoreDevDeps= */ true,
builtinModules,
/* injectedRepositories= */ ImmutableMap.of(),
// Disable printing for modules from registries. We don't want them to be able to spam
// the console during resolution, but module files potentially edited by the user as
// part of a non-registry override should permit printing to aid debugging.
Expand Down Expand Up @@ -287,6 +291,7 @@ private SkyValue computeForRootModule(
ImmutableMap.copyOf(state.includeLabelToCompiledModuleFile),
builtinModules,
MODULE_OVERRIDES.get(env),
INJECTED_REPOSITORIES.get(env),
IGNORE_DEV_DEPS.get(env),
starlarkSemantics,
env.getListener(),
Expand Down Expand Up @@ -420,6 +425,7 @@ public static RootModuleFileValue evaluateRootModuleFile(
ImmutableMap<String, CompiledModuleFile> includeLabelToCompiledModuleFile,
ImmutableMap<String, NonRegistryOverride> builtinModules,
Map<String, ModuleOverride> commandOverrides,
Map<String, PathFragment> injectedRepositories,
boolean ignoreDevDeps,
StarlarkSemantics starlarkSemantics,
ExtendedEventHandler eventHandler,
Expand All @@ -432,6 +438,7 @@ public static RootModuleFileValue evaluateRootModuleFile(
ModuleKey.ROOT,
ignoreDevDeps,
builtinModules,
injectedRepositories,
/* printIsNoop= */ false,
starlarkSemantics,
eventHandler,
Expand Down Expand Up @@ -495,6 +502,7 @@ private static ModuleThreadContext execModuleFile(
ModuleKey moduleKey,
boolean ignoreDevDeps,
ImmutableMap<String, NonRegistryOverride> builtinModules,
Map<String, PathFragment> injectedRepositories,
boolean printIsNoop,
StarlarkSemantics starlarkSemantics,
ExtendedEventHandler eventHandler,
Expand Down Expand Up @@ -524,6 +532,8 @@ private static ModuleThreadContext execModuleFile(
}
}
});

injectRepos(injectedRepositories, context, thread);
compiledRootModuleFile.runOnThread(thread);
} catch (EvalException e) {
eventHandler.handle(Event.error(e.getInnermostLocation(), e.getMessageWithStack()));
Expand All @@ -532,6 +542,31 @@ private static ModuleThreadContext execModuleFile(
return context;
}

// Adds a local_repository for each repository injected via --injected_repositories.
private static void injectRepos(
Map<String, PathFragment> injectedRepositories,
ModuleThreadContext context,
StarlarkThread thread)
throws EvalException {
ModuleThreadContext.ModuleExtensionUsageBuilder newUsageBuilder =
new ModuleThreadContext.ModuleExtensionUsageBuilder(
context,
"//:MODULE.bazel",
"@bazel_tools//tools/build_defs/repo:local.bzl%local_repository",
/* isolate= */ false);
var localRepositoryProxy = new ModuleFileGlobals.RepoRuleProxy(newUsageBuilder);
for (var injectedRepository : injectedRepositories.entrySet()) {
localRepositoryProxy.call(
injectedRepository.getKey(),
/* devDependency= */ true,
Dict.copyOf(
thread.mutability(),
ImmutableMap.of("path", injectedRepository.getValue().getPathString())),
thread);
}
context.getExtensionUsageBuilders().add(newUsageBuilder);
}

/**
* Result of a {@link #getModuleFile} call.
*
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -780,7 +780,8 @@ public RepoRuleProxy useRepoRule(String bzlFile, String ruleName, StarlarkThread
context.setNonModuleCalled();
// Not a valid Starlark identifier so that it can't collide with a real extension.
String extensionName = bzlFile + '%' + ruleName;
// Find or create the builder for the singular "innate" extension of this module.
// Find or create the builder for the singular "innate" extension of this repo rule for this
// module.
for (ModuleExtensionUsageBuilder usageBuilder : context.getExtensionUsageBuilders()) {
if (usageBuilder.isForExtension("//:MODULE.bazel", extensionName)) {
return new RepoRuleProxy(usageBuilder);
Expand All @@ -797,7 +798,7 @@ public RepoRuleProxy useRepoRule(String bzlFile, String ruleName, StarlarkThread
static class RepoRuleProxy implements StarlarkValue {
private final ModuleExtensionUsageBuilder usageBuilder;

private RepoRuleProxy(ModuleExtensionUsageBuilder usageBuilder) {
RepoRuleProxy(ModuleExtensionUsageBuilder usageBuilder) {
this.usageBuilder = usageBuilder;
}

Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/bazel/repository/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/util",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/java/com/google/devtools/common/options",
"//src/main/java/net/starlark/java/eval",
"//third_party:auto_value",
"//third_party:guava",
],
Expand Down
69 changes: 45 additions & 24 deletions src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@

package com.google.devtools.build.lib.bazel.repository;

import com.google.auto.value.AutoValue;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.util.OptionsUtils;
Expand All @@ -28,6 +27,7 @@
import com.google.devtools.common.options.OptionsBase;
import com.google.devtools.common.options.OptionsParsingException;
import java.util.List;
import net.starlark.java.eval.EvalException;

/** Command-line options for repositories. */
public class RepositoryOptions extends OptionsBase {
Expand Down Expand Up @@ -149,6 +149,16 @@ public class RepositoryOptions extends OptionsBase {
+ " previous overrides.")
public List<RepositoryOverride> repositoryOverrides;

@Option(
name = "inject_repository",
defaultValue = "null",
allowMultiple = true,
converter = RepositoryInjectionConverter.class,
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.UNKNOWN},
help = "Foo")
public List<RepositoryInjection> repositoryInjections;

@Option(
name = "override_module",
defaultValue = "null",
Expand Down Expand Up @@ -362,7 +372,7 @@ public RepositoryOverride convert(String input) throws OptionsParsingException {
OptionsUtils.PathFragmentConverter pathConverter = new OptionsUtils.PathFragmentConverter();
String pathString = pathConverter.convert(pieces[1]).getPathString();
try {
return RepositoryOverride.create(RepositoryName.create(pieces[0]), pathString);
return new RepositoryOverride(RepositoryName.create(pieces[0]), pathString);
} catch (LabelSyntaxException e) {
throw new OptionsParsingException("Invalid repository name given to override", input, e);
}
Expand All @@ -374,6 +384,35 @@ public String getTypeDescription() {
}
}

/**
* Converts from an equals-separated pair of strings into RepositoryName->PathFragment mapping.
*/
public static class RepositoryInjectionConverter
extends Converter.Contextless<RepositoryInjection> {

@Override
public RepositoryInjection convert(String input) throws OptionsParsingException {
String[] pieces = input.split("=", 2);
if (pieces.length != 2) {
throw new OptionsParsingException(
"Repository injections must be of the form 'repository-name=path'", input);
}
OptionsUtils.PathFragmentConverter pathConverter = new OptionsUtils.PathFragmentConverter();
String pathString = pathConverter.convert(pieces[1]).getPathString();
try {
RepositoryName.validateUserProvidedRepoName(pieces[0]);
return new RepositoryInjection(pieces[0], pathString);
} catch (EvalException e) {
throw new OptionsParsingException("Invalid repository name given to inject", input, e);
}
}

@Override
public String getTypeDescription() {
return "an equals-separated mapping of repository name to path";
}
}

/** Converts from an equals-separated pair of strings into ModuleName->PathFragment mapping. */
public static class ModuleOverrideConverter extends Converter.Contextless<ModuleOverride> {

Expand All @@ -396,7 +435,7 @@ public ModuleOverride convert(String input) throws OptionsParsingException {

OptionsUtils.PathFragmentConverter pathConverter = new OptionsUtils.PathFragmentConverter();
String pathString = pathConverter.convert(pieces[1]).getPathString();
return ModuleOverride.create(pieces[0], pathString);
return new ModuleOverride(pieces[0], pathString);
}

@Override
Expand All @@ -406,28 +445,10 @@ public String getTypeDescription() {
}

/** A repository override, represented by a name and an absolute path to a repository. */
@AutoValue
public abstract static class RepositoryOverride {
public record RepositoryOverride(RepositoryName repositoryName, String path) {}

private static RepositoryOverride create(RepositoryName repositoryName, String path) {
return new AutoValue_RepositoryOptions_RepositoryOverride(repositoryName, path);
}

public abstract RepositoryName repositoryName();

public abstract String path();
}
public record RepositoryInjection(String apparentName, String path) {}

/** A module override, represented by a name and an absolute path to a module. */
@AutoValue
public abstract static class ModuleOverride {

private static ModuleOverride create(String moduleName, String path) {
return new AutoValue_RepositoryOptions_ModuleOverride(moduleName, path);
}

public abstract String moduleName();

public abstract String path();
}
public record ModuleOverride(String moduleName, String path) {}
}
47 changes: 24 additions & 23 deletions src/main/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import javax.annotation.Nullable;
import net.starlark.java.eval.StarlarkSemantics;

Expand All @@ -56,14 +55,32 @@ public RepositoryMappingFunction(RuleClassProvider ruleClassProvider) {
@Override
public SkyValue compute(SkyKey skyKey, Environment env)
throws SkyFunctionException, InterruptedException {
RepositoryMappingValue.Key key = (RepositoryMappingValue.Key) skyKey;
RepositoryMappingValue repositoryMappingValue = computeWithoutOverrides(key, env);
if (repositoryMappingValue == null) {
return null;
}
RepositoryName repoName = key.repoName();
if (repositoryMappingValue == RepositoryMappingValue.NOT_FOUND_VALUE
&& REPOSITORY_OVERRIDES.get(env).containsKey(repoName)) {
throw new RepositoryMappingFunctionException(
String.format(
"the repository %s does not exist, but has been specified as overridden with --override_repository. Use --inject_repository instead to add a new repository.",
repoName));
}
return repositoryMappingValue;
}

private RepositoryMappingValue computeWithoutOverrides(
RepositoryMappingValue.Key skyKey, Environment env)
throws SkyFunctionException, InterruptedException {
StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env);
if (starlarkSemantics == null) {
return null;
}
RepositoryName repositoryName = ((RepositoryMappingValue.Key) skyKey).repoName();
RepositoryName repositoryName = skyKey.repoName();
boolean enableBzlmod = starlarkSemantics.getBool(BuildLanguageOptions.ENABLE_BZLMOD);
boolean enableWorkspace = starlarkSemantics.getBool(BuildLanguageOptions.ENABLE_WORKSPACE);
Set<RepositoryName> repositoryOverrides = REPOSITORY_OVERRIDES.get(env).keySet();

if (!enableBzlmod && !enableWorkspace) {
throw new RepositoryMappingFunctionException(
Expand Down Expand Up @@ -115,7 +132,7 @@ public SkyValue compute(SkyKey skyKey, Environment env)
}

if (repositoryName.isMain()
&& ((RepositoryMappingValue.Key) skyKey).rootModuleShouldSeeWorkspaceRepos()
&& skyKey.rootModuleShouldSeeWorkspaceRepos()
&& enableWorkspace) {
// The root module should be able to see repos defined in WORKSPACE. Therefore, we find all
// workspace repos and add them as extra visible repos in root module's repo mappings.
Expand All @@ -134,7 +151,7 @@ public SkyValue compute(SkyKey skyKey, Environment env)
.collect(
toMap(
Entry::getKey, entry -> RepositoryName.createUnvalidated(entry.getKey())));
return computeForBazelModuleRepo(repositoryName, bazelDepGraphValue, repositoryOverrides)
return computeForBazelModuleRepo(repositoryName, bazelDepGraphValue)
.get()
// For the transitional period, we need to map the workspace name to the main repo.
.withAdditionalMappings(
Expand All @@ -146,7 +163,7 @@ public SkyValue compute(SkyKey skyKey, Environment env)

// Try and see if this is a repo generated from a Bazel module.
Optional<RepositoryMappingValue> mappingValue =
computeForBazelModuleRepo(repositoryName, bazelDepGraphValue, repositoryOverrides);
computeForBazelModuleRepo(repositoryName, bazelDepGraphValue);
if (mappingValue.isPresent()) {
return repositoryName.isMain()
? mappingValue.get().withCachedInverseMap()
Expand All @@ -169,15 +186,6 @@ public SkyValue compute(SkyKey skyKey, Environment env)
repoMappingEntriesValue.getModuleKey().name(),
repoMappingEntriesValue.getModuleKey().version());
}

if (!enableWorkspace) {
// If a repo is overridden without a previous definition, it uses the main repo's mapping.
if (REPOSITORY_OVERRIDES.get(env).containsKey(repositoryName)) {
return computeForBazelModuleRepo(
RepositoryName.MAIN, bazelDepGraphValue, repositoryOverrides)
.get();
}
}
}

if (enableWorkspace) {
Expand Down Expand Up @@ -211,19 +219,12 @@ public SkyValue compute(SkyKey skyKey, Environment env)
* Optional.empty().
*/
private Optional<RepositoryMappingValue> computeForBazelModuleRepo(
RepositoryName repositoryName,
BazelDepGraphValue bazelDepGraphValue,
Set<RepositoryName> repositoryOverrides) {
RepositoryName repositoryName, BazelDepGraphValue bazelDepGraphValue) {
ModuleKey moduleKey = bazelDepGraphValue.getCanonicalRepoNameLookup().get(repositoryName);
if (moduleKey == null) {
return Optional.empty();
}
RepositoryMapping repoMapping = bazelDepGraphValue.getFullRepoMapping(moduleKey);
if (repositoryName.isMain()) {
repoMapping =
repoMapping.withAdditionalMappings(
repositoryOverrides.stream().collect(toMap(RepositoryName::getName, name -> name)));
}
Module module = bazelDepGraphValue.getDepGraph().get(moduleKey);
return Optional.of(
RepositoryMappingValue.createForBzlmodRepo(
Expand Down
Loading

0 comments on commit 273012d

Please sign in to comment.