Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.1.0] Reproducible extension #21306

Merged
merged 5 commits into from
Feb 12, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ build --java_language_version=11
build --tool_java_language_version=11

# Fail if a glob doesn't match anything (https://github.com/bazelbuild/bazel/issues/8195)
build --incompatible_disallow_empty_glob
common --incompatible_disallow_empty_glob

# Manually enable cc toolchain resolution before it is flipped. https://github.com/bazelbuild/bazel/issues/7260
build --incompatible_enable_cc_toolchain_resolution
Expand Down
12 changes: 12 additions & 0 deletions site/en/external/overview.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,18 @@ Normally, Bazel only fetches a repo when it needs something from the repo,
and the repo hasn't already been fetched. If the repo has already been fetched
before, Bazel only re-fetches it if its definition has changed.

The `fetch` command can be used to initiate a pre-fetch for a repository,
target, or all necessary repositories to perform any build. This capability
enables offline builds using the `--nofetch` option.

The `--fetch` option serves to manage network access. Its default value is true.
However, when set to false (`--nofetch`), the command will utilize any cached
version of the dependency, and if none exists, the command will result in
failure.

See [fetch options](/reference/command-line-reference#fetch-options) for more
information about controlling fetch.

### Directory layout {:#directory-layout}

After being fetched, the repo can be found in the subdirectory `external` in the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,18 +121,23 @@ public void afterCommand() throws AbruptExitException {

// Add the new resolved extensions
for (var event : extensionResolutionEventsMap.values()) {
LockFileModuleExtension extension = event.getModuleExtension();
if (!extension.shouldLockExtesnsion()) {
continue;
}

var oldExtensionEntries = updatedExtensionMap.get(event.getExtensionId());
ImmutableMap<ModuleExtensionEvalFactors, LockFileModuleExtension> extensionEntries;
if (oldExtensionEntries != null) {
// extension exists, add the new entry to the existing map
extensionEntries =
new ImmutableMap.Builder<ModuleExtensionEvalFactors, LockFileModuleExtension>()
.putAll(oldExtensionEntries)
.put(event.getExtensionFactors(), event.getModuleExtension())
.put(event.getExtensionFactors(), extension)
.buildKeepingLast();
} else {
// new extension
extensionEntries = ImmutableMap.of(event.getExtensionFactors(), event.getModuleExtension());
extensionEntries = ImmutableMap.of(event.getExtensionFactors(), extension);
}
updatedExtensionMap.put(event.getExtensionId(), extensionEntries);
}
Expand Down Expand Up @@ -164,12 +169,13 @@ private boolean shouldKeepExtension(
// If there is a new event for this extension, compare it with the existing ones
ModuleExtensionResolutionEvent extEvent = extensionResolutionEventsMap.get(extensionId);
if (extEvent != null) {
boolean doNotLockExtension = !extEvent.getModuleExtension().shouldLockExtesnsion();
boolean dependencyOnOsChanged =
lockedExtensionKey.getOs().isEmpty() != extEvent.getExtensionFactors().getOs().isEmpty();
boolean dependencyOnArchChanged =
lockedExtensionKey.getArch().isEmpty()
!= extEvent.getExtensionFactors().getArch().isEmpty();
if (dependencyOnOsChanged || dependencyOnArchChanged) {
if (doNotLockExtension || dependencyOnOsChanged || dependencyOnArchChanged) {
return false;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ public static Builder builder() {

public abstract Builder toBuilder();

public boolean shouldLockExtesnsion() {
return getModuleExtensionMetadata().isEmpty()
|| !getModuleExtensionMetadata().get().getReproducible();
}

/** Builder type for {@link LockFileModuleExtension}. */
@AutoValue.Builder
public abstract static class Builder {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,11 +202,24 @@ public boolean rootModuleHasNonDevDependency() {
@ParamType(type = String.class),
@ParamType(type = NoneType.class)
}),
@Param(
name = "reproducible",
doc =
"States that this module extension ensures complete reproducibility, thereby it "
+ "should not be stored in the lockfile.",
positional = false,
named = true,
defaultValue = "False",
allowedTypes = {
@ParamType(type = Boolean.class),
}),
})
public ModuleExtensionMetadata extensionMetadata(
Object rootModuleDirectDepsUnchecked, Object rootModuleDirectDevDepsUnchecked)
Object rootModuleDirectDepsUnchecked,
Object rootModuleDirectDevDepsUnchecked,
boolean reproducible)
throws EvalException {
return ModuleExtensionMetadata.create(
rootModuleDirectDepsUnchecked, rootModuleDirectDevDepsUnchecked, extensionId);
rootModuleDirectDepsUnchecked, rootModuleDirectDevDepsUnchecked, reproducible);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -62,40 +62,44 @@ public abstract class ModuleExtensionMetadata implements StarlarkValue {

abstract UseAllRepos getUseAllRepos();

abstract boolean getReproducible();

private static ModuleExtensionMetadata create(
@Nullable Set<String> explicitRootModuleDirectDeps,
@Nullable Set<String> explicitRootModuleDirectDevDeps,
UseAllRepos useAllRepos) {
UseAllRepos useAllRepos,
boolean reproducible) {
return new AutoValue_ModuleExtensionMetadata(
explicitRootModuleDirectDeps != null
? ImmutableSet.copyOf(explicitRootModuleDirectDeps)
: null,
explicitRootModuleDirectDevDeps != null
? ImmutableSet.copyOf(explicitRootModuleDirectDevDeps)
: null,
useAllRepos);
useAllRepos,
reproducible);
}

static ModuleExtensionMetadata create(
Object rootModuleDirectDepsUnchecked,
Object rootModuleDirectDevDepsUnchecked,
ModuleExtensionId extensionId)
boolean reproducible)
throws EvalException {
if (rootModuleDirectDepsUnchecked == Starlark.NONE
&& rootModuleDirectDevDepsUnchecked == Starlark.NONE) {
return create(null, null, UseAllRepos.NO);
return create(null, null, UseAllRepos.NO, reproducible);
}

// When root_module_direct_deps = "all", accept both root_module_direct_dev_deps = None and
// root_module_direct_dev_deps = [], but not root_module_direct_dev_deps = ["some_repo"].
if (rootModuleDirectDepsUnchecked.equals("all")
&& rootModuleDirectDevDepsUnchecked.equals(StarlarkList.immutableOf())) {
return create(null, null, UseAllRepos.REGULAR);
return create(null, null, UseAllRepos.REGULAR, reproducible);
}

if (rootModuleDirectDevDepsUnchecked.equals("all")
&& rootModuleDirectDepsUnchecked.equals(StarlarkList.immutableOf())) {
return create(null, null, UseAllRepos.DEV);
return create(null, null, UseAllRepos.DEV, reproducible);
}

if (rootModuleDirectDepsUnchecked.equals("all")
Expand Down Expand Up @@ -153,7 +157,11 @@ static ModuleExtensionMetadata create(
}
}

return create(explicitRootModuleDirectDeps, explicitRootModuleDirectDevDeps, UseAllRepos.NO);
return create(
explicitRootModuleDirectDeps,
explicitRootModuleDirectDevDeps,
UseAllRepos.NO,
reproducible);
}

public void evaluate(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,20 +152,27 @@ public SkyValue compute(SkyKey skyKey, Environment env)
}

// Check the lockfile first for that module extension
LockFileModuleExtension lockedExtension = null;
LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env);
if (!lockfileMode.equals(LockfileMode.OFF)) {
BazelLockFileValue lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY);
if (lockfile == null) {
return null;
}
try {
SingleExtensionEvalValue singleExtensionEvalValue =
tryGettingValueFromLockFile(env, extensionId, extension, usagesValue, lockfile);
if (singleExtensionEvalValue != null) {
return singleExtensionEvalValue;
var lockedExtensionMap = lockfile.getModuleExtensions().get(extensionId);
lockedExtension =
lockedExtensionMap == null ? null : lockedExtensionMap.get(extension.getEvalFactors());
if (lockedExtension != null) {
try {
SingleExtensionEvalValue singleExtensionEvalValue =
tryGettingValueFromLockFile(
env, extensionId, extension, usagesValue, lockfile, lockedExtension);
if (singleExtensionEvalValue != null) {
return singleExtensionEvalValue;
}
} catch (NeedsSkyframeRestartException e) {
return null;
}
} catch (NeedsSkyframeRestartException e) {
return null;
}
}

Expand All @@ -182,6 +189,24 @@ public SkyValue compute(SkyKey skyKey, Environment env)
Optional<ModuleExtensionMetadata> moduleExtensionMetadata =
moduleExtensionResult.getModuleExtensionMetadata();

if (lockfileMode.equals(LockfileMode.ERROR)) {
boolean extensionShouldHaveBeenLocked =
moduleExtensionMetadata.map(metadata -> !metadata.getReproducible()).orElse(true);
// If this extension was not found in the lockfile, and after evaluation we found that it is
// not reproducible, then error indicating that it was expected to be in the lockfile.
if (lockedExtension == null && extensionShouldHaveBeenLocked) {
throw new SingleExtensionEvalFunctionException(
ExternalDepsException.withMessage(
Code.BAD_MODULE,
"The module extension '%s'%s does not exist in the lockfile",
extensionId,
extension.getEvalFactors().isEmpty()
? ""
: " for platform " + extension.getEvalFactors()),
Transience.PERSISTENT);
}
}

// At this point the extension has been evaluated successfully, but SingleExtensionEvalFunction
// may still fail if imported repositories were not generated. However, since imports do not
// influence the evaluation of the extension and the validation also runs when the extension
Expand Down Expand Up @@ -220,30 +245,13 @@ private SingleExtensionEvalValue tryGettingValueFromLockFile(
ModuleExtensionId extensionId,
RunnableExtension extension,
SingleExtensionUsagesValue usagesValue,
BazelLockFileValue lockfile)
BazelLockFileValue lockfile,
LockFileModuleExtension lockedExtension)
throws SingleExtensionEvalFunctionException,
InterruptedException,
NeedsSkyframeRestartException {
LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env);

var lockedExtensionMap = lockfile.getModuleExtensions().get(extensionId);
LockFileModuleExtension lockedExtension =
lockedExtensionMap == null ? null : lockedExtensionMap.get(extension.getEvalFactors());
if (lockedExtension == null) {
if (lockfileMode.equals(LockfileMode.ERROR)) {
throw new SingleExtensionEvalFunctionException(
ExternalDepsException.withMessage(
Code.BAD_MODULE,
"The module extension '%s'%s does not exist in the lockfile",
extensionId,
extension.getEvalFactors().isEmpty()
? ""
: " for platform " + extension.getEvalFactors()),
Transience.PERSISTENT);
}
return null;
}

ImmutableMap<ModuleKey, ModuleExtensionUsage> lockedExtensionUsages;
try {
// TODO(salmasamy) might be nicer to precompute this table when we construct
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -173,12 +173,14 @@ public ParallelismConverter() throws OptionsParsingException {
public int maxDirectoriesToEagerlyVisitInGlobbing;

@Option(
name = "fetch",
defaultValue = "true",
documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
effectTags = {OptionEffectTag.UNKNOWN},
help = "Allows the command to fetch external dependencies"
)
name = "fetch",
defaultValue = "true",
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.UNKNOWN},
help =
"Allows the command to fetch external dependencies. If set to false, the command will"
+ " utilize any cached version of the dependency, and if none exists, the command"
+ " will result in failure.")
public boolean fetch;

@Option(
Expand Down
Loading
Loading