From 3b9ec35f5a18afe256f679df1e91796dae720fc3 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Fri, 31 Mar 2023 08:07:58 -0700 Subject: [PATCH] Module Lockfile - Created lockfile K/F/V holding module information and its related flags - Updated BazelDepGraph function to use the lockfile - Created type adapters for module, related classes & immutable collections - Updated tests with Module.bazel to work after enabling the lockfile - Added new java test class for the lockfile - Added a flag to enable/disable lockfile (while it's still in progress) - Added integration tests PiperOrigin-RevId: 520925174 Change-Id: Ib8da99aa86a1797da44dc9a899d932f58c4a84c9 --- .../lib/bazel/BazelRepositoryModule.java | 8 +- .../devtools/build/lib/bazel/bzlmod/BUILD | 15 +- .../bazel/bzlmod/BazelDepGraphFunction.java | 142 ++++++-- .../bazel/bzlmod/BazelLockFileFunction.java | 114 +++++++ .../lib/bazel/bzlmod/BazelLockFileValue.java | 58 ++++ .../bzlmod/BazelModuleResolutionFunction.java | 2 +- .../bazel/bzlmod/BzlmodFlagsAndEnvVars.java | 66 ++++ .../bzlmod/DelegateTypeAdapterFactory.java | 126 +++++++ .../lib/bazel/bzlmod/GsonTypeAdapterUtil.java | 133 ++++++++ .../build/lib/bazel/bzlmod/Module.java | 8 + .../bazel/bzlmod/ModuleExtensionUsage.java | 2 + .../lib/bazel/bzlmod/ModuleFileFunction.java | 4 +- .../lib/bazel/bzlmod/ModuleFileValue.java | 7 +- .../build/lib/bazel/bzlmod/RepoSpec.java | 2 + .../devtools/build/lib/bazel/bzlmod/Tag.java | 2 + .../build/lib/cmdline/LabelConstants.java | 3 + .../semantics/BuildLanguageOptions.java | 10 + .../build/lib/skyframe/SkyFunctions.java | 2 + .../build/lib/analysis/util/AnalysisMock.java | 5 +- .../lib/analysis/util/AnalysisTestCase.java | 1 + .../bzlmod/BazelDepGraphFunctionTest.java | 27 +- .../bzlmod/BazelLockFileFunctionTest.java | 310 ++++++++++++++++++ .../BazelModuleResolutionFunctionTest.java | 7 +- .../bzlmod/BzlmodRepoRuleFunctionTest.java | 7 +- .../bzlmod/ModuleExtensionResolutionTest.java | 7 +- .../query2/testutil/SkyframeQueryHelper.java | 1 + .../repository/RepositoryDelegatorTest.java | 4 +- src/test/py/bazel/bzlmod/bazel_module_test.py | 144 ++++++++ 28 files changed, 1181 insertions(+), 36 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/DelegateTypeAdapterFactory.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java create mode 100644 src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 671b43e0d3c97c..9e16cbf75f3385 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -26,6 +26,7 @@ import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; @@ -249,7 +250,12 @@ public ResolutionReason getResolutionReason() { .addSkyFunction( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, directories.getWorkspace(), builtinModules)) - .addSkyFunction(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .addSkyFunction( + SkyFunctions.BAZEL_DEP_GRAPH, + new BazelDepGraphFunction(directories.getWorkspace(), registryFactory)) + .addSkyFunction( + SkyFunctions.BAZEL_LOCK_FILE, + new BazelLockFileFunction(directories.getWorkspace(), registryFactory)) .addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction()) .addSkyFunction(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 5cad88284cada8..afb981497c796e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -23,6 +23,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/net/starlark/java/eval", "//third_party:auto_value", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", ], @@ -52,14 +53,13 @@ java_library( deps = [ "//src/main/java/com/google/devtools/build/docgen/annot", "//src/main/java/com/google/devtools/build/lib/cmdline", - "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", "//src/main/java/net/starlark/java/annot", "//src/main/java/net/starlark/java/eval", "//src/main/java/net/starlark/java/syntax", "//third_party:auto_value", + "//third_party:gson", "//third_party:guava", - "//third_party:jsr305", ], ) @@ -88,7 +88,9 @@ java_library( "AbridgedModule.java", "ArchiveOverride.java", "BazelDepGraphValue.java", + "BazelLockFileValue.java", "BazelModuleResolutionValue.java", + "BzlmodFlagsAndEnvVars.java", "GitOverride.java", "LocalPathOverride.java", "Module.java", @@ -110,7 +112,6 @@ java_library( ":repo_rule_creator", "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories", "//src/main/java/com/google/devtools/build/lib/cmdline", - "//src/main/java/com/google/devtools/build/lib/concurrent", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", "//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions", @@ -120,6 +121,7 @@ java_library( "//src/main/java/net/starlark/java/eval", "//src/main/java/net/starlark/java/syntax", "//third_party:auto_value", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", ], @@ -129,8 +131,12 @@ java_library( name = "resolution_impl", srcs = [ "BazelDepGraphFunction.java", + "BazelLockFileFunction.java", "BazelModuleResolutionFunction.java", + "BzlmodFlagsAndEnvVars.java", + "DelegateTypeAdapterFactory.java", "Discovery.java", + "GsonTypeAdapterUtil.java", "ModuleExtensionContext.java", "ModuleFileFunction.java", "ModuleFileGlobals.java", @@ -158,6 +164,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", + "//src/main/java/com/google/devtools/build/lib/packages/semantics", "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_directory_value", "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function", "//src/main/java/com/google/devtools/build/lib/skyframe:bzl_load_value", @@ -165,6 +172,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/skyframe:client_environment_value", "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value", "//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster", + "//src/main/java/com/google/devtools/build/lib/util", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/skyframe", @@ -175,6 +183,7 @@ java_library( "//src/main/java/net/starlark/java/syntax", "//src/main/protobuf:failure_details_java_proto", "//third_party:auto_value", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", ], diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index dcc6b8e94d9034..c33ce992c9c299 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -15,19 +15,28 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableTable; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction.BazelModuleResolutionFunctionException; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.packages.LabelConverter; +import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; +import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; +import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; @@ -35,6 +44,7 @@ import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import javax.annotation.Nullable; +import net.starlark.java.eval.StarlarkSemantics; /** * This function runs Bazel module resolution, extracts the dependency graph from it and creates a @@ -43,24 +53,123 @@ */ public class BazelDepGraphFunction implements SkyFunction { + private final Path rootDirectory; + private final RegistryFactory registryFactory; + + public BazelDepGraphFunction(Path rootDirectory, RegistryFactory registryFactory) { + this.rootDirectory = rootDirectory; + this.registryFactory = registryFactory; + } + @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException { - BazelModuleResolutionValue selectionResult = - (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); - if (env.valuesMissing()) { + RootModuleFileValue root = + (RootModuleFileValue) env.getValue(ModuleFileValue.KEY_FOR_ROOT_MODULE); + if (root == null) { return null; } + StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env); + if (starlarkSemantics == null) { + return null; + } + + ImmutableMap depGraph = null; + BzlmodFlagsAndEnvVars flags = null; + + // If the module has not changed (has the same contents and flags as the lockfile), + // read the dependency graph from the lock file, else run resolution and update lockfile + if (starlarkSemantics.getBool(BuildLanguageOptions.ENABLE_LOCKFILE)) { + BazelLockFileValue lockFile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); + if (lockFile == null) { + return null; + } + flags = getFlagsAndEnvVars(env); + if (flags == null) { // unable to read environment variables + return null; + } + if (root.getModuleFileHash().equals(lockFile.getModuleFileHash()) + && flags.equals(lockFile.getFlags())) { + depGraph = lockFile.getModuleDepGraph(); + } + } + + if (depGraph == null) { + BazelModuleResolutionValue selectionResult = + (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); + if (env.valuesMissing()) { + return null; + } + depGraph = selectionResult.getResolvedDepGraph(); + if (starlarkSemantics.getBool(BuildLanguageOptions.ENABLE_LOCKFILE)) { + BazelLockFileFunction.updateLockedModule( + root.getModuleFileHash(), depGraph, rootDirectory, registryFactory, flags); + } + } - ImmutableMap depGraph = selectionResult.getResolvedDepGraph(); ImmutableMap canonicalRepoNameLookup = depGraph.keySet().stream() .collect(toImmutableMap(ModuleKey::getCanonicalRepoName, key -> key)); - // For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can - // group all usages by the label + name (the ModuleExtensionId). + ImmutableTable extensionUsagesById = + getExtensionUsagesById(depGraph); + + ImmutableBiMap extensionUniqueNames = + calculateUniqueNameForUsedExtensionId(extensionUsagesById); + + return BazelDepGraphValue.create( + depGraph, + canonicalRepoNameLookup, + depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList()), + extensionUsagesById, + extensionUniqueNames.inverse()); + } + + @Nullable + public static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) + throws InterruptedException { + ClientEnvironmentValue allowedYankedVersionsFromEnv = + (ClientEnvironmentValue) + env.getValue( + ClientEnvironmentFunction.key( + BazelModuleResolutionFunction.BZLMOD_ALLOWED_YANKED_VERSIONS_ENV)); + if (allowedYankedVersionsFromEnv == null) { + return null; + } + + ImmutableList registries = ImmutableList.copyOf(ModuleFileFunction.REGISTRIES.get(env)); + ImmutableMap moduleOverrides = + ModuleFileFunction.MODULE_OVERRIDES.get(env).entrySet().stream() + .collect( + toImmutableMap(e -> e.getKey(), e -> ((LocalPathOverride) e.getValue()).getPath())); + + ImmutableList yankedVersions = + ImmutableList.copyOf(BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.get(env)); + Boolean ignoreDevDeps = ModuleFileFunction.IGNORE_DEV_DEPS.get(env); + String compatabilityMode = + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.get(env).name(); + String directDepsMode = BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.get(env).name(); + + String envYanked = allowedYankedVersionsFromEnv.getValue(); + + return BzlmodFlagsAndEnvVars.create( + registries, + moduleOverrides, + yankedVersions, + nullToEmpty(envYanked), + ignoreDevDeps, + directDepsMode, + compatabilityMode); + } + + /** + * For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can group + * all usages by the label + name (the ModuleExtensionId). + */ + private ImmutableTable getExtensionUsagesById( + ImmutableMap depGraph) throws BazelModuleResolutionFunctionException { ImmutableTable.Builder extensionUsagesTableBuilder = ImmutableTable.builder(); for (Module module : depGraph.values()) { @@ -95,21 +204,18 @@ public SkyValue compute(SkyKey skyKey, Environment env) extensionUsagesTableBuilder.put(moduleExtensionId, module.getKey(), usage); } } - ImmutableTable extensionUsagesById = - extensionUsagesTableBuilder.buildOrThrow(); + return extensionUsagesTableBuilder.buildOrThrow(); + } + private ImmutableBiMap calculateUniqueNameForUsedExtensionId( + ImmutableTable extensionUsagesById) { // Calculate a unique name for each used extension id. BiMap extensionUniqueNames = HashBiMap.create(); for (ModuleExtensionId id : extensionUsagesById.rowKeySet()) { // Ensure that the resulting extension name (and thus the repository names derived from it) do // not start with a tilde. RepositoryName repository = id.getBzlFileLabel().getRepository(); - String nonEmptyRepoPart; - if (repository.isMain()) { - nonEmptyRepoPart = "_main"; - } else { - nonEmptyRepoPart = repository.getName(); - } + String nonEmptyRepoPart = repository.isMain() ? "_main" : repository.getName(); String bestName = nonEmptyRepoPart + "~" + id.getExtensionName(); if (extensionUniqueNames.putIfAbsent(bestName, id) == null) { continue; @@ -119,12 +225,6 @@ public SkyValue compute(SkyKey skyKey, Environment env) suffix++; } } - - return BazelDepGraphValue.create( - depGraph, - canonicalRepoNameLookup, - depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList()), - extensionUsagesById, - ImmutableMap.copyOf(extensionUniqueNames.inverse())); + return ImmutableBiMap.copyOf(extensionUniqueNames); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java new file mode 100644 index 00000000000000..99d0c049dd7de0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -0,0 +1,114 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.FileValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction.BazelModuleResolutionFunctionException; +import com.google.devtools.build.lib.cmdline.LabelConstants; +import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.gson.Gson; +import com.google.gson.JsonIOException; +import java.io.FileNotFoundException; +import java.io.IOException; +import javax.annotation.Nullable; + +/** Reads the contents of the lock file into its value. */ +public class BazelLockFileFunction implements SkyFunction { + + private final Path rootDirectory; + private final RegistryFactory registryFactory; + + private static final BzlmodFlagsAndEnvVars EMPTY_FLAGS = + BzlmodFlagsAndEnvVars.create( + ImmutableList.of(), ImmutableMap.of(), ImmutableList.of(), "", false, "", ""); + + private static final BazelLockFileValue EMPTY_LOCKFILE = + BazelLockFileValue.create( + BazelLockFileValue.LOCK_FILE_VERSION, "", EMPTY_FLAGS, ImmutableMap.of()); + + public BazelLockFileFunction(Path rootDirectory, RegistryFactory registryFactory) { + this.rootDirectory = rootDirectory; + this.registryFactory = registryFactory; + } + + @Override + @Nullable + public SkyValue compute(SkyKey skyKey, Environment env) + throws SkyFunctionException, InterruptedException { + RootedPath lockfilePath = + RootedPath.toRootedPath(Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); + + // Add dependency on the lockfile to recognize changes to it + if (env.getValue(FileValue.key(lockfilePath)) == null) { + return null; + } + + BazelLockFileValue bazelLockFileValue; + Gson gson = GsonTypeAdapterUtil.getLockfileGsonWithTypeAdapters(registryFactory); + try { + String json = FileSystemUtils.readContent(lockfilePath.asPath(), UTF_8); + bazelLockFileValue = gson.fromJson(json, BazelLockFileValue.class); + } catch (FileNotFoundException e) { + bazelLockFileValue = EMPTY_LOCKFILE; + } catch (IOException ex) { + throw new JsonIOException("Failed to read or parse module-lock file", ex); + } + return bazelLockFileValue; + } + + /** + * Updates the stored module in the lock file (ModuleHash, Flags & Dependency graph) + * + * @param hashedModule The hash of the current module file + * @param resolvedDepGraph The resolved dependency graph from the module file + */ + public static void updateLockedModule( + String hashedModule, + ImmutableMap resolvedDepGraph, + Path rootDirectory, + RegistryFactory registryFactory, + BzlmodFlagsAndEnvVars flags) + throws BazelModuleResolutionFunctionException { + RootedPath lockfilePath = + RootedPath.toRootedPath(Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); + + BazelLockFileValue value = + BazelLockFileValue.create( + BazelLockFileValue.LOCK_FILE_VERSION, hashedModule, flags, resolvedDepGraph); + Gson gson = GsonTypeAdapterUtil.getLockfileGsonWithTypeAdapters(registryFactory); + try { + FileSystemUtils.writeContent(lockfilePath.asPath(), UTF_8, gson.toJson(value)); + } catch (IOException e) { + throw new BazelModuleResolutionFunctionException( + ExternalDepsException.withCauseAndMessage( + Code.BAD_MODULE, e, "Unable to update module-lock file"), + Transience.PERSISTENT); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java new file mode 100644 index 00000000000000..0ff4315b7d2e6c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java @@ -0,0 +1,58 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; + +/** + * The result of reading the lockfile. Contains the lockfile version, module hash, definitions of + * module repositories, post-resolution dependency graph and module extensions data (ID, hash, + * definition, usages) + */ +@AutoValue +@GenerateTypeAdapter +public abstract class BazelLockFileValue implements SkyValue { + + public static final int LOCK_FILE_VERSION = 1; + + @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_LOCK_FILE; + + public static BazelLockFileValue create( + int lockFileVersion, + String moduleFileHash, + BzlmodFlagsAndEnvVars flags, + ImmutableMap moduleDepGraph) { + return new AutoValue_BazelLockFileValue(lockFileVersion, moduleFileHash, flags, moduleDepGraph); + } + + /** Current version of the lock file */ + public abstract int getLockFileVersion(); + + /** Hash of the Module file */ + public abstract String getModuleFileHash(); + + /** Command line flags and environment variables that can affect the resolution */ + public abstract BzlmodFlagsAndEnvVars getFlags(); + + /** The post-selection dep graph retrieved from the lock file. */ + public abstract ImmutableMap getModuleDepGraph(); +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java index c44ca83ea57bc7..f7bb9253476dd6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java @@ -60,7 +60,7 @@ public class BazelModuleResolutionFunction implements SkyFunction { public static final Precomputed> ALLOWED_YANKED_VERSIONS = new Precomputed<>("allowed_yanked_versions"); - private static final String BZLMOD_ALLOWED_YANKED_VERSIONS_ENV = "BZLMOD_ALLOW_YANKED_VERSIONS"; + public static final String BZLMOD_ALLOWED_YANKED_VERSIONS_ENV = "BZLMOD_ALLOW_YANKED_VERSIONS"; @Override @Nullable diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java new file mode 100644 index 00000000000000..cb5dafb1c51ac2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java @@ -0,0 +1,66 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; + +/** Stores the values of flags and environment variables that affect the resolution */ +@AutoValue +@GenerateTypeAdapter +abstract class BzlmodFlagsAndEnvVars { + + public static BzlmodFlagsAndEnvVars create( + ImmutableList registries, + ImmutableMap moduleOverrides, + ImmutableList yankedVersions, + String envVarYankedVersions, + boolean ignoreDevDeps, + String directDepsMode, + String compatabilityMode) { + return new AutoValue_BzlmodFlagsAndEnvVars( + registries, + moduleOverrides, + yankedVersions, + envVarYankedVersions, + ignoreDevDeps, + directDepsMode, + compatabilityMode); + } + + /** Registries provided via command line */ + public abstract ImmutableList getCmdRegistries(); + + /** ModulesOverride provided via command line */ + public abstract ImmutableMap getCmdModuleOverrides(); + + /** Allowed yanked version in the dependency graph */ + public abstract ImmutableList getAllowedYankedVersions(); + + /** Allowed yanked version in the dependency graph from environment variables */ + public abstract String getEnvVarAllowedYankedVersions(); + + /** Whether to ignore things declared as dev dependencies or not */ + public abstract boolean ignoreDevDependency(); + + /** Error level of direct dependencies check */ + public abstract String getDirectDependenciesMode(); + + /** Error level of bazel compatability check */ + public abstract String getCompatibilityMode(); +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/DelegateTypeAdapterFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/DelegateTypeAdapterFactory.java new file mode 100644 index 00000000000000..d791789dc46bbb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/DelegateTypeAdapterFactory.java @@ -0,0 +1,126 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.TypeAdapter; +import com.google.gson.TypeAdapterFactory; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import java.io.IOException; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import javax.annotation.Nullable; +import net.starlark.java.eval.Dict; + +/** + * Creates Gson type adapters for parameterized types by using a delegate parameterized type that + * already has a registered type adapter factory. + */ +public final class DelegateTypeAdapterFactory + implements TypeAdapterFactory { + private final Class rawType; + private final Class intermediateToDelegateType; + private final Class delegateType; + private final Function rawToDelegate; + private final Function delegateToRaw; + + private DelegateTypeAdapterFactory( + Class rawType, + Class intermediateToDelegateType, + Class delegateType, + Function rawToDelegate, + Function delegateToRaw) { + this.rawType = rawType; + this.intermediateToDelegateType = intermediateToDelegateType; + this.delegateType = delegateType; + this.rawToDelegate = rawToDelegate; + this.delegateToRaw = delegateToRaw; + } + + public static final TypeAdapterFactory IMMUTABLE_MAP = + new DelegateTypeAdapterFactory<>( + ImmutableMap.class, + Map.class, + LinkedHashMap.class, + raw -> new LinkedHashMap<>((Map) raw), + delegate -> ImmutableMap.copyOf((Map) delegate)); + + public static final TypeAdapterFactory IMMUTABLE_BIMAP = + new DelegateTypeAdapterFactory<>( + ImmutableBiMap.class, + Map.class, + LinkedHashMap.class, + raw -> new LinkedHashMap<>((Map) raw), + delegate -> ImmutableBiMap.copyOf((Map) delegate)); + + public static final TypeAdapterFactory DICT = + new DelegateTypeAdapterFactory<>( + Dict.class, + Map.class, + LinkedHashMap.class, + raw -> new LinkedHashMap<>((Map) raw), + delegate -> Dict.immutableCopyOf((Map) delegate)); + + public static final TypeAdapterFactory IMMUTABLE_LIST = + new DelegateTypeAdapterFactory<>( + ImmutableList.class, + List.class, + ArrayList.class, + raw -> new ArrayList<>((List) raw), + delegate -> ImmutableList.copyOf((List) delegate)); + + @SuppressWarnings("unchecked") + @Override + @Nullable + public TypeAdapter create(Gson gson, TypeToken typeToken) { + Type type = typeToken.getType(); + if (typeToken.getRawType() != rawType || !(type instanceof ParameterizedType)) { + return null; + } + + com.google.common.reflect.TypeToken betterToken = + com.google.common.reflect.TypeToken.of(typeToken.getType()); + final TypeAdapter delegateAdapter = + (TypeAdapter) + gson.getAdapter( + TypeToken.get( + betterToken + .getSupertype((Class) intermediateToDelegateType) + .getSubtype(delegateType) + .getType())); + return new TypeAdapter() { + @Override + public void write(JsonWriter out, T value) throws IOException { + delegateAdapter.write(out, rawToDelegate.apply((R) value)); + } + + @Override + public T read(JsonReader in) throws IOException { + return (T) delegateToRaw.apply((D) delegateAdapter.read(in)); + } + }; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java new file mode 100644 index 00000000000000..4832c117d8d0d9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java @@ -0,0 +1,133 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.DICT; +import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_BIMAP; +import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_LIST; +import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_MAP; + +import com.google.common.base.Splitter; +import com.google.common.base.VerifyException; +import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.List; + +/** + * Utility class to hold type adapters and helper methods to get gson registered with type adapters + */ +public final class GsonTypeAdapterUtil { + + public static final TypeAdapter VERSION_TYPE_ADAPTER = + new TypeAdapter<>() { + @Override + public void write(JsonWriter jsonWriter, Version version) throws IOException { + jsonWriter.value(version.toString()); + } + + @Override + public Version read(JsonReader jsonReader) throws IOException { + Version version; + String versionString = jsonReader.nextString(); + try { + version = Version.parse(versionString); + } catch (ParseException e) { + throw new JsonParseException( + String.format("Unable to parse Version %s from the lockfile", versionString), e); + } + return version; + } + }; + + public static final TypeAdapter MODULE_KEY_TYPE_ADAPTER = + new TypeAdapter<>() { + @Override + public void write(JsonWriter jsonWriter, ModuleKey moduleKey) throws IOException { + jsonWriter.value(moduleKey.toString()); + } + + @Override + public ModuleKey read(JsonReader jsonReader) throws IOException { + String jsonString = jsonReader.nextString(); + if (jsonString.equals("")) { + return ModuleKey.ROOT; + } + List parts = Splitter.on('@').splitToList(jsonString); + if (parts.get(1).equals("_")) { + return ModuleKey.create(parts.get(0), Version.EMPTY); + } + + Version version; + try { + version = Version.parse(parts.get(1)); + } catch (ParseException e) { + throw new JsonParseException( + String.format("Unable to parse ModuleKey %s version from the lockfile", jsonString), + e); + } + return ModuleKey.create(parts.get(0), version); + } + }; + + public static TypeAdapter registryTypeAdapter(RegistryFactory registryFactory) { + return new TypeAdapter<>() { + @Override + public void write(JsonWriter jsonWriter, Registry registry) throws IOException { + jsonWriter.value(registry.getUrl()); + } + + @Override + public Registry read(JsonReader jsonReader) throws IOException { + try { + return registryFactory.getRegistryWithUrl(jsonReader.nextString()); + } catch (URISyntaxException e) { + throw new VerifyException("Lockfile registry URL is not valid", e); + } + } + }; + } + + private static final GsonBuilder adapterGson = + new GsonBuilder() + .registerTypeAdapterFactory(GenerateTypeAdapter.FACTORY) + .registerTypeAdapterFactory(DICT) + .registerTypeAdapterFactory(IMMUTABLE_MAP) + .registerTypeAdapterFactory(IMMUTABLE_LIST) + .registerTypeAdapterFactory(IMMUTABLE_BIMAP) + .registerTypeAdapter(Version.class, VERSION_TYPE_ADAPTER) + .registerTypeAdapter(ModuleKey.class, MODULE_KEY_TYPE_ADAPTER); + + /** + * Gets a gson with registered adapters needed to read the lockfile. + * + * @param registryFactory Registry factory to use in the registry adapter + * @return gson with type adapters + */ + public static Gson getLockfileGsonWithTypeAdapters(RegistryFactory registryFactory) { + return adapterGson + .registerTypeAdapter(Registry.class, registryTypeAdapter(registryFactory)) + .create(); + } + + private GsonTypeAdapterUtil() {} +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java index cfc09237b415b0..66125402de6506 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java @@ -22,6 +22,7 @@ import com.google.devtools.build.lib.cmdline.RepositoryMapping; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import java.util.Map; import java.util.Optional; import java.util.function.UnaryOperator; @@ -35,6 +36,7 @@ * discovery but before selection, or when there's a multiple_version_override in play). */ @AutoValue +@GenerateTypeAdapter public abstract class Module { /** @@ -186,6 +188,8 @@ public abstract static class Builder { /** Optional; defaults to {@link #setName}. */ public abstract Builder setRepoName(String value); + public abstract Builder setBazelCompatibility(ImmutableList value); + abstract ImmutableList.Builder bazelCompatibilityBuilder(); @CanIgnoreReturnValue @@ -194,6 +198,8 @@ public final Builder addBazelCompatibilityValues(Iterable values) { return this; } + public abstract Builder setExecutionPlatformsToRegister(ImmutableList value); + abstract ImmutableList.Builder executionPlatformsToRegisterBuilder(); @CanIgnoreReturnValue @@ -202,6 +208,8 @@ public final Builder addExecutionPlatformsToRegister(Iterable values) { return this; } + public abstract Builder setToolchainsToRegister(ImmutableList value); + abstract ImmutableList.Builder toolchainsToRegisterBuilder(); @CanIgnoreReturnValue diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java index 9266ef39a5ec94..64be185428b300 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import net.starlark.java.syntax.Location; /** @@ -25,6 +26,7 @@ * information pertinent to the proxy object returned from the {@code use_extension} call. */ @AutoValue +@GenerateTypeAdapter public abstract class ModuleExtensionUsage { /** An unresolved label pointing to the Starlark file where the module extension is defined. */ public abstract String getExtensionBzlFile(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java index a7c1a0cbb51d7c..74ca2daa0395c3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java @@ -30,6 +30,7 @@ import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; +import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.Root; @@ -152,6 +153,7 @@ private SkyValue computeForRootModule(StarlarkSemantics starlarkSemantics, Envir return null; } byte[] moduleFile = readFile(moduleFilePath.asPath()); + String moduleFileHash = new Fingerprint().addBytes(moduleFile).hexDigestAndReset(); ModuleFileGlobals moduleFileGlobals = execModuleFile( moduleFile, @@ -184,7 +186,7 @@ private SkyValue computeForRootModule(StarlarkSemantics starlarkSemantics, Envir name -> ModuleKey.create(name, Version.EMPTY).getCanonicalRepoName(), name -> name)); return RootModuleFileValue.create( - module, overrides, nonRegistryOverrideCanonicalRepoNameLookup); + module, moduleFileHash, overrides, nonRegistryOverrideCanonicalRepoNameLookup); } private ModuleFileGlobals execModuleFile( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java index b35e8c7ac5fb85..38bc90e77266a1 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java @@ -53,6 +53,10 @@ public static NonRootModuleFileValue create(Module module) { */ @AutoValue public abstract static class RootModuleFileValue extends ModuleFileValue { + + /** The hash string of Module.bazel (using SHA256) */ + public abstract String getModuleFileHash(); + /** * The overrides specified by the evaluated module file. The key is the module name and the * value is the override itself. @@ -68,10 +72,11 @@ public abstract static class RootModuleFileValue extends ModuleFileValue { public static RootModuleFileValue create( Module module, + String moduleHash, ImmutableMap overrides, ImmutableMap nonRegistryOverrideCanonicalRepoNameLookup) { return new AutoValue_ModuleFileValue_RootModuleFileValue( - module, overrides, nonRegistryOverrideCanonicalRepoNameLookup); + module, moduleHash, overrides, nonRegistryOverrideCanonicalRepoNameLookup); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java index 71324a875cb169..1aee3ae959a28b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java @@ -16,6 +16,7 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableMap; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import java.util.Optional; /** @@ -23,6 +24,7 @@ * defined. */ @AutoValue +@GenerateTypeAdapter public abstract class RepoSpec { /** diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Tag.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Tag.java index 498aacd427c49a..c23454ed7f245a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Tag.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Tag.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.auto.value.AutoValue; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import net.starlark.java.eval.Dict; import net.starlark.java.syntax.Location; @@ -26,6 +27,7 @@ * not when the tag is created, which is during module discovery). */ @AutoValue +@GenerateTypeAdapter public abstract class Tag { public abstract String getTagName(); diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java b/src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java index b133f40ca94927..addc810d13f144 100644 --- a/src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java +++ b/src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java @@ -44,6 +44,9 @@ public class LabelConstants { public static final PathFragment WORKSPACE_DOT_BAZEL_FILE_NAME = PathFragment.create("WORKSPACE.bazel"); public static final PathFragment MODULE_DOT_BAZEL_FILE_NAME = PathFragment.create("MODULE.bazel"); + + public static final PathFragment MODULE_LOCKFILE_NAME = PathFragment.create("MODULE.lock.bazel"); + public static final String DEFAULT_REPOSITORY_DIRECTORY = "__main__"; // With this prefix, non-main repositories are symlinked under diff --git a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java index 4cf7d122d5065a..4856b52f8c91bd 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java +++ b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java @@ -207,6 +207,14 @@ public final class BuildLanguageOptions extends OptionsBase { + " WORKSPACE. See https://bazel.build/docs/bzlmod for more information.") public boolean enableBzlmod; + @Option( + name = "experimental_enable_bzlmod_lockfile", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.STARLARK_SEMANTICS, + effectTags = OptionEffectTag.LOADING_AND_ANALYSIS, + help = "If true, enables the Bzlmod lockfile caching the module contents.") + public boolean enableLockfile; + @Option( name = "experimental_java_proto_library_default_has_services", defaultValue = "true", @@ -678,6 +686,7 @@ public StarlarkSemantics toStarlarkSemantics() { .setBool( EXPERIMENTAL_ENABLE_ANDROID_MIGRATION_APIS, experimentalEnableAndroidMigrationApis) .setBool(ENABLE_BZLMOD, enableBzlmod) + .setBool(ENABLE_LOCKFILE, enableLockfile) .setBool( EXPERIMENTAL_JAVA_PROTO_LIBRARY_DEFAULT_HAS_SERVICES, experimentalJavaProtoLibraryDefaultHasServices) @@ -771,6 +780,7 @@ public StarlarkSemantics toStarlarkSemantics() { public static final String EXPERIMENTAL_ENABLE_ANDROID_MIGRATION_APIS = "-experimental_enable_android_migration_apis"; public static final String ENABLE_BZLMOD = "-enable_bzlmod"; + public static final String ENABLE_LOCKFILE = "-experimental_enable_bzlmod_lockfile"; public static final String EXPERIMENTAL_JAVA_PROTO_LIBRARY_DEFAULT_HAS_SERVICES = "+experimental_java_proto_library_default_has_services"; public static final String INCOMPATIBLE_EXISTING_RULES_IMMUTABLE_VIEW = diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java index 863a9d472976c3..9eb8b4f11577e3 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java @@ -153,6 +153,8 @@ public final class SkyFunctions { SkyFunctionName.createNonHermetic("SINGLE_EXTENSION_EVAL"); public static final SkyFunctionName BAZEL_DEP_GRAPH = SkyFunctionName.createHermetic("BAZEL_DEP_GRAPH"); + public static final SkyFunctionName BAZEL_LOCK_FILE = + SkyFunctionName.createHermetic("BAZEL_LOCK_FILE"); public static Predicate isSkyFunction(SkyFunctionName functionName) { return key -> key.functionName().equals(functionName); diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java index 2501f17dc55937..162c01a546a171 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java @@ -18,6 +18,7 @@ import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; @@ -142,7 +143,9 @@ public ImmutableMap getSkyFunctions(BlazeDirectori directories.getWorkspace(), getBuiltinModules(directories)), SkyFunctions.BAZEL_DEP_GRAPH, - new BazelDepGraphFunction(), + new BazelDepGraphFunction(directories.getWorkspace(), FakeRegistry.DEFAULT_FACTORY), + SkyFunctions.BAZEL_LOCK_FILE, + new BazelLockFileFunction(directories.getWorkspace(), FakeRegistry.DEFAULT_FACTORY), SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction(), SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java index ab968df5d9cd48..8de8c73eb978ff 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java @@ -259,6 +259,7 @@ private void reinitializeSkyframeExecutor() { packageOptions.globbingThreads = 3; BuildLanguageOptions buildLanguageOptions = Options.getDefaults(BuildLanguageOptions.class); buildLanguageOptions.enableBzlmod = true; + buildLanguageOptions.enableLockfile = false; skyframeExecutor.preparePackageLoading( pkgLocator, packageOptions, diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index 305ffa2caf6ae4..9cef1c466e8f6c 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -122,18 +122,28 @@ public void setup() throws Exception { SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put( + SkyFunctions.BAZEL_LOCK_FILE, + new BazelLockFileFunction(rootDirectory, registryFactory)) + .put( + SkyFunctions.BAZEL_DEP_GRAPH, + new BazelDepGraphFunction(rootDirectory, registryFactory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, resolutionFunctionMock) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, - new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of()))) + new ClientEnvironmentFunction( + new AtomicReference<>(ImmutableMap.of("BZLMOD_ALLOW_YANKED_VERSIONS", "")))) .buildOrThrow(), differencer); PrecomputedValue.STARLARK_SEMANTICS.set( differencer, StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build()); + PrecomputedValue.STARLARK_SEMANTICS.set( + differencer, + StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_LOCKFILE, true).build()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.OFF); @@ -144,6 +154,10 @@ public void setup() throws Exception { @Test public void createValue_basic() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + // Root depends on dep@1.0 and dep@2.0 at the same time with a multiple-version override. // Root also depends on rules_cc as a normal dep. // dep@1.0 depends on rules_java, which is overridden by a non-registry override (see below). @@ -190,7 +204,6 @@ public void createValue_basic() throws Exception { .build()) .buildOrThrow(); - // TODO we need to mock bazelModuleResolution function to return depGraph resolutionFunctionMock.setDepGraph(depGraph); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelDepGraphValue.KEY), evaluationContext); @@ -231,6 +244,10 @@ private static ModuleExtensionUsage createModuleExtensionUsage( @Test public void createValue_moduleExtensions() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + Module root = Module.builder() .setName("root") @@ -331,6 +348,10 @@ public void createValue_moduleExtensions() throws Exception { @Test public void useExtensionBadLabelFails() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='module', version='1.0')"); + Module root = Module.builder() .addExtensionUsage(createModuleExtensionUsage("@foo//:defs.bzl", "bar")) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java new file mode 100644 index 00000000000000..924c2051254ba2 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -0,0 +1,310 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.FileValue; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.ServerDirectories; +import com.google.devtools.build.lib.analysis.util.AnalysisMock; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.clock.BlazeClock; +import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; +import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; +import com.google.devtools.build.lib.skyframe.ExternalFilesHelper; +import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; +import com.google.devtools.build.lib.skyframe.FileFunction; +import com.google.devtools.build.lib.skyframe.FileStateFunction; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.testutil.FoundationTestCase; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.FileStateKey; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.SyscallCache; +import com.google.devtools.build.skyframe.EvaluationContext; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator; +import com.google.devtools.build.skyframe.MemoizingEvaluator; +import com.google.devtools.build.skyframe.RecordingDifferencer; +import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.Nullable; +import net.starlark.java.eval.StarlarkSemantics; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link BazelLockFileFunction}. */ +@RunWith(JUnit4.class) +public class BazelLockFileFunctionTest extends FoundationTestCase { + + private MemoizingEvaluator evaluator; + private RecordingDifferencer differencer; + private EvaluationContext evaluationContext; + private FakeRegistry.Factory registryFactory; + private static SkyFunctionName updateLockfileFunction; + + @Before + public void setup() throws Exception { + differencer = new SequencedRecordingDifferencer(); + registryFactory = new FakeRegistry.Factory(); + evaluationContext = + EvaluationContext.newBuilder().setParallelism(8).setEventHandler(reporter).build(); + + AtomicReference packageLocator = + new AtomicReference<>( + new PathPackageLocator( + outputBase, + ImmutableList.of(Root.fromPath(rootDirectory)), + BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY)); + BlazeDirectories directories = + new BlazeDirectories( + new ServerDirectories(rootDirectory, outputBase, rootDirectory), + rootDirectory, + /* defaultSystemJavabase= */ null, + AnalysisMock.get().getProductName()); + ExternalFilesHelper externalFilesHelper = + ExternalFilesHelper.createForTesting( + packageLocator, + ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, + directories); + + updateLockfileFunction = SkyFunctionName.createHermetic("LockfileWrite"); + + evaluator = + new InMemoryMemoizingEvaluator( + ImmutableMap.builder() + .put(FileValue.FILE, new FileFunction(packageLocator, directories)) + .put( + FileStateKey.FILE_STATE, + new FileStateFunction( + Suppliers.ofInstance( + new TimestampGranularityMonitor(BlazeClock.instance())), + SyscallCache.NO_CACHE, + externalFilesHelper)) + .put( + SkyFunctions.MODULE_FILE, + new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) + .put( + SkyFunctions.BAZEL_LOCK_FILE, + new BazelLockFileFunction(rootDirectory, registryFactory)) + .put( + SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, + new ClientEnvironmentFunction( + new AtomicReference<>(ImmutableMap.of("BZLMOD_ALLOW_YANKED_VERSIONS", "")))) + .put( + updateLockfileFunction, + new SkyFunction() { + @Nullable + @Override + public SkyValue compute(SkyKey skyKey, Environment env) + throws SkyFunctionException, InterruptedException { + + UpdateLockFileKey key = (UpdateLockFileKey) skyKey; + BzlmodFlagsAndEnvVars flags = BazelDepGraphFunction.getFlagsAndEnvVars(env); + if (flags == null) { + return null; + } + BazelLockFileFunction.updateLockedModule( + key.moduleHash(), + key.depGraph(), + rootDirectory, + registryFactory, + flags); + return new SkyValue() {}; + } + }) + .buildOrThrow(), + differencer); + + PrecomputedValue.STARLARK_SEMANTICS.set( + differencer, + StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build()); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); + ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); + ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); + BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( + differencer, BazelCompatibilityMode.ERROR); + BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( + differencer, CheckDirectDepsMode.ERROR); + } + + @Test + public void simpleModule() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')", + "bazel_dep(name = 'dep_1', version = '1.0')", + "bazel_dep(name = 'dep_2', version = '2.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + + ImmutableMap depGraph = + ImmutableMap.builder() + .put(ModuleKey.ROOT, rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE).getModule()) + .buildOrThrow(); + + UpdateLockFileKey key = UpdateLockFileKey.create("moduleHash", depGraph); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + BazelLockFileValue value = result.get(BazelLockFileValue.KEY); + assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); + } + + @Test + public void simpleModuleWithFlags() throws Exception { + // Test having --ovseride_module, --ignore_dev_dependency, --check_bazel_compatibility & + // --check_direct_dependencies + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + + ImmutableMap depGraph = + ImmutableMap.builder() + .put(ModuleKey.ROOT, rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE).getModule()) + .buildOrThrow(); + + ImmutableList yankedVersions = ImmutableList.of("2.4", "2.3"); + LocalPathOverride override = LocalPathOverride.create("override_path"); + + ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); + ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of("my_dep_1", override)); + + BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, yankedVersions); + BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( + differencer, CheckDirectDepsMode.ERROR); + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( + differencer, BazelCompatibilityMode.ERROR); + + UpdateLockFileKey key = UpdateLockFileKey.create("moduleHash", depGraph); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + BazelLockFileValue value = result.get(BazelLockFileValue.KEY); + assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); + assertThat(value.getFlags().ignoreDevDependency()).isTrue(); + assertThat(value.getFlags().getAllowedYankedVersions()).isEqualTo(yankedVersions); + assertThat(value.getFlags().getDirectDependenciesMode()) + .isEqualTo(CheckDirectDepsMode.ERROR.toString()); + assertThat(value.getFlags().getCompatibilityMode()) + .isEqualTo(BazelCompatibilityMode.ERROR.toString()); + } + + @Test + public void fullModule() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')", + "register_toolchains('//my:toolchain', '//my:toolchain2')", + "ext1 = use_extension('//:defs.bzl','ext_1')", + "use_repo(ext1,'myrepo')", + "ext2 = use_extension('@ext//:defs.bzl','ext_2')", + "ext2.tag(file='@myrepo//:Hello1.txt')", + "ext2.tag(file='@myrepo//:Hello2.txt')", + "use_repo(ext2,'ext_repo')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + + ImmutableMap depGraph = + ImmutableMap.builder() + .put(ModuleKey.ROOT, rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE).getModule()) + .buildOrThrow(); + + UpdateLockFileKey key = UpdateLockFileKey.create("moduleHash", depGraph); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + BazelLockFileValue value = result.get(BazelLockFileValue.KEY); + assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); + } + + @AutoValue + abstract static class UpdateLockFileKey implements SkyKey { + + abstract String moduleHash(); + + abstract ImmutableMap depGraph(); + + static UpdateLockFileKey create(String moduleHash, ImmutableMap depGraph) { + return new AutoValue_BazelLockFileFunctionTest_UpdateLockFileKey(moduleHash, depGraph); + } + + @Override + public SkyFunctionName functionName() { + return updateLockfileFunction; + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 262101b7741dd1..8cf78d71f3e517 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -111,7 +111,12 @@ public void setup() throws Exception { SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put( + SkyFunctions.BAZEL_DEP_GRAPH, + new BazelDepGraphFunction(rootDirectory, registryFactory)) + .put( + SkyFunctions.BAZEL_LOCK_FILE, + new BazelLockFileFunction(rootDirectory, registryFactory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index dcf7f2ef3fe682..f59d874fd0e3ef 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -123,7 +123,12 @@ public void setup() throws Exception { .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put( + SkyFunctions.BAZEL_DEP_GRAPH, + new BazelDepGraphFunction(rootDirectory, registryFactory)) + .put( + SkyFunctions.BAZEL_LOCK_FILE, + new BazelLockFileFunction(rootDirectory, registryFactory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( SkyFunctions.MODULE_FILE, diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 2b488324e115b0..35e06507ea1705 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -247,7 +247,12 @@ public void setup() throws Exception { .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put( + SkyFunctions.BAZEL_LOCK_FILE, + new BazelLockFileFunction(rootDirectory, registryFactory)) + .put( + SkyFunctions.BAZEL_DEP_GRAPH, + new BazelDepGraphFunction(rootDirectory, registryFactory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction()) .put(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction) diff --git a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java index abfed66709ac50..181ab7c7144b06 100644 --- a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java +++ b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java @@ -322,6 +322,7 @@ protected void initTargetPatternEvaluator(ConfiguredRuleClassProvider ruleClassP BuildLanguageOptions buildLanguageOptions = Options.getDefaults(BuildLanguageOptions.class); buildLanguageOptions.enableBzlmod = enableBzlmod(); + buildLanguageOptions.enableLockfile = false; PathPackageLocator packageLocator = skyframeExecutor.createPackageLocator( diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java index a44bd8774373e2..32e0f628188c1b 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -225,7 +225,9 @@ public void setupDelegator() throws Exception { .put( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootPath, ImmutableMap.of())) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put( + SkyFunctions.BAZEL_DEP_GRAPH, + new BazelDepGraphFunction(rootDirectory, registryFactory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, diff --git a/src/test/py/bazel/bzlmod/bazel_module_test.py b/src/test/py/bazel/bzlmod/bazel_module_test.py index 4f61be2c8809c1..d6a73cb5144076 100644 --- a/src/test/py/bazel/bzlmod/bazel_module_test.py +++ b/src/test/py/bazel/bzlmod/bazel_module_test.py @@ -63,6 +63,7 @@ def writeBazelrcFile(self, allow_yanked_versions=True): # Set an explicit Java language version 'common --java_language_version=8', 'common --tool_java_language_version=8', + 'common --experimental_enable_bzlmod_lockfile', ] + ( [ @@ -1019,6 +1020,149 @@ def testNativePackageRelativeLabel(self): self.assertIn('5th: @@bleb//bleb:bleb', stderr) self.assertIn('6th: @@//bleb:bleb', stderr) + def testChangeModuleInRegistryWithoutLockfile(self): + # Add module 'sss' to the registry with dep on 'aaa' + self.main_registry.createCcModule('sss', '1.3', {'aaa': '1.1'}) + # Create a project with deps on 'sss' + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "sss", version = "1.3")', + ], + ) + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = ["@sss//:lib_sss"],', + ')', + ], + ) + self.RunBazel(['build', '--nobuild', '//:main'], allow_failure=False) + + # Change registry -> update 'sss' module file (corrupt it) + module_dir = self.main_registry.root.joinpath('modules', 'sss', '1.3') + scratchFile(module_dir.joinpath('MODULE.bazel'), ['whatever!']) + + # Clean bazel to empty any cache of the deps tree + self.RunBazel(['clean', '--expunge']) + # Runing again will try to get 'sss' which should produce an error + exit_code, _, stderr = self.RunBazel( + [ + 'build', + '--nobuild', + '--experimental_enable_bzlmod_lockfile=false', + '//:main', + ], + allow_failure=True, + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + ( + 'ERROR: Error computing the main repository mapping: error parsing' + ' MODULE.bazel file for sss@1.3' + ), + stderr, + ) + + def testChangeModuleInRegistryWithLockfile(self): + # Add module 'sss' to the registry with dep on 'aaa' + self.main_registry.createCcModule('sss', '1.3', {'aaa': '1.1'}) + # Create a project with deps on 'sss' + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "sss", version = "1.3")', + ], + ) + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = ["@sss//:lib_sss"],', + ')', + ], + ) + self.RunBazel( + [ + 'build', + '--nobuild', + '//:main', + ], + allow_failure=False, + ) + + # Change registry -> update 'sss' module file (corrupt it) + module_dir = self.main_registry.root.joinpath('modules', 'sss', '1.3') + scratchFile(module_dir.joinpath('MODULE.bazel'), ['whatever!']) + + # Clean bazel to empty any cache of the deps tree + self.RunBazel(['clean', '--expunge']) + # Running with the lockfile, should not recognize the registry changes + # hence find no errors + self.RunBazel( + [ + 'build', + '--nobuild', + '//:main', + ], + allow_failure=False, + ) + + def testChangeFlagWithLockfile(self): + # Add module 'sss' to the registry with dep on 'aaa' + self.main_registry.createCcModule('sss', '1.3', {'aaa': '1.1'}) + # Create a project with deps on 'sss' + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "sss", version = "1.3")', + ], + ) + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = ["@sss//:lib_sss"],', + ')', + ], + ) + self.RunBazel( + [ + 'build', + '--nobuild', + '//:main', + ], + allow_failure=False, + ) + + # Change registry -> update 'sss' module file (corrupt it) + module_dir = self.main_registry.root.joinpath('modules', 'sss', '1.3') + scratchFile(module_dir.joinpath('MODULE.bazel'), ['whatever!']) + + # Clean bazel to empty any cache of the deps tree + self.RunBazel(['clean', '--expunge']) + # Running with the lockfile, but adding a flag should cause resolution rerun + exit_code, _, stderr = self.RunBazel( + [ + 'build', + '--nobuild', + '--check_direct_dependencies=error', + '//:main', + ], + allow_failure=True, + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + "ERROR: sss@1.3/MODULE.bazel:1:9: invalid character: '!'", stderr + ) + def testWorkspaceEvaluatedBzlCanSeeRootModuleMappings(self): self.ScratchFile( 'MODULE.bazel',