diff --git a/site/en/_book.yaml b/site/en/_book.yaml index 1b3e1d53268e02..19bdc921b9d8aa 100644 --- a/site/en/_book.yaml +++ b/site/en/_book.yaml @@ -125,6 +125,21 @@ upper_tabs: path: /run/scripts - title: Client/server implementation path: /run/client-server + - heading: External dependencies + - title: Overview + path: /external/overview + - title: Bazel modules + path: /external/module + - title: Bazel registries + path: /external/registry + - title: Module extensions + path: /external/extension + - title: Module lockfile + path: /external/lockfile + - title: Bzlmod migration guide + path: /external/migration + - title: Advanced topics + path: /external/advanced - heading: Querying your build - title: Query quickstart path: /query/quickstart diff --git a/site/en/external/lockfile.md b/site/en/external/lockfile.md new file mode 100644 index 00000000000000..39ed0e8e680a2a --- /dev/null +++ b/site/en/external/lockfile.md @@ -0,0 +1,191 @@ +Project: /_project.yaml +Book: /_book.yaml +keywords: product:Bazel,lockfile,Bzlmod + +# Bazel Lockfile + +{% include "_buttons.html" %} + +The lockfile feature in Bazel enables the recording of specific versions or +dependencies of software libraries or packages required by a project. It +achieves this by storing the result of module resolution and extension +evaluation. The lockfile promotes reproducible builds, ensuring consistent +development environments. Additionally, it enhances build efficiency by allowing +Bazel to skip the resolution process when there are no changes in project +dependencies. Furthermore, the lockfile improves stability by preventing +unexpected updates or breaking changes in external libraries, thereby reducing +the risk of introducing bugs. + +## Lockfile Generation {:#lockfile-generation} + +The lockfile is generated under the workspace root with the name +`MODULE.bazel.lock`. It is created or updated during the build process, +specifically after module resolution and extension evaluation. The lockfile +captures the current state of the project, including the MODULE file, flags, +overrides, and other relevant information. Importantly, it only includes +dependencies that are included in the current invocation of the build. + +When changes occur in the project that affect its dependencies, the lockfile is +automatically updated to reflect the new state. This ensures that the lockfile +remains focused on the specific set of dependencies required for the current +build, providing an accurate representation of the project's resolved +dependencies. + +## Lockfile Usage {:#lockfile-usage} + +The lockfile can be controlled by the flag +[`--lockfile_mode`](/reference/command-line-reference#flag--lockfile_mode) to +customize the behavior of Bazel when the project state differs from the +lockfile. The available modes are: + +* `update` (Default): If the project state matches the lockfile, the + resolution result is immediately returned from the lockfile. Otherwise, + resolution is executed, and the lockfile is updated to reflect the current + state. +* `error`: If the project state matches the lockfile, the resolution result is + returned from the lockfile. Otherwise, Bazel throws an error indicating the + variations between the project and the lockfile. This mode is particularly + useful when you want to ensure that your project's dependencies remain + unchanged, and any differences are treated as errors. +* `off`: The lockfile is not checked at all. + +## Lockfile Benefits {:#lockfile-benefits} + +The lockfile offers several benefits and can be utilized in various ways: + +- **Reproducible builds.** By capturing the specific versions or dependencies + of software libraries, the lockfile ensures that builds are reproducible + across different environments and over time. Developers can rely on + consistent and predictable results when building their projects. + +- **Efficient resolution skipping.** The lockfile enables Bazel to skip the + resolution process if there are no changes in the project dependencies since + the last build. This significantly improves build efficiency, especially in + scenarios where resolution can be time-consuming. + +- **Stability and risk reduction.** The lockfile helps maintain stability by + preventing unexpected updates or breaking changes in external libraries. By + locking the dependencies to specific versions, the risk of introducing bugs + due to incompatible or untested updates is reduced. + +## Lockfile Contents {:#lockfile-contents} + +The lockfile contains all the necessary information to determine whether the +project state has changed. It also includes the result of building the project +in the current state. The lockfile consists of two main parts: + +1. Inputs of the module resolution, such as `moduleFileHash`, `flags` and + `localOverrideHashes`, as well as the output of the resolution, which is + `moduleDepGraph`. +2. For each module extension, the lockfile includes inputs that affect it, + represented by `transitiveDigest`, and the output of running that extension + referred to as `generatedRepoSpecs` + +Here is an example that demonstrates the structure of the lockfile, along with +explanations for each section: + +```json +{ + "lockFileVersion": 1, + "moduleFileHash": "b0f47b98a67ee15f9.......8dff8721c66b721e370", + "flags": { + "cmdRegistries": [ + "https://bcr.bazel.build/" + ], + "cmdModuleOverrides": {}, + "allowedYankedVersions": [], + "envVarAllowedYankedVersions": "", + "ignoreDevDependency": false, + "directDependenciesMode": "WARNING", + "compatibilityMode": "ERROR" + }, + "localOverrideHashes": { + "bazel_tools": "b5ae1fa37632140aff8.......15c6fe84a1231d6af9" + }, + "moduleDepGraph": { + "": { + "name": "", + "version": "", + "executionPlatformsToRegister": [], + "toolchainsToRegister": [], + "extensionUsages": [ + { + "extensionBzlFile": "extension.bzl", + "extensionName": "lockfile_ext" + } + ], + ... + } + }, + "moduleExtensions": { + "//:extension.bzl%lockfile_ext": { + "transitiveDigest": "oWDzxG/aLnyY6Ubrfy....+Jp6maQvEPxn0pBM=", + "generatedRepoSpecs": { + "hello": { + "bzlFile": "@@//:extension.bzl", + ... + } + } + } + } +} +``` + +### Module File Hash {:#module-file-hash} + +The `moduleFileHash` represents the hash of the `MODULE.bazel` file contents. If +any changes occur in this file, the hash value differs. + +### Flags {:#flags} + +The `Flags` object stores all the flags that can affect the resolution result. + +### Local Override Hashes {:#local-override-hashes} + +If the root module includes `local_path_overrides`, this section stores the hash +of the `MODULE.bazel` file in the local repository. It allows tracking changes +to this dependency. + +### Module Dependency Graph {:#module-dep-graph} + +The `moduleDepGraph` represents the result of the resolution process using the +inputs mentioned above. It forms the dependency graph of all the modules +required to run the project. + +### Module Extensions {:#module-extensions} + +The `moduleExtensions` section is a map that includes only the extensions used +in the current invocation or previously invoked, while excluding any extensions +that are no longer utilized. In other words, if an extension is not being used +anymore across the dependency graph, it is removed from the `moduleExtensions` +map. + +Each entry in this map corresponds to a used extension and is identified by its +containing file and name. The corresponding value for each entry contains the +relevant information associated with that extension: + +1. The `transitiveDigest` the digest of the extension implementation and its + transitive .bzl files. +2. The `generatedRepoSpecs` the result of running that extension with the + current input. + +An additional factor that can affect the extension results is their _usages_. +Although not stored in the lockfile, the usages are considered when comparing +the current state of the extension with the one in the lockfile. + +## Best Practices {:#best-practices} + +To maximize the benefits of the lockfile feature, consider the following best +practices: + +* Regularly update the lockfile to reflect changes in project dependencies or + configuration. This ensures that subsequent builds are based on the most + up-to-date and accurate set of dependencies. + +* Include the lockfile in version control to facilitate collaboration and + ensure that all team members have access to the same lockfile, promoting + consistent development environments across the project. + +By following these best practices, you can effectively utilize the lockfile +feature in Bazel, leading to more efficient, reliable, and collaborative +software development workflows. \ No newline at end of file diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD index 3c29ea712d2909..bd7402b7f9d77a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD @@ -142,6 +142,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info", "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:bazel_lockfile_module", "//src/main/java/com/google/devtools/build/lib/bazel/coverage", "//src/main/java/com/google/devtools/build/lib/bazel/debug:workspace-rule-module", "//src/main/java/com/google/devtools/build/lib/bazel/repository", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java index e35b11a8386f5d..7fb055d87e2c30 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java @@ -63,6 +63,7 @@ public final class Bazel { com.google.devtools.build.lib.bazel.repository.RepositoryResolvedModule.class, com.google.devtools.build.lib.bazel.repository.CacheHitReportingModule.class, com.google.devtools.build.lib.bazel.SpawnLogModule.class, + com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileModule.class, com.google.devtools.build.lib.outputfilter.OutputFilteringModule.class, com.google.devtools.build.lib.worker.WorkerModule.class, com.google.devtools.build.lib.runtime.CacheFileDigestsModule.class, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 5e87645a450f50..04b0a36d7a4a84 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -151,7 +151,7 @@ public class BazelRepositoryModule extends BlazeModule { private final AtomicBoolean ignoreDevDeps = new AtomicBoolean(false); private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING; private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR; - private LockfileMode bazelLockfileMode = LockfileMode.OFF; + private LockfileMode bazelLockfileMode = LockfileMode.UPDATE; private List allowedYankedVersions = ImmutableList.of(); private SingleExtensionEvalFunction singleExtensionEvalFunction; @@ -265,8 +265,7 @@ public ResolutionReason getResolutionReason() { .addSkyFunction( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, directories.getWorkspace(), builtinModules)) - .addSkyFunction( - SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(directories.getWorkspace())) + .addSkyFunction(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .addSkyFunction( SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace())) .addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction()) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 4b45705309b979..61015909ecdbb1 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -85,6 +85,25 @@ java_library( ], ) +java_library( + name = "bazel_lockfile_module", + srcs = ["BazelLockFileModule.java"], + deps = [ + ":resolution", + ":resolution_impl", + "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", + "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//third_party:flogger", + "//third_party:gson", + "//third_party:guava", + "//third_party:jsr305", + ], +) + java_library( name = "resolution", srcs = [ @@ -92,14 +111,17 @@ java_library( "ArchiveOverride.java", "BazelDepGraphValue.java", "BazelLockFileValue.java", + "BazelModuleResolutionEvent.java", "BazelModuleResolutionValue.java", "BzlmodFlagsAndEnvVars.java", "GitOverride.java", "InterimModule.java", "LocalPathOverride.java", + "LockFileModuleExtension.java", "Module.java", "ModuleBase.java", "ModuleExtensionEvalStarlarkThreadContext.java", + "ModuleExtensionResolutionEvent.java", "ModuleFileValue.java", "ModuleOverride.java", "MultipleVersionOverride.java", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index f99b33b627a371..f6d863f7e0cc8c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -35,14 +35,12 @@ import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue; -import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; -import java.util.List; import java.util.Map; import java.util.Map.Entry; import javax.annotation.Nullable; @@ -54,16 +52,12 @@ */ public class BazelDepGraphFunction implements SkyFunction { - private final Path rootDirectory; - - public BazelDepGraphFunction(Path rootDirectory) { - this.rootDirectory = rootDirectory; - } + public BazelDepGraphFunction() {} @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { + throws BazelDepGraphFunctionException, InterruptedException { RootModuleFileValue root = (RootModuleFileValue) env.getValue(ModuleFileValue.KEY_FOR_ROOT_MODULE); if (root == null) { @@ -74,12 +68,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) ImmutableMap localOverrideHashes = null; ImmutableMap depGraph = null; BzlmodFlagsAndEnvVars flags = null; + BazelLockFileValue lockfile = null; // If the module has not changed (has the same contents and flags as the lockfile), // read the dependency graph from the lock file, else run resolution and update lockfile if (!lockfileMode.equals(LockfileMode.OFF)) { - BazelLockFileValue lockFile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); - if (lockFile == null) { + lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); + if (lockfile == null) { return null; } flags = getFlagsAndEnvVars(env); @@ -87,17 +82,17 @@ public SkyValue compute(SkyKey skyKey, Environment env) return null; } localOverrideHashes = getLocalOverridesHashes(root.getOverrides(), env); - if (localOverrideHashes == null) { // trying to read override module + if (localOverrideHashes == null) { // still reading an override "module" return null; } - if (root.getModuleFileHash().equals(lockFile.getModuleFileHash()) - && flags.equals(lockFile.getFlags()) - && localOverrideHashes.equals(lockFile.getLocalOverrideHashes())) { - depGraph = lockFile.getModuleDepGraph(); + if (root.getModuleFileHash().equals(lockfile.getModuleFileHash()) + && flags.equals(lockfile.getFlags()) + && localOverrideHashes.equals(lockfile.getLocalOverrideHashes())) { + depGraph = lockfile.getModuleDepGraph(); } else if (lockfileMode.equals(LockfileMode.ERROR)) { - List diffLockfile = - lockFile.getDiffLockfile(root.getModuleFileHash(), localOverrideHashes, flags); + ImmutableList diffLockfile = + lockfile.getModuleAndFlagsDiff(root.getModuleFileHash(), localOverrideHashes, flags); throw new BazelDepGraphFunctionException( ExternalDepsException.withMessage( Code.BAD_MODULE, @@ -114,22 +109,34 @@ public SkyValue compute(SkyKey skyKey, Environment env) return null; } depGraph = selectionResult.getResolvedDepGraph(); - if (lockfileMode.equals(LockfileMode.UPDATE)) { - BazelLockFileFunction.updateLockedModule( - rootDirectory, root.getModuleFileHash(), flags, localOverrideHashes, depGraph); - } } ImmutableMap canonicalRepoNameLookup = depGraph.keySet().stream() .collect(toImmutableMap(ModuleKey::getCanonicalRepoName, key -> key)); - ImmutableTable extensionUsagesById = - getExtensionUsagesById(depGraph); + ImmutableTable extensionUsagesById; + try { + extensionUsagesById = getExtensionUsagesById(depGraph); + } catch (ExternalDepsException e) { + throw new BazelDepGraphFunctionException(e, Transience.PERSISTENT); + } ImmutableBiMap extensionUniqueNames = calculateUniqueNameForUsedExtensionId(extensionUsagesById); + if (!lockfileMode.equals(LockfileMode.OFF)) { + BazelLockFileValue updateLockfile = + lockfile.toBuilder() + .setModuleFileHash(root.getModuleFileHash()) + .setFlags(flags) + .setLocalOverrideHashes(localOverrideHashes) + .setModuleDepGraph(depGraph) + .build(); + env.getListener() + .post(BazelModuleResolutionEvent.create(updateLockfile, extensionUsagesById)); + } + return BazelDepGraphValue.create( depGraph, canonicalRepoNameLookup, @@ -200,8 +207,9 @@ static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws Interrup * For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can group * all usages by the label + name (the ModuleExtensionId). */ - private ImmutableTable getExtensionUsagesById( - ImmutableMap depGraph) throws BazelDepGraphFunctionException { + public static ImmutableTable + getExtensionUsagesById(ImmutableMap depGraph) + throws ExternalDepsException { ImmutableTable.Builder extensionUsagesTableBuilder = ImmutableTable.builder(); for (Module module : depGraph.values()) { @@ -214,26 +222,20 @@ private ImmutableTable getEx try { moduleExtensionId = ModuleExtensionId.create( - labelConverter.convert(usage.getExtensionBzlFile()), - usage.getExtensionName(), - usage.getIsolationKey()); + labelConverter.convert(usage.getExtensionBzlFile()), usage.getExtensionName(), usage.getIsolationKey()); } catch (LabelSyntaxException e) { - throw new BazelDepGraphFunctionException( - ExternalDepsException.withCauseAndMessage( - Code.BAD_MODULE, - e, - "invalid label for module extension found at %s", - usage.getLocation()), - Transience.PERSISTENT); + throw ExternalDepsException.withCauseAndMessage( + Code.BAD_MODULE, + e, + "invalid label for module extension found at %s", + usage.getLocation()); } if (!moduleExtensionId.getBzlFileLabel().getRepository().isVisible()) { - throw new BazelDepGraphFunctionException( - ExternalDepsException.withMessage( - Code.BAD_MODULE, - "invalid label for module extension found at %s: no repo visible as '@%s' here", - usage.getLocation(), - moduleExtensionId.getBzlFileLabel().getRepository().getName()), - Transience.PERSISTENT); + throw ExternalDepsException.withMessage( + Code.BAD_MODULE, + "invalid label for module extension found at %s: no repo visible as '@%s' here", + usage.getLocation(), + moduleExtensionId.getBzlFileLabel().getRepository().getName()); } extensionUsagesTableBuilder.put(moduleExtensionId, module.getKey(), usage); } @@ -250,31 +252,12 @@ private ImmutableBiMap calculateUniqueNameForUsedExte // not start with a tilde. RepositoryName repository = id.getBzlFileLabel().getRepository(); String nonEmptyRepoPart = repository.isMain() ? "_main" : repository.getName(); - // When using a namespace, prefix the extension name with "_" to distinguish the prefix from - // those generated by non-namespaced extension usages. Extension names are identified by their - // Starlark identifier, which in the case of an exported symbol cannot start with "_". - // We also include whether the isolated usage is a dev usage as well as its index in the - // MODULE.bazel file to ensure that canonical repository names don't change depending on - // whether dev dependencies are ignored. This removes potential for confusion and also - // prevents unnecessary refetches when --ignore_dev_dependency is toggled. - String bestName = - id.getIsolationKey() - .map( - namespace -> - String.format( - "%s~_%s~%s~%s~%s%d", - nonEmptyRepoPart, - id.getExtensionName(), - namespace.getModule().getName(), - namespace.getModule().getVersion(), - namespace.isDevUsage() ? "dev" : "", - namespace.getIsolatedUsageIndex())) - .orElse(nonEmptyRepoPart + "~" + id.getExtensionName()); + String bestName = nonEmptyRepoPart + "~" + id.getExtensionName(); if (extensionUniqueNames.putIfAbsent(bestName, id) == null) { continue; } int suffix = 2; - while (extensionUniqueNames.putIfAbsent(bestName + "~" + suffix, id) != null) { + while (extensionUniqueNames.putIfAbsent(bestName + suffix, id) != null) { suffix++; } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index cfeb31fa0f57bb..41cff393000ac1 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.FileValue; -import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction.BazelDepGraphFunctionException; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; @@ -35,7 +34,7 @@ import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; -import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; import java.io.FileNotFoundException; import java.io.IOException; import javax.annotation.Nullable; @@ -52,12 +51,14 @@ public class BazelLockFileFunction implements SkyFunction { ImmutableList.of(), ImmutableMap.of(), ImmutableList.of(), "", false, "", ""); private static final BazelLockFileValue EMPTY_LOCKFILE = - BazelLockFileValue.create( - BazelLockFileValue.LOCK_FILE_VERSION, - "", - EMPTY_FLAGS, - ImmutableMap.of(), - ImmutableMap.of()); + BazelLockFileValue.builder() + .setLockFileVersion(BazelLockFileValue.LOCK_FILE_VERSION) + .setModuleFileHash("") + .setFlags(EMPTY_FLAGS) + .setLocalOverrideHashes(ImmutableMap.of()) + .setModuleDepGraph(ImmutableMap.of()) + .setModuleExtensions(ImmutableMap.of()) + .build(); public BazelLockFileFunction(Path rootDirectory) { this.rootDirectory = rootDirectory; @@ -66,7 +67,7 @@ public BazelLockFileFunction(Path rootDirectory) { @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { + throws BazelLockfileFunctionException, InterruptedException { RootedPath lockfilePath = RootedPath.toRootedPath(Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); @@ -75,48 +76,35 @@ public SkyValue compute(SkyKey skyKey, Environment env) return null; } + try { + return getLockfileValue(lockfilePath); + } catch (IOException | JsonSyntaxException | NullPointerException e) { + throw new BazelLockfileFunctionException( + ExternalDepsException.withMessage( + Code.BAD_MODULE, + "Failed to read and parse the MODULE.bazel.lock file with error: %s." + + " Try deleting it and rerun the build.", + e.getMessage()), + Transience.PERSISTENT); + } + } + + public static BazelLockFileValue getLockfileValue(RootedPath lockfilePath) throws IOException { BazelLockFileValue bazelLockFileValue; try { String json = FileSystemUtils.readContent(lockfilePath.asPath(), UTF_8); bazelLockFileValue = LOCKFILE_GSON.fromJson(json, BazelLockFileValue.class); } catch (FileNotFoundException e) { bazelLockFileValue = EMPTY_LOCKFILE; - } catch (IOException ex) { - throw new JsonIOException("Failed to read or parse module-lock file", ex); } return bazelLockFileValue; } - /** - * Updates the stored module in the lock file (ModuleHash, Flags & Dependency graph) - * - * @param moduleFileHash The hash of the current module file - * @param resolvedDepGraph The resolved dependency graph from the module file - */ - public static void updateLockedModule( - Path rootDirectory, - String moduleFileHash, - BzlmodFlagsAndEnvVars flags, - ImmutableMap localOverrideHashes, - ImmutableMap resolvedDepGraph) - throws BazelDepGraphFunctionException { - RootedPath lockfilePath = - RootedPath.toRootedPath(Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); - BazelLockFileValue value = - BazelLockFileValue.create( - BazelLockFileValue.LOCK_FILE_VERSION, - moduleFileHash, - flags, - localOverrideHashes, - resolvedDepGraph); - try { - FileSystemUtils.writeContent(lockfilePath.asPath(), UTF_8, LOCKFILE_GSON.toJson(value)); - } catch (IOException e) { - throw new BazelDepGraphFunctionException( - ExternalDepsException.withCauseAndMessage( - Code.BAD_MODULE, e, "Unable to update module-lock file"), - Transience.PERSISTENT); + static final class BazelLockfileFunctionException extends SkyFunctionException { + + BazelLockfileFunctionException(ExternalDepsException cause, Transience transience) { + super(cause, transience); } } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java new file mode 100644 index 00000000000000..5bd3b40a06bd16 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java @@ -0,0 +1,155 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static com.google.devtools.build.lib.bazel.bzlmod.GsonTypeAdapterUtil.LOCKFILE_GSON; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableMap; +import com.google.common.eventbus.Subscribe; +import com.google.common.flogger.GoogleLogger; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; +import com.google.devtools.build.lib.cmdline.LabelConstants; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.gson.JsonSyntaxException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; + +/** + * Module collecting Bazel module and module extensions resolution results and updating the + * lockfile. + */ +public class BazelLockFileModule extends BlazeModule { + + private Path workspaceRoot; + @Nullable private BazelModuleResolutionEvent moduleResolutionEvent; + private final List extensionResolutionEvents = new ArrayList<>(); + + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + @Override + public void beforeCommand(CommandEnvironment env) { + workspaceRoot = env.getWorkspace(); + RepositoryOptions options = env.getOptions().getOptions(RepositoryOptions.class); + if (options.lockfileMode.equals(LockfileMode.UPDATE)) { + env.getEventBus().register(this); + } + } + + @Override + public void afterCommand() throws AbruptExitException { + if (moduleResolutionEvent == null && extensionResolutionEvents.isEmpty()) { + return; // nothing changed, do nothing! + } + + RootedPath lockfilePath = + RootedPath.toRootedPath(Root.fromPath(workspaceRoot), LabelConstants.MODULE_LOCKFILE_NAME); + + // Create an updated version of the lockfile with the events updates + BazelLockFileValue lockfile; + if (moduleResolutionEvent != null) { + lockfile = moduleResolutionEvent.getLockfileValue(); + } else { + // Read the existing lockfile (if none exists, will get an empty lockfile value) + try { + lockfile = BazelLockFileFunction.getLockfileValue(lockfilePath); + } catch (IOException | JsonSyntaxException | NullPointerException e) { + logger.atSevere().withCause(e).log( + "Failed to read and parse the MODULE.bazel.lock file with error: %s." + + " Try deleting it and rerun the build.", + e.getMessage()); + return; + } + } + lockfile = + lockfile.toBuilder() + .setModuleExtensions(combineModuleExtensions(lockfile.getModuleExtensions())) + .build(); + + // Write the new value to the file + updateLockfile(lockfilePath, lockfile); + this.moduleResolutionEvent = null; + this.extensionResolutionEvents.clear(); + } + + /** + * Combines the old extensions stored in the lockfile -that are still used- with the new + * extensions from the events (if any) + * + * @param oldModuleExtensions Module extensions stored in the current lockfile + */ + private ImmutableMap combineModuleExtensions( + ImmutableMap oldModuleExtensions) { + ImmutableMap.Builder updatedExtensionMap = + ImmutableMap.builder(); + + // This event being null means that no changes occurred to the usages of the stored extensions, + // hence no changes to any module resulted in re-running resolution. So we can just add all the + // old stored extensions. Otherwise, check the usage of each one. + if (moduleResolutionEvent == null) { + updatedExtensionMap.putAll(oldModuleExtensions); + } else { + // Add the old extensions (stored in the lockfile) only if it still has a usage somewhere + for (Map.Entry extensionEntry : + oldModuleExtensions.entrySet()) { + if (moduleResolutionEvent.getExtensionUsagesById().containsRow(extensionEntry.getKey())) { + updatedExtensionMap.put(extensionEntry); + } + } + } + + // Add the new resolved extensions + for (ModuleExtensionResolutionEvent extensionEvent : extensionResolutionEvents) { + updatedExtensionMap.put(extensionEvent.getExtensionId(), extensionEvent.getModuleExtension()); + } + return updatedExtensionMap.buildKeepingLast(); + } + + /** + * Updates the data stored in the lockfile (MODULE.bazel.lock) + * + * @param lockfilePath Rooted path to lockfile + * @param updatedLockfile The updated lockfile data to save + */ + public static void updateLockfile(RootedPath lockfilePath, BazelLockFileValue updatedLockfile) { + try { + FileSystemUtils.writeContent( + lockfilePath.asPath(), UTF_8, LOCKFILE_GSON.toJson(updatedLockfile)); + } catch (IOException e) { + logger.atSevere().withCause(e).log( + "Error while updating MODULE.bazel.lock file: %s", e.getMessage()); + } + } + + @Subscribe + public void bazelModuleResolved(BazelModuleResolutionEvent moduleResolutionEvent) { + this.moduleResolutionEvent = moduleResolutionEvent; + } + + @Subscribe + public void moduleExtensionResolved(ModuleExtensionResolutionEvent extensionResolutionEvent) { + this.extensionResolutionEvents.add(extensionResolutionEvent); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java index 6f32405f61dd2c..c0d0c06aed41e4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java @@ -17,13 +17,15 @@ import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Map; /** @@ -33,20 +35,16 @@ */ @AutoValue @GenerateTypeAdapter -public abstract class BazelLockFileValue implements SkyValue { +public abstract class BazelLockFileValue implements SkyValue, Postable { public static final int LOCK_FILE_VERSION = 1; @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_LOCK_FILE; - public static BazelLockFileValue create( - int lockFileVersion, - String moduleFileHash, - BzlmodFlagsAndEnvVars flags, - ImmutableMap localOverrideHashes, - ImmutableMap moduleDepGraph) { - return new AutoValue_BazelLockFileValue( - lockFileVersion, moduleFileHash, flags, localOverrideHashes, moduleDepGraph); + static Builder builder() { + return new AutoValue_BazelLockFileValue.Builder() + .setLockFileVersion(LOCK_FILE_VERSION) + .setModuleExtensions(ImmutableMap.of()); } /** Current version of the lock file */ @@ -64,27 +62,73 @@ public static BazelLockFileValue create( /** The post-selection dep graph retrieved from the lock file. */ public abstract ImmutableMap getModuleDepGraph(); + /** Mapping the extension id to the module extension data */ + public abstract ImmutableMap getModuleExtensions(); + + public abstract Builder toBuilder(); + + /** Builder type for {@link BazelLockFileValue}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setLockFileVersion(int value); + + public abstract Builder setModuleFileHash(String value); + + public abstract Builder setFlags(BzlmodFlagsAndEnvVars value); + + public abstract Builder setLocalOverrideHashes(ImmutableMap value); + + public abstract Builder setModuleDepGraph(ImmutableMap value); + + public abstract Builder setModuleExtensions( + ImmutableMap value); + + public abstract BazelLockFileValue build(); + } + /** Returns the difference between the lockfile and the current module & flags */ - public ArrayList getDiffLockfile( + public ImmutableList getModuleAndFlagsDiff( String moduleFileHash, ImmutableMap localOverrideHashes, BzlmodFlagsAndEnvVars flags) { - ArrayList diffLockfile = new ArrayList<>(); + ImmutableList.Builder moduleDiff = new ImmutableList.Builder<>(); if (!moduleFileHash.equals(getModuleFileHash())) { - diffLockfile.add("the root MODULE.bazel has been modified"); + moduleDiff.add("the root MODULE.bazel has been modified"); } - diffLockfile.addAll(getFlags().getDiffFlags(flags)); + moduleDiff.addAll(getFlags().getDiffFlags(flags)); for (Map.Entry entry : localOverrideHashes.entrySet()) { String currentValue = entry.getValue(); String lockfileValue = getLocalOverrideHashes().get(entry.getKey()); // If the lockfile value is null, the module hash would be different anyway if (lockfileValue != null && !currentValue.equals(lockfileValue)) { - diffLockfile.add( + moduleDiff.add( "The MODULE.bazel file has changed for the overriden module: " + entry.getKey()); } } + return moduleDiff.build(); + } - return diffLockfile; + public ImmutableList getModuleExtensionDiff( + LockFileModuleExtension lockedExtension, + ImmutableMap lockedExtensionUsages, + ModuleExtensionId extensionId, + byte[] transitiveDigest, + ImmutableMap extensionUsages) { + ImmutableList.Builder extDiff = new ImmutableList.Builder<>(); + if (lockedExtension == null) { + extDiff.add("The module extension '" + extensionId + "' does not exist in the lockfile"); + } else { + if (!Arrays.equals(transitiveDigest, lockedExtension.getBzlTransitiveDigest())) { + extDiff.add( + "The implementation of the extension '" + + extensionId + + "' or one of its transitive .bzl files has changed"); + } + if (!extensionUsages.equals(lockedExtensionUsages)) { + extDiff.add("The usages of the extension named '" + extensionId + "' has changed"); + } + } + return extDiff.build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java new file mode 100644 index 00000000000000..81a0dbaa144b9c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java @@ -0,0 +1,39 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableTable; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; + +/** + * After resolving bazel module this event is sent from {@link BazelDepGraphFunction} holding the + * lockfile value with module updates, and the module extension usages. It will be received in + * {@link BazelLockFileModule} to be used to update the lockfile content + */ +@AutoValue +public abstract class BazelModuleResolutionEvent implements Postable { + + public static BazelModuleResolutionEvent create( + BazelLockFileValue lockFileValue, + ImmutableTable extensionUsagesById) { + return new AutoValue_BazelModuleResolutionEvent(lockFileValue, extensionUsagesById); + } + + public abstract BazelLockFileValue getLockfileValue(); + + public abstract ImmutableTable + getExtensionUsagesById(); +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java index f5f483f716c04b..a69e53791f247f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// package com.google.devtools.build.lib.bazel.bzlmod; @@ -19,7 +18,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; -import java.util.ArrayList; /** Stores the values of flags and environment variables that affect the resolution */ @AutoValue @@ -65,8 +63,8 @@ public static BzlmodFlagsAndEnvVars create( /** Error level of bazel compatability check */ public abstract String compatibilityMode(); - public ArrayList getDiffFlags(BzlmodFlagsAndEnvVars flags) { - ArrayList diffFlags = new ArrayList<>(); + public ImmutableList getDiffFlags(BzlmodFlagsAndEnvVars flags) { + ImmutableList.Builder diffFlags = new ImmutableList.Builder<>(); if (!flags.cmdRegistries().equals(cmdRegistries())) { diffFlags.add("the value of --registry flag has been modified"); } @@ -89,6 +87,6 @@ public ArrayList getDiffFlags(BzlmodFlagsAndEnvVars flags) { if (!flags.compatibilityMode().equals(compatibilityMode())) { diffFlags.add("the value of --check_bazel_compatibility flag has been modified"); } - return diffFlags; + return diffFlags.build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java index 15518989781133..c558e806e9d6d3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java @@ -23,6 +23,8 @@ import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParseException; @@ -36,6 +38,7 @@ import java.io.IOException; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; +import java.util.Base64; import java.util.List; import java.util.Optional; import javax.annotation.Nullable; @@ -96,6 +99,45 @@ public ModuleKey read(JsonReader jsonReader) throws IOException { } }; + // TODO(salmasamy) need to handle "isolated" in module extensions when it is stable + public static final TypeAdapter MODULE_EXTENSION_ID_TYPE_ADAPTER = + new TypeAdapter<>() { + @Override + public void write(JsonWriter jsonWriter, ModuleExtensionId moduleExtId) throws IOException { + jsonWriter.value(moduleExtId.getBzlFileLabel() + "%" + moduleExtId.getExtensionName()); + } + + @Override + public ModuleExtensionId read(JsonReader jsonReader) throws IOException { + String jsonString = jsonReader.nextString(); + // [0] is labelString, [1] is extensionName + List extIdParts = Splitter.on("%").splitToList(jsonString); + try { + return ModuleExtensionId.create( + Label.parseCanonical(extIdParts.get(0)), extIdParts.get(1), Optional.empty()); + } catch (LabelSyntaxException e) { + throw new JsonParseException( + String.format( + "Unable to parse ModuleExtensionID bzl file label: '%s' from the lockfile", + extIdParts.get(0)), + e); + } + } + }; + + public static final TypeAdapter BYTE_ARRAY_TYPE_ADAPTER = + new TypeAdapter<>() { + @Override + public void write(JsonWriter jsonWriter, byte[] value) throws IOException { + jsonWriter.value(Base64.getEncoder().encodeToString(value)); + } + + @Override + public byte[] read(JsonReader jsonReader) throws IOException { + return Base64.getDecoder().decode(jsonReader.nextString()); + } + }; + public static final TypeAdapterFactory OPTIONAL = new TypeAdapterFactory() { @Nullable @@ -149,6 +191,8 @@ public Optional read(JsonReader jsonReader) throws IOException { public static final Gson LOCKFILE_GSON = new GsonBuilder() .setPrettyPrinting() + .disableHtmlEscaping() + .enableComplexMapKeySerialization() .registerTypeAdapterFactory(GenerateTypeAdapter.FACTORY) .registerTypeAdapterFactory(DICT) .registerTypeAdapterFactory(IMMUTABLE_MAP) @@ -158,7 +202,9 @@ public Optional read(JsonReader jsonReader) throws IOException { .registerTypeAdapterFactory(OPTIONAL) .registerTypeAdapter(Version.class, VERSION_TYPE_ADAPTER) .registerTypeAdapter(ModuleKey.class, MODULE_KEY_TYPE_ADAPTER) + .registerTypeAdapter(ModuleExtensionId.class, MODULE_EXTENSION_ID_TYPE_ADAPTER) .registerTypeAdapter(AttributeValues.class, new AttributeValuesAdapter()) + .registerTypeAdapter(byte[].class, BYTE_ARRAY_TYPE_ADAPTER) .create(); private GsonTypeAdapterUtil() {} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java new file mode 100644 index 00000000000000..5431d6cf0fabe6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java @@ -0,0 +1,40 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; + +/** + * This object serves as a container for the transitive digest (obtained from transitive .bzl files) + * and the generated repositories from evaluating a module extension. Its purpose is to store this + * information within the lockfile. + */ +@AutoValue +@GenerateTypeAdapter +public abstract class LockFileModuleExtension implements Postable { + + @SuppressWarnings("mutable") + public abstract byte[] getBzlTransitiveDigest(); + + public abstract ImmutableMap getGeneratedRepoSpecs(); + + public static LockFileModuleExtension create( + byte[] transitiveDigest, ImmutableMap generatedRepoSpecs) { + return new AutoValue_LockFileModuleExtension(transitiveDigest, generatedRepoSpecs); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java new file mode 100644 index 00000000000000..4013e7bb4c5965 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java @@ -0,0 +1,36 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; + +/** + * After evaluating any module extension this event is sent from {@link SingleExtensionEvalFunction} + * holding the extension id and the resolution data LockFileModuleExtension. It will be received in + * {@link BazelLockFileModule} to be used to update the lockfile content + */ +@AutoValue +public abstract class ModuleExtensionResolutionEvent implements Postable { + + public static ModuleExtensionResolutionEvent create( + ModuleExtensionId extensionId, LockFileModuleExtension lockfileModuleExtension) { + return new AutoValue_ModuleExtensionResolutionEvent(extensionId, lockfileModuleExtension); + } + + public abstract ModuleExtensionId getExtensionId(); + + public abstract LockFileModuleExtension getModuleExtension(); +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java index b76131c9761da0..a3ee9af7e88468 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java @@ -18,10 +18,15 @@ import static com.google.common.collect.ImmutableBiMap.toImmutableBiMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableTable; import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.cmdline.BazelModuleContext; +import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -29,6 +34,7 @@ import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException; import com.google.devtools.build.lib.runtime.ProcessWrapper; import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor; +import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; import com.google.devtools.build.lib.skyframe.BzlLoadFunction; import com.google.devtools.build.lib.skyframe.BzlLoadFunction.BzlLoadFailedException; @@ -42,6 +48,7 @@ import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Map; import java.util.Map.Entry; import java.util.function.Function; @@ -94,7 +101,7 @@ public void setRepositoryRemoteExecutor(RepositoryRemoteExecutor repositoryRemot @Nullable @Override public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { + throws SingleExtensionEvalFunctionException, InterruptedException { StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env); if (starlarkSemantics == null) { return null; @@ -108,37 +115,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) } Location sampleUsageLocation = usagesValue.getExtensionUsages().values().iterator().next().getLocation(); - - // Check that the .bzl label isn't crazy. - try { - BzlLoadFunction.checkValidLoadLabel( - extensionId.getBzlFileLabel(), /*fromBuiltinsRepo=*/ false); - } catch (LabelSyntaxException e) { - throw new SingleExtensionEvalFunctionException( - ExternalDepsException.withCauseAndMessage( - Code.BAD_MODULE, e, "invalid module extension label"), - Transience.PERSISTENT); - } - - // Load the .bzl file pointed to by the label. - BzlLoadValue bzlLoadValue; - try { - bzlLoadValue = - (BzlLoadValue) - env.getValueOrThrow( - BzlLoadValue.keyForBzlmod(extensionId.getBzlFileLabel()), - BzlLoadFailedException.class); - } catch (BzlLoadFailedException e) { - throw new SingleExtensionEvalFunctionException( - ExternalDepsException.withCauseAndMessage( - Code.BAD_MODULE, - e, - "Error loading '%s' for module extensions, requested by %s: %s", - extensionId.getBzlFileLabel(), - sampleUsageLocation, - e.getMessage()), - Transience.PERSISTENT); - } + BzlLoadValue bzlLoadValue = + loadBzlFile(extensionId.getBzlFileLabel(), sampleUsageLocation, starlarkSemantics, env); if (bzlLoadValue == null) { return null; } @@ -152,12 +130,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (!(exported instanceof ModuleExtension.InStarlark)) { ImmutableSet exportedExtensions = bzlLoadValue.getModule().getGlobals().entrySet().stream() - .filter(e -> e.getValue() instanceof ModuleExtension.InStarlark) + .filter(e -> e.getValue() instanceof ModuleExtension) .map(Entry::getKey) .collect(toImmutableSet()); throw new SingleExtensionEvalFunctionException( ExternalDepsException.withMessage( - Code.BAD_MODULE, + ExternalDeps.Code.BAD_MODULE, "%s does not export a module extension called %s, yet its use is requested at %s%s", extensionId.getBzlFileLabel(), extensionId.getExtensionName(), @@ -166,13 +144,176 @@ public SkyValue compute(SkyKey skyKey, Environment env) Transience.PERSISTENT); } + // Check the lockfile first for that module extension + byte[] bzlTransitiveDigest = + BazelModuleContext.of(bzlLoadValue.getModule()).bzlTransitiveDigest(); + LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env); + if (!lockfileMode.equals(LockfileMode.OFF)) { + BazelLockFileValue lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); + if (lockfile == null) { + return null; + } + SingleExtensionEvalValue singleExtensionEvalValue = + tryGettingValueFromLockFile( + extensionId, usagesValue, bzlTransitiveDigest, lockfileMode, lockfile); + if (singleExtensionEvalValue != null) { + return singleExtensionEvalValue; + } + } + // Run that extension! ModuleExtension extension = ((ModuleExtension.InStarlark) exported).get(); + ImmutableMap generatedRepoSpecs = + runModuleExtension( + extensionId, extension, usagesValue, bzlLoadValue.getModule(), starlarkSemantics, env); + if (generatedRepoSpecs == null) { + return null; + } + // Check that all imported repos have been actually generated + validateAllImportsAreGenerated(generatedRepoSpecs, usagesValue, extensionId); + + if (lockfileMode.equals(LockfileMode.UPDATE)) { + env.getListener() + .post( + ModuleExtensionResolutionEvent.create( + extensionId, + LockFileModuleExtension.create(bzlTransitiveDigest, generatedRepoSpecs))); + } + return createSingleExtentionValue(generatedRepoSpecs, usagesValue); + } + + @Nullable + private SingleExtensionEvalValue tryGettingValueFromLockFile( + ModuleExtensionId extensionId, + SingleExtensionUsagesValue usagesValue, + byte[] bzlTransitiveDigest, + LockfileMode lockfileMode, + BazelLockFileValue lockfile) + throws SingleExtensionEvalFunctionException { + LockFileModuleExtension lockedExtension = lockfile.getModuleExtensions().get(extensionId); + ImmutableMap lockedExtensionUsages; + try { + // TODO(salmasamy) might be nicer to precompute this table when we construct + // BazelLockFileValue, without adding it to the json file + ImmutableTable extensionUsagesById = + BazelDepGraphFunction.getExtensionUsagesById(lockfile.getModuleDepGraph()); + lockedExtensionUsages = extensionUsagesById.row(extensionId); + } catch (ExternalDepsException e) { + throw new SingleExtensionEvalFunctionException(e, Transience.PERSISTENT); + } + + // If we have the extension, check if the implementation and usage haven't changed + if (lockedExtension != null + && Arrays.equals(bzlTransitiveDigest, lockedExtension.getBzlTransitiveDigest()) + && usagesValue.getExtensionUsages().equals(lockedExtensionUsages)) { + return createSingleExtentionValue(lockedExtension.getGeneratedRepoSpecs(), usagesValue); + } else if (lockfileMode.equals(LockfileMode.ERROR)) { + ImmutableList extDiff = + lockfile.getModuleExtensionDiff( + lockedExtension, + lockedExtensionUsages, + extensionId, + bzlTransitiveDigest, + usagesValue.getExtensionUsages()); + throw new SingleExtensionEvalFunctionException( + ExternalDepsException.withMessage( + Code.BAD_MODULE, + "Lock file is no longer up-to-date because: %s", + String.join(", ", extDiff)), + Transience.PERSISTENT); + } + return null; + } + + private SingleExtensionEvalValue createSingleExtentionValue( + ImmutableMap generatedRepoSpecs, SingleExtensionUsagesValue usagesValue) { + return SingleExtensionEvalValue.create( + generatedRepoSpecs, + generatedRepoSpecs.keySet().stream() + .collect( + toImmutableBiMap( + e -> + RepositoryName.createUnvalidated( + usagesValue.getExtensionUniqueName() + "~" + e), + Function.identity()))); + } + + private void validateAllImportsAreGenerated( + ImmutableMap generatedRepoSpecs, + SingleExtensionUsagesValue usagesValue, + ModuleExtensionId extensionId) + throws SingleExtensionEvalFunctionException { + for (ModuleExtensionUsage usage : usagesValue.getExtensionUsages().values()) { + for (Entry repoImport : usage.getImports().entrySet()) { + if (!generatedRepoSpecs.containsKey(repoImport.getValue())) { + throw new SingleExtensionEvalFunctionException( + ExternalDepsException.withMessage( + Code.BAD_MODULE, + "module extension \"%s\" from \"%s\" does not generate repository \"%s\", yet it" + + " is imported as \"%s\" in the usage at %s%s", + extensionId.getExtensionName(), + extensionId.getBzlFileLabel(), + repoImport.getValue(), + repoImport.getKey(), + usage.getLocation(), + SpellChecker.didYouMean(repoImport.getValue(), generatedRepoSpecs.keySet())), + Transience.PERSISTENT); + } + } + } + } + + private BzlLoadValue loadBzlFile( + Label bzlFileLabel, + Location sampleUsageLocation, + StarlarkSemantics starlarkSemantics, + Environment env) + throws SingleExtensionEvalFunctionException, InterruptedException { + // Check that the .bzl label isn't crazy. + try { + BzlLoadFunction.checkValidLoadLabel(bzlFileLabel, /*fromBuiltinsRepo=*/ false); + } catch (LabelSyntaxException e) { + throw new SingleExtensionEvalFunctionException( + ExternalDepsException.withCauseAndMessage( + Code.BAD_MODULE, e, "invalid module extension label"), + Transience.PERSISTENT); + } + + // Load the .bzl file pointed to by the label. + BzlLoadValue bzlLoadValue; + try { + bzlLoadValue = + (BzlLoadValue) + env.getValueOrThrow( + BzlLoadValue.keyForBzlmod(bzlFileLabel), BzlLoadFailedException.class); + } catch (BzlLoadFailedException e) { + throw new SingleExtensionEvalFunctionException( + ExternalDepsException.withCauseAndMessage( + Code.BAD_MODULE, + e, + "Error loading '%s' for module extensions, requested by %s: %s", + bzlFileLabel, + sampleUsageLocation, + e.getMessage()), + Transience.PERSISTENT); + } + return bzlLoadValue; + } + + @Nullable + private ImmutableMap runModuleExtension( + ModuleExtensionId extensionId, + ModuleExtension extension, + SingleExtensionUsagesValue usagesValue, + net.starlark.java.eval.Module module, + StarlarkSemantics starlarkSemantics, + Environment env) + throws SingleExtensionEvalFunctionException, InterruptedException { ModuleExtensionEvalStarlarkThreadContext threadContext = new ModuleExtensionEvalStarlarkThreadContext( usagesValue.getExtensionUniqueName() + "~", extensionId.getBzlFileLabel().getPackageIdentifier(), - BazelModuleContext.of(bzlLoadValue.getModule()).repoMapping(), + BazelModuleContext.of(module).repoMapping(), directories, env.getListener()); try (Mutability mu = @@ -189,7 +330,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (returnValue != Starlark.NONE && !(returnValue instanceof ModuleExtensionMetadata)) { throw new SingleExtensionEvalFunctionException( ExternalDepsException.withMessage( - Code.BAD_MODULE, + ExternalDeps.Code.BAD_MODULE, "expected module extension %s in %s to return None or extension_metadata, got %s", extensionId.getExtensionName(), extensionId.getBzlFileLabel(), @@ -210,51 +351,27 @@ public SkyValue compute(SkyKey skyKey, Environment env) moduleContext.getWorkingDirectory().deleteTree(); } } catch (IOException e1) { - throw new SingleExtensionEvalFunctionException(e1, Transience.TRANSIENT); + ExternalDepsException externalDepsException = + ExternalDepsException.withCauseAndMessage( + ExternalDeps.Code.UNRECOGNIZED, + e1, + "Failed to clean up module context directory"); + throw new SingleExtensionEvalFunctionException( + externalDepsException, Transience.TRANSIENT); } return null; } catch (EvalException e) { env.getListener().handle(Event.error(e.getMessageWithStack())); throw new SingleExtensionEvalFunctionException( ExternalDepsException.withMessage( - Code.BAD_MODULE, + ExternalDeps.Code.BAD_MODULE, "error evaluating module extension %s in %s", extensionId.getExtensionName(), extensionId.getBzlFileLabel()), Transience.TRANSIENT); } } - - // Check that all imported repos have been actually generated - for (ModuleExtensionUsage usage : usagesValue.getExtensionUsages().values()) { - for (Entry repoImport : usage.getImports().entrySet()) { - if (!threadContext.getGeneratedRepoSpecs().containsKey(repoImport.getValue())) { - throw new SingleExtensionEvalFunctionException( - ExternalDepsException.withMessage( - Code.BAD_MODULE, - "module extension \"%s\" from \"%s\" does not generate repository \"%s\", yet it" - + " is imported as \"%s\" in the usage at %s%s", - extensionId.getExtensionName(), - extensionId.getBzlFileLabel(), - repoImport.getValue(), - repoImport.getKey(), - usage.getLocation(), - SpellChecker.didYouMean( - repoImport.getValue(), threadContext.getGeneratedRepoSpecs().keySet())), - Transience.PERSISTENT); - } - } - } - - return SingleExtensionEvalValue.create( - threadContext.getGeneratedRepoSpecs(), - threadContext.getGeneratedRepoSpecs().keySet().stream() - .collect( - toImmutableBiMap( - e -> - RepositoryName.createUnvalidated( - usagesValue.getExtensionUniqueName() + "~" + e), - Function.identity()))); + return threadContext.getGeneratedRepoSpecs(); } private ModuleExtensionContext createContext( @@ -298,7 +415,7 @@ private ModuleExtensionContext createContext( static final class SingleExtensionEvalFunctionException extends SkyFunctionException { - SingleExtensionEvalFunctionException(Exception cause, Transience transience) { + SingleExtensionEvalFunctionException(ExternalDepsException cause, Transience transience) { super(cause, transience); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java index f2cbea78e44119..d11631b4556502 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java @@ -277,8 +277,8 @@ public class RepositoryOptions extends OptionsBase { @Option( name = "lockfile_mode", - defaultValue = "off", // TODO(salmasamy) later will be changed to 'update' converter = LockfileMode.Converter.class, + defaultValue = "update", documentationCategory = OptionDocumentationCategory.BZLMOD, effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS}, help = diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java index 33469c7367e421..8c79a470d6f76e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java @@ -83,7 +83,6 @@ public void afterCommand() { } try (Writer writer = Files.newWriter(new File(resolvedFile), StandardCharsets.UTF_8)) { writer.write(EXPORTED_NAME + " = " + new ValuePrinter().repr(resultBuilder.build())); - writer.close(); } catch (IOException e) { logger.atWarning().withCause(e).log("IO Error writing to file %s", resolvedFile); } diff --git a/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java b/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java index edb5955f579b64..aa5894aa028452 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java @@ -56,9 +56,8 @@ protected ImmutableList extraPrecomputedValues() throws Exception { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF), - PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of())); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE), + PrecomputedValue.injected(BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of())); } @Before diff --git a/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java b/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java index ea8cf9f5106196..4876868cb7520d 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java @@ -77,7 +77,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Override diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java index a511ce2d7d0c8e..047c2aa1741b70 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java @@ -143,7 +143,7 @@ public ImmutableMap getSkyFunctions(BlazeDirectori directories.getWorkspace(), getBuiltinModules(directories)), SkyFunctions.BAZEL_DEP_GRAPH, - new BazelDepGraphFunction(directories.getWorkspace()), + new BazelDepGraphFunction(), SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace()), SkyFunctions.BAZEL_MODULE_RESOLUTION, diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java index 373668e06a2c62..51b0eef293b7c2 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java @@ -245,7 +245,7 @@ protected void useRuleClassProvider(ConfiguredRuleClassProvider ruleClassProvide BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), PrecomputedValue.injected( - BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF))) + BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE))) .build(ruleClassProvider, fileSystem); useConfiguration(); skyframeExecutor = createSkyframeExecutor(pkgFactory); @@ -294,7 +294,7 @@ private void reinitializeSkyframeExecutor() { PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.WARNING), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF))); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE))); } /** Resets the SkyframeExecutor, as if a clean had been executed. */ diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 8c2c76bf5bc1d1..50de8c6f8cb7ef 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -30,6 +30,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info", "//src/main/java/com/google/devtools/build/lib/analysis:server_directories", "//src/main/java/com/google/devtools/build/lib/authandtls", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:bazel_lockfile_module", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:exception", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index 6894c77a50fe03..5dda67fd839ba2 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -127,7 +127,7 @@ public void setup() throws Exception { new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, resolutionFunctionMock) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, @@ -139,7 +139,6 @@ public void setup() throws Exception { PrecomputedValue.STARLARK_SEMANTICS.set( differencer, StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build()); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); @@ -147,7 +146,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.OFF); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } @@ -291,7 +290,7 @@ public void createValue_moduleExtensions() throws Exception { maven, "rules_jvm_external~1.0~maven", pip, "rules_python~2.0~pip", myext, "dep~2.0~myext", - myext2, "dep~2.0~myext~2"); + myext2, "dep~2.0~myext2"); assertThat(value.getFullRepoMapping(ModuleKey.ROOT)) .isEqualTo( @@ -322,7 +321,7 @@ public void createValue_moduleExtensions() throws Exception { "oneext", "dep~2.0~myext~myext", "twoext", - "dep~2.0~myext~2~myext")); + "dep~2.0~myext2~myext")); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index e10d52d15924fb..9461583a3d57f1 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -27,12 +27,14 @@ import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ServerDirectories; import com.google.devtools.build.lib.analysis.util.AnalysisMock; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction.BazelLockfileFunctionException; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule; import com.google.devtools.build.lib.clock.BlazeClock; +import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; import com.google.devtools.build.lib.rules.repository.LocalRepositoryFunction; @@ -55,6 +57,7 @@ import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; import com.google.devtools.build.lib.vfs.FileStateKey; import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; @@ -63,12 +66,14 @@ import com.google.devtools.build.skyframe.RecordingDifferencer; import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; import com.google.devtools.build.skyframe.SkyFunction; -import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; import javax.annotation.Nullable; import net.starlark.java.eval.StarlarkSemantics; import org.junit.Before; @@ -161,7 +166,7 @@ public void setup() throws Exception { @Nullable @Override public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { + throws BazelLockfileFunctionException, InterruptedException { UpdateLockFileKey key = (UpdateLockFileKey) skyKey; BzlmodFlagsAndEnvVars flags = BazelDepGraphFunction.getFlagsAndEnvVars(env); @@ -174,12 +179,18 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (localOverrideHashes == null) { return null; } - BazelLockFileFunction.updateLockedModule( - rootDirectory, - key.moduleHash(), - flags, - localOverrideHashes, - key.depGraph()); + RootedPath lockfilePath = + RootedPath.toRootedPath( + Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); + BazelLockFileModule.updateLockfile( + lockfilePath, + BazelLockFileValue.builder() + .setModuleFileHash(key.moduleHash()) + .setFlags(flags) + .setLocalOverrideHashes(localOverrideHashes) + .setModuleDepGraph(key.depGraph()) + .build()); + return new SkyValue() {}; } }) @@ -197,7 +208,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) differencer, BazelCompatibilityMode.ERROR); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.set(differencer, ImmutableMap.of()); RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING.set( differencer, RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY); @@ -266,16 +277,17 @@ public void moduleWithFlags() throws Exception { ImmutableList yankedVersions = ImmutableList.of("2.4", "2.3"); LocalPathOverride override = LocalPathOverride.create("override_path"); + ImmutableList registries = ImmutableList.of("registry1", "registry2"); + ImmutableMap moduleOverride = ImmutableMap.of("my_dep_1", override.getPath()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); + ModuleFileFunction.REGISTRIES.set(differencer, registries); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of("my_dep_1", override)); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, yankedVersions); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( - differencer, CheckDirectDepsMode.ERROR); + differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( - differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + differencer, BazelCompatibilityMode.WARNING); UpdateLockFileKey key = UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); @@ -293,11 +305,13 @@ public void moduleWithFlags() throws Exception { BazelLockFileValue value = result.get(BazelLockFileValue.KEY); assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); assertThat(value.getFlags().ignoreDevDependency()).isTrue(); + assertThat(value.getFlags().cmdRegistries()).isEqualTo(registries); + assertThat(value.getFlags().cmdModuleOverrides()).isEqualTo(moduleOverride); assertThat(value.getFlags().allowedYankedVersions()).isEqualTo(yankedVersions); assertThat(value.getFlags().directDependenciesMode()) - .isEqualTo(CheckDirectDepsMode.ERROR.toString()); + .isEqualTo(CheckDirectDepsMode.WARNING.toString()); assertThat(value.getFlags().compatibilityMode()) - .isEqualTo(BazelCompatibilityMode.ERROR.toString()); + .isEqualTo(BazelCompatibilityMode.WARNING.toString()); } @Test @@ -387,6 +401,141 @@ public void fullModule() throws Exception { assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); } + @Test + public void invalidLockfileEmptyFile() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.of( + ModuleKey.ROOT, + BazelModuleResolutionFunction.moduleFromInterimModule( + rootValue.getModule(), null, null)); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + scratch.overwriteFile(rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), "{}"); + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (!result.hasError()) { + fail("expected error about missing field in the lockfile, but succeeded"); + } + assertThat(result.getError().toString()) + .contains( + "Failed to read and parse the MODULE.bazel.lock file with error: " + + "java.lang.IllegalStateException: Missing required properties: moduleFileHash " + + "flags localOverrideHashes moduleDepGraph. Try deleting it and rerun the build."); + } + + @Test + public void invalidLockfileNullFlag() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.of( + ModuleKey.ROOT, + BazelModuleResolutionFunction.moduleFromInterimModule( + rootValue.getModule(), null, null)); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + JsonObject jsonObject = + (JsonObject) JsonParser.parseString(scratch.readFile("MODULE.bazel.lock")); + jsonObject.get("flags").getAsJsonObject().remove("directDependenciesMode"); + scratch.overwriteFile( + rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), jsonObject.toString()); + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (!result.hasError()) { + fail("expected error about missing field in the lockfile, but succeeded"); + } + assertThat(result.getError().toString()) + .contains( + "Failed to read and parse the MODULE.bazel.lock file with error: Null" + + " directDependenciesMode. Try deleting it and rerun the build."); + } + + @Test + public void invalidLockfileMalformed() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.of( + ModuleKey.ROOT, + BazelModuleResolutionFunction.moduleFromInterimModule( + rootValue.getModule(), null, null)); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + JsonObject jsonObject = + (JsonObject) JsonParser.parseString(scratch.readFile("MODULE.bazel.lock")); + jsonObject.get("flags").getAsJsonObject().addProperty("allowedYankedVersions", "string!"); + scratch.overwriteFile( + rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), jsonObject.toString()); + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (!result.hasError()) { + fail("expected error about invalid field value in the lockfile, but succeeded"); + } + Pattern expectedExceptionMessage = + Pattern.compile( + Pattern.quote( + "Failed to read and parse the MODULE.bazel.lock file with error:" + + " java.lang.IllegalStateException: Expected BEGIN_ARRAY but was STRING at" + + " line 1 column 129 path $.flags.allowedYankedVersions") + + ".*" + + Pattern.quote("Try deleting it and rerun the build."), + Pattern.DOTALL); + assertThat(result.getError().toString()).containsMatch(expectedExceptionMessage); + } + @AutoValue abstract static class UpdateLockFileKey implements SkyKey { diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 03fdbacf5e170f..1a05544e9d2fd0 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -112,7 +112,7 @@ public void setup() throws Exception { SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( @@ -130,7 +130,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.OFF); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 212bf5a2a2214b..92ff555778779d 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -124,7 +124,7 @@ public void setup() throws Exception { .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( @@ -145,7 +145,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); } @Test @@ -178,50 +178,6 @@ public void testRepoSpec_bazelModule() throws Exception { assertThat(repoRule.getAttr("path", Type.STRING)).isEqualTo("/usr/local/modules/ccc~2.0"); } - @Test - public void testRepoSpec_lockfile() throws Exception { - scratch.file( - workspaceRoot.getRelative("MODULE.bazel").getPathString(), - "bazel_dep(name='bbb',version='2.0')"); - - FakeRegistry registry = - registryFactory - .newFakeRegistry("/usr/local/modules") - .addModule(createModuleKey("bbb", "2.0"), "module(name='bbb', version='2.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); - - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); - - RepositoryName repo = RepositoryName.create("bbb~2.0"); - EvaluationResult result = - evaluator.evaluate(ImmutableList.of(BzlmodRepoRuleValue.key(repo)), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - BzlmodRepoRuleValue bzlmodRepoRuleValue = result.get(BzlmodRepoRuleValue.key(repo)); - Rule repoRule = bzlmodRepoRuleValue.getRule(); - assertThat(repoRule.getName()).isEqualTo("bbb~2.0"); - - /* Rerun the setup to: - 1.Reset evaluator to remove the sky values cache. - 2.Reset registry factory to be empty (now "bbb" doesn't exist at all) - without the lockfile "bbb" should not be found and this should fail - */ - setup(); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); - registry = registryFactory.newFakeRegistry("/usr/local/modules"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); - - // Calling again should read from lockfile and still find ccc - result = evaluator.evaluate(ImmutableList.of(BzlmodRepoRuleValue.key(repo)), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - bzlmodRepoRuleValue = result.get(BzlmodRepoRuleValue.key(repo)); - repoRule = bzlmodRepoRuleValue.getRule(); - assertThat(repoRule.getName()).isEqualTo("bbb~2.0"); - } - @Test public void testRepoSpec_nonRegistryOverride() throws Exception { scratch.file( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 31fce676ac7875..d73b2d479795ae 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -251,7 +251,7 @@ public void setup() throws Exception { BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction()) .put(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction) @@ -282,7 +282,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); // Set up a simple repo rule. registry.addModule( diff --git a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java index 6e705ff8b87833..3ce62e94f51e74 100644 --- a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java +++ b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java @@ -354,7 +354,7 @@ public void setSyscallCache(SyscallCache syscallCache) { protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ruleClassProvider) { PackageFactory pkgFactory = ((PackageFactoryBuilderWithSkyframeForTesting) - TestPackageFactoryBuilderFactory.getInstance().builder(directories)) + TestPackageFactoryBuilderFactory.getInstance().builder(directories)) .setExtraSkyFunctions(analysisMock.getSkyFunctions(directories)) .setExtraPrecomputeValues( ImmutableList.of( diff --git a/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java b/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java index 39e7aeafc023d7..7ce085431e5bb5 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java @@ -60,7 +60,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } private void writeRulesBzl(String type) throws Exception { diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java index b33273dc392f3e..6ca0e6fff59be6 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -238,7 +238,8 @@ public void setupDelegator() throws Exception { .put( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootPath, ImmutableMap.of())) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, @@ -270,7 +271,7 @@ public void setupDelegator() throws Exception { differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java index 61230364befbd8..2b98361118813b 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java @@ -113,7 +113,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Before diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java index 2d00ff7c3ed816..aaf6bd05afee16 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java @@ -247,7 +247,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } // Helpers: diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java index cda1434744fe9c..5097e5e24982c2 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java @@ -113,7 +113,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java index 2374e2c62126cb..b7b54d9e88bd5c 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java @@ -69,7 +69,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java index 06672b18ddc308..b7dcd438e337e1 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java @@ -88,7 +88,7 @@ protected ImmutableList extraPrecomputedValues() thro BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Override diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py index f9fe557bbd1851..03381c12e895e5 100644 --- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -14,6 +14,7 @@ # limitations under the License. # pylint: disable=g-long-ternary +import json import os import tempfile import unittest @@ -241,12 +242,7 @@ def testLocalOverrideWithErrorMode(self): ], ) exit_code, _, stderr = self.RunBazel( - [ - 'build', - '--nobuild', - '--lockfile_mode=error', - '//:all', - ], + ['build', '--nobuild', '--lockfile_mode=error', '//:all'], allow_failure=True, ) self.AssertExitCode(exit_code, 48, stderr) @@ -259,6 +255,256 @@ def testLocalOverrideWithErrorMode(self): stderr, ) + def testModuleExtension(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'lockfile_ext = use_extension("extension.bzl", "lockfile_ext")', + 'lockfile_ext.dep(name = "bmbm", versions = ["v1", "v2"])', + 'use_repo(lockfile_ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + '', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + '', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other side!")', + ' repo_rule(name="hello")', + ' for mod in ctx.modules:', + ' for dep in mod.tags.dep:', + ' print("Name:", dep.name, ", Versions:", dep.versions)', + '', + ( + '_dep = tag_class(attrs={"name": attr.string(), "versions":' + ' attr.string_list()})' + ), + 'lockfile_ext = module_extension(', + ' implementation=_module_ext_impl,', + ' tag_classes={"dep": _dep},', + ')', + ], + ) + + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertIn('Hello from the other side!', ''.join(stderr)) + self.assertIn('Name: bmbm , Versions: ["v1", "v2"]', ''.join(stderr)) + + self.RunBazel(['shutdown']) + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertNotIn('Hello from the other side!', ''.join(stderr)) + + def testModuleExtensionsInDifferentBuilds(self): + # Test that the module extension stays in the lockfile (as long as it's + # used in the module) even if it is not in the current build + self.ScratchFile( + 'MODULE.bazel', + [ + 'extA = use_extension("extension.bzl", "extA")', + 'extB = use_extension("extension.bzl", "extB")', + 'use_repo(extA, "hello_ext_A")', + 'use_repo(extB, "hello_ext_B")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + 'def _ext_a_impl(ctx):', + ' repo_rule(name="hello_ext_A")', + 'def _ext_b_impl(ctx):', + ' repo_rule(name="hello_ext_B")', + 'extA = module_extension(implementation=_ext_a_impl)', + 'extB = module_extension(implementation=_ext_b_impl)', + ], + ) + + self.RunBazel(['build', '@hello_ext_A//:all']) + self.RunBazel(['build', '@hello_ext_B//:all']) + + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + self.assertGreater(len(lockfile['moduleDepGraph']), 0) + self.assertEqual( + list(lockfile['moduleExtensions'].keys()), + ['//:extension.bzl%extA', '//:extension.bzl%extB'], + ) + + def testUpdateModuleExtension(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'lockfile_ext = use_extension("extension.bzl", "lockfile_ext")', + 'use_repo(lockfile_ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\\"lala\\")")', + 'repo_rule = repository_rule(implementation = _repo_rule_impl)', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other side!")', + ' repo_rule(name= "hello")', + ( + 'lockfile_ext = module_extension(implementation =' + ' _module_ext_impl)' + ), + ], + ) + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertIn('Hello from the other side!', ''.join(stderr)) + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + old_impl = lockfile['moduleExtensions']['//:extension.bzl%lockfile_ext'][ + 'bzlTransitiveDigest' + ] + + # Run again to make sure the resolution value is cached. So even if module + # resolution doesn't rerun (its event is null), the lockfile is still + # updated with the newest extension eval results + self.RunBazel(['build', '@hello//:all']) + + # Update extension. Make sure that it is executed and updated in the + # lockfile without errors (since it's already in the lockfile) + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\\"lala\\")")', + 'repo_rule = repository_rule(implementation = _repo_rule_impl)', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other town!")', + ' repo_rule(name= "hello")', + ( + 'lockfile_ext = module_extension(implementation =' + ' _module_ext_impl)' + ), + ], + ) + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertIn('Hello from the other town!', ''.join(stderr)) + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + new_impl = lockfile['moduleExtensions']['//:extension.bzl%lockfile_ext'][ + 'bzlTransitiveDigest' + ] + self.assertNotEqual(new_impl, old_impl) + + def testUpdateModuleExtensionErrorMode(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'lockfile_ext = use_extension("extension.bzl", "lockfile_ext")', + 'use_repo(lockfile_ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\\"lala\\")")', + 'repo_rule = repository_rule(implementation = _repo_rule_impl)', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other side!")', + ' repo_rule(name= "hello")', + ( + 'lockfile_ext = module_extension(implementation =' + ' _module_ext_impl)' + ), + ], + ) + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertIn('Hello from the other side!', ''.join(stderr)) + self.RunBazel(['shutdown']) + + # Update extension. + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\\"lalo\\")")', + 'repo_rule = repository_rule(implementation = _repo_rule_impl)', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other town!")', + ' repo_rule(name= "hello")', + ( + 'lockfile_ext = module_extension(implementation =' + ' _module_ext_impl)' + ), + ], + ) + + exit_code, _, stderr = self.RunBazel( + ['build', '--nobuild', '--lockfile_mode=error', '@hello//:all'], + allow_failure=True, + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + ( + 'ERROR: Lock file is no longer up-to-date because: The ' + 'implementation of the extension ' + "'ModuleExtensionId{bzlFileLabel=//:extension.bzl, " + "extensionName=lockfile_ext, isolationKey=Optional.empty}' or one " + 'of its transitive .bzl files has changed' + ), + stderr, + ) + + def testRemoveModuleExtensionsNotUsed(self): + # Test that the module extension is removed from the lockfile if it is not + # used anymore + self.ScratchFile( + 'MODULE.bazel', + [ + 'ext = use_extension("extension.bzl", "ext")', + 'use_repo(ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + 'def _ext_impl(ctx):', + ' repo_rule(name="hello")', + 'ext = module_extension(implementation=_ext_impl)', + ], + ) + + self.RunBazel(['build', '@hello//:all']) + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + self.assertEqual( + list(lockfile['moduleExtensions'].keys()), ['//:extension.bzl%ext'] + ) + + self.ScratchFile('MODULE.bazel', []) + self.RunBazel(['build', '//:all']) + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + self.assertEqual(len(lockfile['moduleExtensions']), 0) + if __name__ == '__main__': unittest.main()