From f88921f763ca0801122839719ff703715b41c195 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Tue, 6 Jun 2023 04:45:14 -0700 Subject: [PATCH 01/20] Update lockfile function exception Add new exception for this function to handle any syntax errors or missing data within the lockfile Related: https://github.com/bazelbuild/bazel/issues/18455 PiperOrigin-RevId: 538144339 Change-Id: I82160f3bff6598c26b3f99c824fe85ea86086c1f --- .../bazel/bzlmod/BazelLockFileFunction.java | 26 +++- .../bzlmod/BazelLockFileFunctionTest.java | 132 ++++++++++++++++++ 2 files changed, 151 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index cfeb31fa0f57bb..ba6c5b6c192cdf 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.FileValue; -import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction.BazelDepGraphFunctionException; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; @@ -35,7 +34,7 @@ import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; -import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; import java.io.FileNotFoundException; import java.io.IOException; import javax.annotation.Nullable; @@ -81,8 +80,14 @@ public SkyValue compute(SkyKey skyKey, Environment env) bazelLockFileValue = LOCKFILE_GSON.fromJson(json, BazelLockFileValue.class); } catch (FileNotFoundException e) { bazelLockFileValue = EMPTY_LOCKFILE; - } catch (IOException ex) { - throw new JsonIOException("Failed to read or parse module-lock file", ex); + } catch (IOException | JsonSyntaxException | NullPointerException e) { + throw new BazelLockfileFunctionException( + ExternalDepsException.withMessage( + Code.BAD_MODULE, + "Failed to read and parse the MODULE.bazel.lock file with error: %s." + + " Try deleting it and rerun the build.", + e.getMessage()), + Transience.PERSISTENT); } return bazelLockFileValue; } @@ -99,7 +104,7 @@ public static void updateLockedModule( BzlmodFlagsAndEnvVars flags, ImmutableMap localOverrideHashes, ImmutableMap resolvedDepGraph) - throws BazelDepGraphFunctionException { + throws BazelLockfileFunctionException { RootedPath lockfilePath = RootedPath.toRootedPath(Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); @@ -113,10 +118,17 @@ public static void updateLockedModule( try { FileSystemUtils.writeContent(lockfilePath.asPath(), UTF_8, LOCKFILE_GSON.toJson(value)); } catch (IOException e) { - throw new BazelDepGraphFunctionException( + throw new BazelLockfileFunctionException( ExternalDepsException.withCauseAndMessage( - Code.BAD_MODULE, e, "Unable to update module-lock file"), + Code.BAD_MODULE, e, "Unable to update the MODULE.bazel.lock file"), Transience.PERSISTENT); } } + + static final class BazelLockfileFunctionException extends SkyFunctionException { + + BazelLockfileFunctionException(Exception cause, Transience transience) { + super(cause, transience); + } + } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index e10d52d15924fb..f1c775b201fbd7 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -67,6 +67,8 @@ import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; @@ -387,6 +389,136 @@ public void fullModule() throws Exception { assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); } + @Test + public void invalidLockfileEmptyFile() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.of( + ModuleKey.ROOT, + BazelModuleResolutionFunction.moduleFromInterimModule( + rootValue.getModule(), null, null)); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + scratch.overwriteFile(rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), "{}"); + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (!result.hasError()) { + fail("expected error about missing field in the lockfile, but succeeded"); + } + assertThat(result.getError().toString()) + .contains( + "Failed to read and parse the MODULE.bazel.lock file with error: Null moduleFileHash." + + " Try deleting it and rerun the build."); + } + + @Test + public void invalidLockfileNullFlag() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.of( + ModuleKey.ROOT, + BazelModuleResolutionFunction.moduleFromInterimModule( + rootValue.getModule(), null, null)); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + JsonObject jsonObject = + (JsonObject) JsonParser.parseString(scratch.readFile("MODULE.bazel.lock")); + jsonObject.get("flags").getAsJsonObject().remove("directDependenciesMode"); + scratch.overwriteFile( + rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), jsonObject.toString()); + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (!result.hasError()) { + fail("expected error about missing field in the lockfile, but succeeded"); + } + assertThat(result.getError().toString()) + .contains( + "Failed to read and parse the MODULE.bazel.lock file with error: Null" + + " directDependenciesMode. Try deleting it and rerun the build."); + } + + @Test + public void invalidLockfileMalformed() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.of( + ModuleKey.ROOT, + BazelModuleResolutionFunction.moduleFromInterimModule( + rootValue.getModule(), null, null)); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + JsonObject jsonObject = + (JsonObject) JsonParser.parseString(scratch.readFile("MODULE.bazel.lock")); + jsonObject.get("flags").getAsJsonObject().addProperty("allowedYankedVersions", "string!"); + scratch.overwriteFile( + rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), jsonObject.toString()); + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (!result.hasError()) { + fail("expected error about invalid field value in the lockfile, but succeeded"); + } + assertThat(result.getError().toString()) + .contains( + "Failed to read and parse the MODULE.bazel.lock file with error:" + + " java.lang.IllegalStateException: Expected BEGIN_ARRAY but was STRING at line 1" + + " column 129 path $.flags.allowedYankedVersions. Try deleting it and rerun the" + + " build."); + } + @AutoValue abstract static class UpdateLockFileKey implements SkyKey { From a34649c6219697912b17b3d8d56cdd3d1f761c92 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Thu, 15 Jun 2023 05:30:52 -0700 Subject: [PATCH 02/20] Update lockfile writing logic to be event triggered. The writing will only happen at the end of resolution in the after command function (at this point, the module and all needed module extensions are resolved). Which solves the problem of reading and writing into the lockfile multiple times in one invocation. PiperOrigin-RevId: 540552139 Change-Id: I4a78412a388bde2ff7949d119831318c40d49047 # Conflicts: # src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java # src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java # src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java --- .../com/google/devtools/build/lib/bazel/BUILD | 1 + .../devtools/build/lib/bazel/Bazel.java | 1 + .../lib/bazel/BazelRepositoryModule.java | 3 +- .../devtools/build/lib/bazel/bzlmod/BUILD | 16 ++++ .../bazel/bzlmod/BazelDepGraphFunction.java | 49 ++++------- .../bazel/bzlmod/BazelLockFileFunction.java | 50 ++--------- .../lib/bazel/bzlmod/BazelLockFileModule.java | 86 +++++++++++++++++++ .../lib/bazel/bzlmod/BazelLockFileValue.java | 31 +++++-- .../repository/RepositoryResolvedModule.java | 1 - .../build/lib/analysis/util/AnalysisMock.java | 2 +- .../devtools/build/lib/bazel/bzlmod/BUILD | 1 + .../bzlmod/BazelDepGraphFunctionTest.java | 2 +- .../bzlmod/BazelLockFileFunctionTest.java | 46 ++++++---- .../BazelModuleResolutionFunctionTest.java | 2 +- .../bzlmod/BzlmodRepoRuleFunctionTest.java | 46 +--------- .../bzlmod/ModuleExtensionResolutionTest.java | 2 +- .../repository/RepositoryDelegatorTest.java | 2 +- 17 files changed, 187 insertions(+), 154 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD index 3c29ea712d2909..bd7402b7f9d77a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD @@ -142,6 +142,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info", "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:bazel_lockfile_module", "//src/main/java/com/google/devtools/build/lib/bazel/coverage", "//src/main/java/com/google/devtools/build/lib/bazel/debug:workspace-rule-module", "//src/main/java/com/google/devtools/build/lib/bazel/repository", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java index e35b11a8386f5d..7fb055d87e2c30 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java @@ -63,6 +63,7 @@ public final class Bazel { com.google.devtools.build.lib.bazel.repository.RepositoryResolvedModule.class, com.google.devtools.build.lib.bazel.repository.CacheHitReportingModule.class, com.google.devtools.build.lib.bazel.SpawnLogModule.class, + com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileModule.class, com.google.devtools.build.lib.outputfilter.OutputFilteringModule.class, com.google.devtools.build.lib.worker.WorkerModule.class, com.google.devtools.build.lib.runtime.CacheFileDigestsModule.class, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 5e87645a450f50..1e19a99de35e2d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -265,8 +265,7 @@ public ResolutionReason getResolutionReason() { .addSkyFunction( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, directories.getWorkspace(), builtinModules)) - .addSkyFunction( - SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(directories.getWorkspace())) + .addSkyFunction(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .addSkyFunction( SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace())) .addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction()) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 4b45705309b979..02967174997589 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -85,6 +85,22 @@ java_library( ], ) +java_library( + name = "bazel_lockfile_module", + srcs = ["BazelLockFileModule.java"], + deps = [ + ":resolution", + ":resolution_impl", + "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//third_party:flogger", + "//third_party:guava", + "//third_party:jsr305", + ], +) + java_library( name = "resolution", srcs = [ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index f99b33b627a371..f55e3fbc8b6b13 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -35,7 +35,6 @@ import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue; -import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; @@ -54,16 +53,12 @@ */ public class BazelDepGraphFunction implements SkyFunction { - private final Path rootDirectory; - - public BazelDepGraphFunction(Path rootDirectory) { - this.rootDirectory = rootDirectory; - } + public BazelDepGraphFunction() {} @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { + throws BazelDepGraphFunctionException, InterruptedException { RootModuleFileValue root = (RootModuleFileValue) env.getValue(ModuleFileValue.KEY_FOR_ROOT_MODULE); if (root == null) { @@ -74,11 +69,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) ImmutableMap localOverrideHashes = null; ImmutableMap depGraph = null; BzlmodFlagsAndEnvVars flags = null; + BazelLockFileValue lockFile = null; // If the module has not changed (has the same contents and flags as the lockfile), // read the dependency graph from the lock file, else run resolution and update lockfile if (!lockfileMode.equals(LockfileMode.OFF)) { - BazelLockFileValue lockFile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); + lockFile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); if (lockFile == null) { return null; } @@ -115,8 +111,14 @@ public SkyValue compute(SkyKey skyKey, Environment env) } depGraph = selectionResult.getResolvedDepGraph(); if (lockfileMode.equals(LockfileMode.UPDATE)) { - BazelLockFileFunction.updateLockedModule( - rootDirectory, root.getModuleFileHash(), flags, localOverrideHashes, depGraph); + BazelLockFileValue updatedLockFile = + lockFile.toBuilder() + .setModuleFileHash(root.getModuleFileHash()) + .setFlags(flags) + .setLocalOverrideHashes(localOverrideHashes) + .setModuleDepGraph(depGraph) + .build(); + env.getListener().post(updatedLockFile); } } @@ -214,9 +216,7 @@ private ImmutableTable getEx try { moduleExtensionId = ModuleExtensionId.create( - labelConverter.convert(usage.getExtensionBzlFile()), - usage.getExtensionName(), - usage.getIsolationKey()); + labelConverter.convert(usage.getExtensionBzlFile()), usage.getExtensionName()); } catch (LabelSyntaxException e) { throw new BazelDepGraphFunctionException( ExternalDepsException.withCauseAndMessage( @@ -250,31 +250,12 @@ private ImmutableBiMap calculateUniqueNameForUsedExte // not start with a tilde. RepositoryName repository = id.getBzlFileLabel().getRepository(); String nonEmptyRepoPart = repository.isMain() ? "_main" : repository.getName(); - // When using a namespace, prefix the extension name with "_" to distinguish the prefix from - // those generated by non-namespaced extension usages. Extension names are identified by their - // Starlark identifier, which in the case of an exported symbol cannot start with "_". - // We also include whether the isolated usage is a dev usage as well as its index in the - // MODULE.bazel file to ensure that canonical repository names don't change depending on - // whether dev dependencies are ignored. This removes potential for confusion and also - // prevents unnecessary refetches when --ignore_dev_dependency is toggled. - String bestName = - id.getIsolationKey() - .map( - namespace -> - String.format( - "%s~_%s~%s~%s~%s%d", - nonEmptyRepoPart, - id.getExtensionName(), - namespace.getModule().getName(), - namespace.getModule().getVersion(), - namespace.isDevUsage() ? "dev" : "", - namespace.getIsolatedUsageIndex())) - .orElse(nonEmptyRepoPart + "~" + id.getExtensionName()); + String bestName = nonEmptyRepoPart + "~" + id.getExtensionName(); if (extensionUniqueNames.putIfAbsent(bestName, id) == null) { continue; } int suffix = 2; - while (extensionUniqueNames.putIfAbsent(bestName + "~" + suffix, id) != null) { + while (extensionUniqueNames.putIfAbsent(bestName + suffix, id) != null) { suffix++; } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index ba6c5b6c192cdf..5200381621fc7e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -51,12 +51,13 @@ public class BazelLockFileFunction implements SkyFunction { ImmutableList.of(), ImmutableMap.of(), ImmutableList.of(), "", false, "", ""); private static final BazelLockFileValue EMPTY_LOCKFILE = - BazelLockFileValue.create( - BazelLockFileValue.LOCK_FILE_VERSION, - "", - EMPTY_FLAGS, - ImmutableMap.of(), - ImmutableMap.of()); + BazelLockFileValue.builder() + .setLockFileVersion(BazelLockFileValue.LOCK_FILE_VERSION) + .setModuleFileHash("") + .setFlags(EMPTY_FLAGS) + .setLocalOverrideHashes(ImmutableMap.of()) + .setModuleDepGraph(ImmutableMap.of()) + .build(); public BazelLockFileFunction(Path rootDirectory) { this.rootDirectory = rootDirectory; @@ -65,7 +66,7 @@ public BazelLockFileFunction(Path rootDirectory) { @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { + throws BazelLockfileFunctionException, InterruptedException { RootedPath lockfilePath = RootedPath.toRootedPath(Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); @@ -92,42 +93,9 @@ public SkyValue compute(SkyKey skyKey, Environment env) return bazelLockFileValue; } - /** - * Updates the stored module in the lock file (ModuleHash, Flags & Dependency graph) - * - * @param moduleFileHash The hash of the current module file - * @param resolvedDepGraph The resolved dependency graph from the module file - */ - public static void updateLockedModule( - Path rootDirectory, - String moduleFileHash, - BzlmodFlagsAndEnvVars flags, - ImmutableMap localOverrideHashes, - ImmutableMap resolvedDepGraph) - throws BazelLockfileFunctionException { - RootedPath lockfilePath = - RootedPath.toRootedPath(Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); - - BazelLockFileValue value = - BazelLockFileValue.create( - BazelLockFileValue.LOCK_FILE_VERSION, - moduleFileHash, - flags, - localOverrideHashes, - resolvedDepGraph); - try { - FileSystemUtils.writeContent(lockfilePath.asPath(), UTF_8, LOCKFILE_GSON.toJson(value)); - } catch (IOException e) { - throw new BazelLockfileFunctionException( - ExternalDepsException.withCauseAndMessage( - Code.BAD_MODULE, e, "Unable to update the MODULE.bazel.lock file"), - Transience.PERSISTENT); - } - } - static final class BazelLockfileFunctionException extends SkyFunctionException { - BazelLockfileFunctionException(Exception cause, Transience transience) { + BazelLockfileFunctionException(ExternalDepsException cause, Transience transience) { super(cause, transience); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java new file mode 100644 index 00000000000000..962c70a8f98d9c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java @@ -0,0 +1,86 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static com.google.devtools.build.lib.bazel.bzlmod.GsonTypeAdapterUtil.LOCKFILE_GSON; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.eventbus.Subscribe; +import com.google.common.flogger.GoogleLogger; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; +import com.google.devtools.build.lib.cmdline.LabelConstants; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; +import java.io.IOException; +import javax.annotation.Nullable; + +/** + * Module collecting Bazel module and module extensions resolution results and updating the + * lockfile. + */ +public class BazelLockFileModule extends BlazeModule { + + private Path workspaceRoot; + + @Nullable private BazelLockFileValue lockfileValue; + + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + @Override + public void beforeCommand(CommandEnvironment env) { + workspaceRoot = env.getWorkspace(); + RepositoryOptions options = env.getOptions().getOptions(RepositoryOptions.class); + if (options.lockfileMode.equals(LockfileMode.UPDATE)) { + env.getEventBus().register(this); + } + } + + @Override + public void afterCommand() { + if (lockfileValue == null) { // module didn't change --> no event was posted + return; + } + updateLockfile(workspaceRoot, lockfileValue); + this.lockfileValue = null; + } + + /** + * Updates the data stored in the lockfile (MODULE.bazel.lock) + * + * @param workspaceRoot Where to update the lockfile + * @param updatedLockfile The updated lockfile data to save + */ + public static void updateLockfile(Path workspaceRoot, BazelLockFileValue updatedLockfile) { + RootedPath lockfilePath = + RootedPath.toRootedPath(Root.fromPath(workspaceRoot), LabelConstants.MODULE_LOCKFILE_NAME); + try { + FileSystemUtils.writeContent( + lockfilePath.asPath(), UTF_8, LOCKFILE_GSON.toJson(updatedLockfile)); + } catch (IOException e) { + logger.atSevere().withCause(e).log( + "Error while updating MODULE.bazel.lock file: %s", e.getMessage()); + } + } + + @Subscribe + public void bazelModuleResolved(BazelLockFileValue lockfileValue) { + this.lockfileValue = lockfileValue; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java index 6f32405f61dd2c..fa93c4f39874e4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java @@ -18,6 +18,7 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.skyframe.SkyKey; @@ -33,20 +34,14 @@ */ @AutoValue @GenerateTypeAdapter -public abstract class BazelLockFileValue implements SkyValue { +public abstract class BazelLockFileValue implements SkyValue, Postable { public static final int LOCK_FILE_VERSION = 1; @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_LOCK_FILE; - public static BazelLockFileValue create( - int lockFileVersion, - String moduleFileHash, - BzlmodFlagsAndEnvVars flags, - ImmutableMap localOverrideHashes, - ImmutableMap moduleDepGraph) { - return new AutoValue_BazelLockFileValue( - lockFileVersion, moduleFileHash, flags, localOverrideHashes, moduleDepGraph); + static Builder builder() { + return new AutoValue_BazelLockFileValue.Builder().setLockFileVersion(LOCK_FILE_VERSION); } /** Current version of the lock file */ @@ -64,6 +59,24 @@ public static BazelLockFileValue create( /** The post-selection dep graph retrieved from the lock file. */ public abstract ImmutableMap getModuleDepGraph(); + public abstract Builder toBuilder(); + + /** Builder type for {@link BazelLockFileValue}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setLockFileVersion(int value); + + public abstract Builder setModuleFileHash(String value); + + public abstract Builder setFlags(BzlmodFlagsAndEnvVars value); + + public abstract Builder setLocalOverrideHashes(ImmutableMap value); + + public abstract Builder setModuleDepGraph(ImmutableMap value); + + public abstract BazelLockFileValue build(); + } + /** Returns the difference between the lockfile and the current module & flags */ public ArrayList getDiffLockfile( String moduleFileHash, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java index 33469c7367e421..8c79a470d6f76e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java @@ -83,7 +83,6 @@ public void afterCommand() { } try (Writer writer = Files.newWriter(new File(resolvedFile), StandardCharsets.UTF_8)) { writer.write(EXPORTED_NAME + " = " + new ValuePrinter().repr(resultBuilder.build())); - writer.close(); } catch (IOException e) { logger.atWarning().withCause(e).log("IO Error writing to file %s", resolvedFile); } diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java index a511ce2d7d0c8e..047c2aa1741b70 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java @@ -143,7 +143,7 @@ public ImmutableMap getSkyFunctions(BlazeDirectori directories.getWorkspace(), getBuiltinModules(directories)), SkyFunctions.BAZEL_DEP_GRAPH, - new BazelDepGraphFunction(directories.getWorkspace()), + new BazelDepGraphFunction(), SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace()), SkyFunctions.BAZEL_MODULE_RESOLUTION, diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 8c2c76bf5bc1d1..50de8c6f8cb7ef 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -30,6 +30,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info", "//src/main/java/com/google/devtools/build/lib/analysis:server_directories", "//src/main/java/com/google/devtools/build/lib/authandtls", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:bazel_lockfile_module", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:exception", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index 6894c77a50fe03..dfdab83e416e2a 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -127,7 +127,7 @@ public void setup() throws Exception { new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, resolutionFunctionMock) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index f1c775b201fbd7..269869f3866d92 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -27,6 +27,7 @@ import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ServerDirectories; import com.google.devtools.build.lib.analysis.util.AnalysisMock; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction.BazelLockfileFunctionException; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; @@ -63,7 +64,6 @@ import com.google.devtools.build.skyframe.RecordingDifferencer; import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; import com.google.devtools.build.skyframe.SkyFunction; -import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; @@ -71,6 +71,7 @@ import com.google.gson.JsonParser; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; import javax.annotation.Nullable; import net.starlark.java.eval.StarlarkSemantics; import org.junit.Before; @@ -93,7 +94,7 @@ public void setup() throws Exception { differencer = new SequencedRecordingDifferencer(); registryFactory = new FakeRegistry.Factory(); evaluationContext = - EvaluationContext.newBuilder().setNumThreads(8).setEventHandler(reporter).build(); + EvaluationContext.newBuilder().setParallelism(8).setEventHandler(reporter).build(); AtomicReference packageLocator = new AtomicReference<>( @@ -163,7 +164,7 @@ public void setup() throws Exception { @Nullable @Override public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { + throws BazelLockfileFunctionException, InterruptedException { UpdateLockFileKey key = (UpdateLockFileKey) skyKey; BzlmodFlagsAndEnvVars flags = BazelDepGraphFunction.getFlagsAndEnvVars(env); @@ -176,12 +177,14 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (localOverrideHashes == null) { return null; } - BazelLockFileFunction.updateLockedModule( + BazelLockFileModule.updateLockfile( rootDirectory, - key.moduleHash(), - flags, - localOverrideHashes, - key.depGraph()); + BazelLockFileValue.builder() + .setModuleFileHash(key.moduleHash()) + .setFlags(flags) + .setLocalOverrideHashes(localOverrideHashes) + .setModuleDepGraph(key.depGraph()) + .build()); return new SkyValue() {}; } }) @@ -268,10 +271,12 @@ public void moduleWithFlags() throws Exception { ImmutableList yankedVersions = ImmutableList.of("2.4", "2.3"); LocalPathOverride override = LocalPathOverride.create("override_path"); + ImmutableList registries = ImmutableList.of("registry1", "registry2"); + ImmutableMap moduleOverride = ImmutableMap.of("my_dep_1", override.getPath()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); + ModuleFileFunction.REGISTRIES.set(differencer, registries); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of("my_dep_1", override)); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, yankedVersions); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.ERROR); @@ -295,6 +300,8 @@ public void moduleWithFlags() throws Exception { BazelLockFileValue value = result.get(BazelLockFileValue.KEY); assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); assertThat(value.getFlags().ignoreDevDependency()).isTrue(); + assertThat(value.getFlags().cmdRegistries()).isEqualTo(registries); + assertThat(value.getFlags().cmdModuleOverrides()).isEqualTo(moduleOverride); assertThat(value.getFlags().allowedYankedVersions()).isEqualTo(yankedVersions); assertThat(value.getFlags().directDependenciesMode()) .isEqualTo(CheckDirectDepsMode.ERROR.toString()); @@ -425,8 +432,9 @@ public void invalidLockfileEmptyFile() throws Exception { } assertThat(result.getError().toString()) .contains( - "Failed to read and parse the MODULE.bazel.lock file with error: Null moduleFileHash." - + " Try deleting it and rerun the build."); + "Failed to read and parse the MODULE.bazel.lock file with error: " + + "java.lang.IllegalStateException: Missing required properties: moduleFileHash " + + "flags localOverrideHashes moduleDepGraph. Try deleting it and rerun the build."); } @Test @@ -511,12 +519,16 @@ public void invalidLockfileMalformed() throws Exception { if (!result.hasError()) { fail("expected error about invalid field value in the lockfile, but succeeded"); } - assertThat(result.getError().toString()) - .contains( - "Failed to read and parse the MODULE.bazel.lock file with error:" - + " java.lang.IllegalStateException: Expected BEGIN_ARRAY but was STRING at line 1" - + " column 129 path $.flags.allowedYankedVersions. Try deleting it and rerun the" - + " build."); + Pattern expectedExceptionMessage = + Pattern.compile( + Pattern.quote( + "Failed to read and parse the MODULE.bazel.lock file with error:" + + " java.lang.IllegalStateException: Expected BEGIN_ARRAY but was STRING at" + + " line 1 column 129 path $.flags.allowedYankedVersions") + + ".*" + + Pattern.quote("Try deleting it and rerun the build."), + Pattern.DOTALL); + assertThat(result.getError().toString()).containsMatch(expectedExceptionMessage); } @AutoValue diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 03fdbacf5e170f..427450dd5cd237 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -112,7 +112,7 @@ public void setup() throws Exception { SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 212bf5a2a2214b..d1a4ef5b5037c2 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -124,7 +124,7 @@ public void setup() throws Exception { .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( @@ -178,50 +178,6 @@ public void testRepoSpec_bazelModule() throws Exception { assertThat(repoRule.getAttr("path", Type.STRING)).isEqualTo("/usr/local/modules/ccc~2.0"); } - @Test - public void testRepoSpec_lockfile() throws Exception { - scratch.file( - workspaceRoot.getRelative("MODULE.bazel").getPathString(), - "bazel_dep(name='bbb',version='2.0')"); - - FakeRegistry registry = - registryFactory - .newFakeRegistry("/usr/local/modules") - .addModule(createModuleKey("bbb", "2.0"), "module(name='bbb', version='2.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); - - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); - - RepositoryName repo = RepositoryName.create("bbb~2.0"); - EvaluationResult result = - evaluator.evaluate(ImmutableList.of(BzlmodRepoRuleValue.key(repo)), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - BzlmodRepoRuleValue bzlmodRepoRuleValue = result.get(BzlmodRepoRuleValue.key(repo)); - Rule repoRule = bzlmodRepoRuleValue.getRule(); - assertThat(repoRule.getName()).isEqualTo("bbb~2.0"); - - /* Rerun the setup to: - 1.Reset evaluator to remove the sky values cache. - 2.Reset registry factory to be empty (now "bbb" doesn't exist at all) - without the lockfile "bbb" should not be found and this should fail - */ - setup(); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); - registry = registryFactory.newFakeRegistry("/usr/local/modules"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); - - // Calling again should read from lockfile and still find ccc - result = evaluator.evaluate(ImmutableList.of(BzlmodRepoRuleValue.key(repo)), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - bzlmodRepoRuleValue = result.get(BzlmodRepoRuleValue.key(repo)); - repoRule = bzlmodRepoRuleValue.getRule(); - assertThat(repoRule.getName()).isEqualTo("bbb~2.0"); - } - @Test public void testRepoSpec_nonRegistryOverride() throws Exception { scratch.file( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 31fce676ac7875..84e6defea0e153 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -251,7 +251,7 @@ public void setup() throws Exception { BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction()) .put(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction) diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java index b33273dc392f3e..94615cb8fd98a4 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -238,7 +238,7 @@ public void setupDelegator() throws Exception { .put( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootPath, ImmutableMap.of())) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, From fd050aab4915d09273aeae23c68ec7cbe81575fd Mon Sep 17 00:00:00 2001 From: salma-samy Date: Mon, 26 Jun 2023 08:24:33 -0700 Subject: [PATCH 03/20] Add module extension to lockfile PiperOrigin-RevId: 543445157 Change-Id: Ib32a2c9fdc22c5b228d78141fc9648b3ef5edf7d # Conflicts: # src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java # src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java --- .../devtools/build/lib/bazel/bzlmod/BUILD | 6 + .../bazel/bzlmod/BazelDepGraphFunction.java | 80 +++--- .../bazel/bzlmod/BazelLockFileFunction.java | 18 +- .../lib/bazel/bzlmod/BazelLockFileModule.java | 88 +++++- .../lib/bazel/bzlmod/BazelLockFileValue.java | 47 +++- .../bzlmod/BazelModuleResolutionEvent.java | 39 +++ .../bazel/bzlmod/BzlmodFlagsAndEnvVars.java | 8 +- .../lib/bazel/bzlmod/GsonTypeAdapterUtil.java | 46 ++++ .../bazel/bzlmod/LockFileModuleExtension.java | 40 +++ .../ModuleExtensionResolutionEvent.java | 36 +++ .../bzlmod/SingleExtensionEvalFunction.java | 257 +++++++++++++----- .../bzlmod/BazelLockFileFunctionTest.java | 8 +- .../bzlmod/ModuleExtensionResolutionTest.java | 2 +- .../py/bazel/bzlmod/bazel_lockfile_test.py | 243 ++++++++++++++++- 14 files changed, 770 insertions(+), 148 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 02967174997589..61015909ecdbb1 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -92,10 +92,13 @@ java_library( ":resolution", ":resolution_impl", "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception", "//src/main/java/com/google/devtools/build/lib/vfs", "//third_party:flogger", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", ], @@ -108,14 +111,17 @@ java_library( "ArchiveOverride.java", "BazelDepGraphValue.java", "BazelLockFileValue.java", + "BazelModuleResolutionEvent.java", "BazelModuleResolutionValue.java", "BzlmodFlagsAndEnvVars.java", "GitOverride.java", "InterimModule.java", "LocalPathOverride.java", + "LockFileModuleExtension.java", "Module.java", "ModuleBase.java", "ModuleExtensionEvalStarlarkThreadContext.java", + "ModuleExtensionResolutionEvent.java", "ModuleFileValue.java", "ModuleOverride.java", "MultipleVersionOverride.java", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index f55e3fbc8b6b13..2852c88116e9a3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -41,7 +41,6 @@ import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; -import java.util.List; import java.util.Map; import java.util.Map.Entry; import javax.annotation.Nullable; @@ -69,13 +68,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) ImmutableMap localOverrideHashes = null; ImmutableMap depGraph = null; BzlmodFlagsAndEnvVars flags = null; - BazelLockFileValue lockFile = null; + BazelLockFileValue lockfile = null; // If the module has not changed (has the same contents and flags as the lockfile), // read the dependency graph from the lock file, else run resolution and update lockfile if (!lockfileMode.equals(LockfileMode.OFF)) { - lockFile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); - if (lockFile == null) { + lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); + if (lockfile == null) { return null; } flags = getFlagsAndEnvVars(env); @@ -83,17 +82,17 @@ public SkyValue compute(SkyKey skyKey, Environment env) return null; } localOverrideHashes = getLocalOverridesHashes(root.getOverrides(), env); - if (localOverrideHashes == null) { // trying to read override module + if (localOverrideHashes == null) { // still reading an override "module" return null; } - if (root.getModuleFileHash().equals(lockFile.getModuleFileHash()) - && flags.equals(lockFile.getFlags()) - && localOverrideHashes.equals(lockFile.getLocalOverrideHashes())) { - depGraph = lockFile.getModuleDepGraph(); + if (root.getModuleFileHash().equals(lockfile.getModuleFileHash()) + && flags.equals(lockfile.getFlags()) + && localOverrideHashes.equals(lockfile.getLocalOverrideHashes())) { + depGraph = lockfile.getModuleDepGraph(); } else if (lockfileMode.equals(LockfileMode.ERROR)) { - List diffLockfile = - lockFile.getDiffLockfile(root.getModuleFileHash(), localOverrideHashes, flags); + ImmutableList diffLockfile = + lockfile.getModuleAndFlagsDiff(root.getModuleFileHash(), localOverrideHashes, flags); throw new BazelDepGraphFunctionException( ExternalDepsException.withMessage( Code.BAD_MODULE, @@ -110,28 +109,34 @@ public SkyValue compute(SkyKey skyKey, Environment env) return null; } depGraph = selectionResult.getResolvedDepGraph(); - if (lockfileMode.equals(LockfileMode.UPDATE)) { - BazelLockFileValue updatedLockFile = - lockFile.toBuilder() - .setModuleFileHash(root.getModuleFileHash()) - .setFlags(flags) - .setLocalOverrideHashes(localOverrideHashes) - .setModuleDepGraph(depGraph) - .build(); - env.getListener().post(updatedLockFile); - } } ImmutableMap canonicalRepoNameLookup = depGraph.keySet().stream() .collect(toImmutableMap(ModuleKey::getCanonicalRepoName, key -> key)); - ImmutableTable extensionUsagesById = - getExtensionUsagesById(depGraph); + ImmutableTable extensionUsagesById; + try { + extensionUsagesById = getExtensionUsagesById(depGraph); + } catch (ExternalDepsException e) { + throw new BazelDepGraphFunctionException(e, Transience.PERSISTENT); + } ImmutableBiMap extensionUniqueNames = calculateUniqueNameForUsedExtensionId(extensionUsagesById); + if (!lockfileMode.equals(LockfileMode.OFF)) { + BazelLockFileValue updateLockfile = + lockfile.toBuilder() + .setModuleFileHash(root.getModuleFileHash()) + .setFlags(flags) + .setLocalOverrideHashes(localOverrideHashes) + .setModuleDepGraph(depGraph) + .build(); + env.getListener() + .post(BazelModuleResolutionEvent.create(updateLockfile, extensionUsagesById)); + } + return BazelDepGraphValue.create( depGraph, canonicalRepoNameLookup, @@ -202,8 +207,9 @@ static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws Interrup * For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can group * all usages by the label + name (the ModuleExtensionId). */ - private ImmutableTable getExtensionUsagesById( - ImmutableMap depGraph) throws BazelDepGraphFunctionException { + public static ImmutableTable + getExtensionUsagesById(ImmutableMap depGraph) + throws ExternalDepsException { ImmutableTable.Builder extensionUsagesTableBuilder = ImmutableTable.builder(); for (Module module : depGraph.values()) { @@ -218,22 +224,18 @@ private ImmutableTable getEx ModuleExtensionId.create( labelConverter.convert(usage.getExtensionBzlFile()), usage.getExtensionName()); } catch (LabelSyntaxException e) { - throw new BazelDepGraphFunctionException( - ExternalDepsException.withCauseAndMessage( - Code.BAD_MODULE, - e, - "invalid label for module extension found at %s", - usage.getLocation()), - Transience.PERSISTENT); + throw ExternalDepsException.withCauseAndMessage( + Code.BAD_MODULE, + e, + "invalid label for module extension found at %s", + usage.getLocation()); } if (!moduleExtensionId.getBzlFileLabel().getRepository().isVisible()) { - throw new BazelDepGraphFunctionException( - ExternalDepsException.withMessage( - Code.BAD_MODULE, - "invalid label for module extension found at %s: no repo visible as '@%s' here", - usage.getLocation(), - moduleExtensionId.getBzlFileLabel().getRepository().getName()), - Transience.PERSISTENT); + throw ExternalDepsException.withMessage( + Code.BAD_MODULE, + "invalid label for module extension found at %s: no repo visible as '@%s' here", + usage.getLocation(), + moduleExtensionId.getBzlFileLabel().getRepository().getName()); } extensionUsagesTableBuilder.put(moduleExtensionId, module.getKey(), usage); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index 5200381621fc7e..41cff393000ac1 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -57,6 +57,7 @@ public class BazelLockFileFunction implements SkyFunction { .setFlags(EMPTY_FLAGS) .setLocalOverrideHashes(ImmutableMap.of()) .setModuleDepGraph(ImmutableMap.of()) + .setModuleExtensions(ImmutableMap.of()) .build(); public BazelLockFileFunction(Path rootDirectory) { @@ -75,12 +76,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) return null; } - BazelLockFileValue bazelLockFileValue; try { - String json = FileSystemUtils.readContent(lockfilePath.asPath(), UTF_8); - bazelLockFileValue = LOCKFILE_GSON.fromJson(json, BazelLockFileValue.class); - } catch (FileNotFoundException e) { - bazelLockFileValue = EMPTY_LOCKFILE; + return getLockfileValue(lockfilePath); } catch (IOException | JsonSyntaxException | NullPointerException e) { throw new BazelLockfileFunctionException( ExternalDepsException.withMessage( @@ -90,9 +87,20 @@ public SkyValue compute(SkyKey skyKey, Environment env) e.getMessage()), Transience.PERSISTENT); } + } + + public static BazelLockFileValue getLockfileValue(RootedPath lockfilePath) throws IOException { + BazelLockFileValue bazelLockFileValue; + try { + String json = FileSystemUtils.readContent(lockfilePath.asPath(), UTF_8); + bazelLockFileValue = LOCKFILE_GSON.fromJson(json, BazelLockFileValue.class); + } catch (FileNotFoundException e) { + bazelLockFileValue = EMPTY_LOCKFILE; + } return bazelLockFileValue; } + static final class BazelLockfileFunctionException extends SkyFunctionException { BazelLockfileFunctionException(ExternalDepsException cause, Transience transience) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java index 962c70a8f98d9c..521dd09a1a6dde 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java @@ -17,6 +17,7 @@ import static com.google.devtools.build.lib.bazel.bzlmod.GsonTypeAdapterUtil.LOCKFILE_GSON; import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.collect.ImmutableMap; import com.google.common.eventbus.Subscribe; import com.google.common.flogger.GoogleLogger; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; @@ -24,11 +25,16 @@ import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.Root; import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.gson.JsonSyntaxException; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import javax.annotation.Nullable; /** @@ -38,8 +44,8 @@ public class BazelLockFileModule extends BlazeModule { private Path workspaceRoot; - - @Nullable private BazelLockFileValue lockfileValue; + @Nullable private BazelModuleResolutionEvent moduleResolutionEvent; + private final List extensionResolutionEvents = new ArrayList<>(); private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); @@ -53,23 +59,74 @@ public void beforeCommand(CommandEnvironment env) { } @Override - public void afterCommand() { - if (lockfileValue == null) { // module didn't change --> no event was posted - return; + public void afterCommand() throws AbruptExitException { + if (moduleResolutionEvent == null && extensionResolutionEvents.isEmpty()) { + return; // nothing changed, do nothing! + } + + RootedPath lockfilePath = + RootedPath.toRootedPath(Root.fromPath(workspaceRoot), LabelConstants.MODULE_LOCKFILE_NAME); + + // Create an updated version of the lockfile with the events updates + BazelLockFileValue lockfile; + if (moduleResolutionEvent != null) { + lockfile = moduleResolutionEvent.getLockfileValue(); + } else { + // Read the existing lockfile (if none exists, will get an empty lockfile value) + try { + lockfile = BazelLockFileFunction.getLockfileValue(lockfilePath); + } catch (IOException | JsonSyntaxException | NullPointerException e) { + logger.atSevere().withCause(e).log( + "Failed to read and parse the MODULE.bazel.lock file with error: %s." + + " Try deleting it and rerun the build.", + e.getMessage()); + return; + } } - updateLockfile(workspaceRoot, lockfileValue); - this.lockfileValue = null; + lockfile = + lockfile.toBuilder() + .setModuleExtensions(combineModuleExtensions(lockfile.getModuleExtensions())) + .build(); + + // Write the new value to the file + updateLockfile(lockfilePath, lockfile); + this.moduleResolutionEvent = null; + this.extensionResolutionEvents.clear(); + } + + /** + * Combines the old extensions stored in the lockfile -that are still used- with the new + * extensions from the events (if any) + * + * @param oldModuleExtensions Module extensions stored in the current lockfile + */ + private ImmutableMap combineModuleExtensions( + ImmutableMap oldModuleExtensions) { + ImmutableMap.Builder updatedExtensionMap = + ImmutableMap.builder(); + + // Add the old extensions (stored in the lockfile) only if it still has a usage somewhere + for (Map.Entry extensionEntry : + oldModuleExtensions.entrySet()) { + if (moduleResolutionEvent.getExtensionUsagesById().containsRow(extensionEntry.getKey())) { + updatedExtensionMap.put(extensionEntry); + } + } + + // Add the new resolved extensions + for (ModuleExtensionResolutionEvent extensionEvent : extensionResolutionEvents) { + updatedExtensionMap.put(extensionEvent.getExtensionId(), extensionEvent.getModuleExtension()); + } + return updatedExtensionMap.buildKeepingLast(); } /** * Updates the data stored in the lockfile (MODULE.bazel.lock) * - * @param workspaceRoot Where to update the lockfile + * @param lockfilePath Rooted path to lockfile * @param updatedLockfile The updated lockfile data to save */ - public static void updateLockfile(Path workspaceRoot, BazelLockFileValue updatedLockfile) { - RootedPath lockfilePath = - RootedPath.toRootedPath(Root.fromPath(workspaceRoot), LabelConstants.MODULE_LOCKFILE_NAME); + public static void updateLockfile(RootedPath lockfilePath, BazelLockFileValue updatedLockfile) { try { FileSystemUtils.writeContent( lockfilePath.asPath(), UTF_8, LOCKFILE_GSON.toJson(updatedLockfile)); @@ -80,7 +137,12 @@ public static void updateLockfile(Path workspaceRoot, BazelLockFileValue updated } @Subscribe - public void bazelModuleResolved(BazelLockFileValue lockfileValue) { - this.lockfileValue = lockfileValue; + public void bazelModuleResolved(BazelModuleResolutionEvent moduleResolutionEvent) { + this.moduleResolutionEvent = moduleResolutionEvent; + } + + @Subscribe + public void moduleExtensionResolved(ModuleExtensionResolutionEvent extensionResolutionEvent) { + this.extensionResolutionEvents.add(extensionResolutionEvent); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java index fa93c4f39874e4..c0d0c06aed41e4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java @@ -17,6 +17,7 @@ import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; import com.google.devtools.build.lib.skyframe.SkyFunctions; @@ -24,7 +25,7 @@ import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Map; /** @@ -41,7 +42,9 @@ public abstract class BazelLockFileValue implements SkyValue, Postable { @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_LOCK_FILE; static Builder builder() { - return new AutoValue_BazelLockFileValue.Builder().setLockFileVersion(LOCK_FILE_VERSION); + return new AutoValue_BazelLockFileValue.Builder() + .setLockFileVersion(LOCK_FILE_VERSION) + .setModuleExtensions(ImmutableMap.of()); } /** Current version of the lock file */ @@ -59,6 +62,9 @@ static Builder builder() { /** The post-selection dep graph retrieved from the lock file. */ public abstract ImmutableMap getModuleDepGraph(); + /** Mapping the extension id to the module extension data */ + public abstract ImmutableMap getModuleExtensions(); + public abstract Builder toBuilder(); /** Builder type for {@link BazelLockFileValue}. */ @@ -74,30 +80,55 @@ public abstract static class Builder { public abstract Builder setModuleDepGraph(ImmutableMap value); + public abstract Builder setModuleExtensions( + ImmutableMap value); + public abstract BazelLockFileValue build(); } /** Returns the difference between the lockfile and the current module & flags */ - public ArrayList getDiffLockfile( + public ImmutableList getModuleAndFlagsDiff( String moduleFileHash, ImmutableMap localOverrideHashes, BzlmodFlagsAndEnvVars flags) { - ArrayList diffLockfile = new ArrayList<>(); + ImmutableList.Builder moduleDiff = new ImmutableList.Builder<>(); if (!moduleFileHash.equals(getModuleFileHash())) { - diffLockfile.add("the root MODULE.bazel has been modified"); + moduleDiff.add("the root MODULE.bazel has been modified"); } - diffLockfile.addAll(getFlags().getDiffFlags(flags)); + moduleDiff.addAll(getFlags().getDiffFlags(flags)); for (Map.Entry entry : localOverrideHashes.entrySet()) { String currentValue = entry.getValue(); String lockfileValue = getLocalOverrideHashes().get(entry.getKey()); // If the lockfile value is null, the module hash would be different anyway if (lockfileValue != null && !currentValue.equals(lockfileValue)) { - diffLockfile.add( + moduleDiff.add( "The MODULE.bazel file has changed for the overriden module: " + entry.getKey()); } } + return moduleDiff.build(); + } - return diffLockfile; + public ImmutableList getModuleExtensionDiff( + LockFileModuleExtension lockedExtension, + ImmutableMap lockedExtensionUsages, + ModuleExtensionId extensionId, + byte[] transitiveDigest, + ImmutableMap extensionUsages) { + ImmutableList.Builder extDiff = new ImmutableList.Builder<>(); + if (lockedExtension == null) { + extDiff.add("The module extension '" + extensionId + "' does not exist in the lockfile"); + } else { + if (!Arrays.equals(transitiveDigest, lockedExtension.getBzlTransitiveDigest())) { + extDiff.add( + "The implementation of the extension '" + + extensionId + + "' or one of its transitive .bzl files has changed"); + } + if (!extensionUsages.equals(lockedExtensionUsages)) { + extDiff.add("The usages of the extension named '" + extensionId + "' has changed"); + } + } + return extDiff.build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java new file mode 100644 index 00000000000000..81a0dbaa144b9c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java @@ -0,0 +1,39 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableTable; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; + +/** + * After resolving bazel module this event is sent from {@link BazelDepGraphFunction} holding the + * lockfile value with module updates, and the module extension usages. It will be received in + * {@link BazelLockFileModule} to be used to update the lockfile content + */ +@AutoValue +public abstract class BazelModuleResolutionEvent implements Postable { + + public static BazelModuleResolutionEvent create( + BazelLockFileValue lockFileValue, + ImmutableTable extensionUsagesById) { + return new AutoValue_BazelModuleResolutionEvent(lockFileValue, extensionUsagesById); + } + + public abstract BazelLockFileValue getLockfileValue(); + + public abstract ImmutableTable + getExtensionUsagesById(); +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java index f5f483f716c04b..a69e53791f247f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// package com.google.devtools.build.lib.bazel.bzlmod; @@ -19,7 +18,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; -import java.util.ArrayList; /** Stores the values of flags and environment variables that affect the resolution */ @AutoValue @@ -65,8 +63,8 @@ public static BzlmodFlagsAndEnvVars create( /** Error level of bazel compatability check */ public abstract String compatibilityMode(); - public ArrayList getDiffFlags(BzlmodFlagsAndEnvVars flags) { - ArrayList diffFlags = new ArrayList<>(); + public ImmutableList getDiffFlags(BzlmodFlagsAndEnvVars flags) { + ImmutableList.Builder diffFlags = new ImmutableList.Builder<>(); if (!flags.cmdRegistries().equals(cmdRegistries())) { diffFlags.add("the value of --registry flag has been modified"); } @@ -89,6 +87,6 @@ public ArrayList getDiffFlags(BzlmodFlagsAndEnvVars flags) { if (!flags.compatibilityMode().equals(compatibilityMode())) { diffFlags.add("the value of --check_bazel_compatibility flag has been modified"); } - return diffFlags; + return diffFlags.build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java index 15518989781133..c558e806e9d6d3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java @@ -23,6 +23,8 @@ import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParseException; @@ -36,6 +38,7 @@ import java.io.IOException; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; +import java.util.Base64; import java.util.List; import java.util.Optional; import javax.annotation.Nullable; @@ -96,6 +99,45 @@ public ModuleKey read(JsonReader jsonReader) throws IOException { } }; + // TODO(salmasamy) need to handle "isolated" in module extensions when it is stable + public static final TypeAdapter MODULE_EXTENSION_ID_TYPE_ADAPTER = + new TypeAdapter<>() { + @Override + public void write(JsonWriter jsonWriter, ModuleExtensionId moduleExtId) throws IOException { + jsonWriter.value(moduleExtId.getBzlFileLabel() + "%" + moduleExtId.getExtensionName()); + } + + @Override + public ModuleExtensionId read(JsonReader jsonReader) throws IOException { + String jsonString = jsonReader.nextString(); + // [0] is labelString, [1] is extensionName + List extIdParts = Splitter.on("%").splitToList(jsonString); + try { + return ModuleExtensionId.create( + Label.parseCanonical(extIdParts.get(0)), extIdParts.get(1), Optional.empty()); + } catch (LabelSyntaxException e) { + throw new JsonParseException( + String.format( + "Unable to parse ModuleExtensionID bzl file label: '%s' from the lockfile", + extIdParts.get(0)), + e); + } + } + }; + + public static final TypeAdapter BYTE_ARRAY_TYPE_ADAPTER = + new TypeAdapter<>() { + @Override + public void write(JsonWriter jsonWriter, byte[] value) throws IOException { + jsonWriter.value(Base64.getEncoder().encodeToString(value)); + } + + @Override + public byte[] read(JsonReader jsonReader) throws IOException { + return Base64.getDecoder().decode(jsonReader.nextString()); + } + }; + public static final TypeAdapterFactory OPTIONAL = new TypeAdapterFactory() { @Nullable @@ -149,6 +191,8 @@ public Optional read(JsonReader jsonReader) throws IOException { public static final Gson LOCKFILE_GSON = new GsonBuilder() .setPrettyPrinting() + .disableHtmlEscaping() + .enableComplexMapKeySerialization() .registerTypeAdapterFactory(GenerateTypeAdapter.FACTORY) .registerTypeAdapterFactory(DICT) .registerTypeAdapterFactory(IMMUTABLE_MAP) @@ -158,7 +202,9 @@ public Optional read(JsonReader jsonReader) throws IOException { .registerTypeAdapterFactory(OPTIONAL) .registerTypeAdapter(Version.class, VERSION_TYPE_ADAPTER) .registerTypeAdapter(ModuleKey.class, MODULE_KEY_TYPE_ADAPTER) + .registerTypeAdapter(ModuleExtensionId.class, MODULE_EXTENSION_ID_TYPE_ADAPTER) .registerTypeAdapter(AttributeValues.class, new AttributeValuesAdapter()) + .registerTypeAdapter(byte[].class, BYTE_ARRAY_TYPE_ADAPTER) .create(); private GsonTypeAdapterUtil() {} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java new file mode 100644 index 00000000000000..af4da1e131ee18 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java @@ -0,0 +1,40 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; + +/** + * This object serves as a container for the transitive digest (obtained from transitive .bzl files) + * and the generated repositories from evaluating a module extension. Its purpose is to store this + * information within the lockfile. + */ +@AutoValue +@GenerateTypeAdapter +public abstract class LockFileModuleExtension implements Postable { + + @SuppressWarnings("AutoValueImmutableFields") + public abstract byte[] getBzlTransitiveDigest(); + + public abstract ImmutableMap getGeneratedRepoSpecs(); + + public static LockFileModuleExtension create( + byte[] transitiveDigest, ImmutableMap generatedRepoSpecs) { + return new AutoValue_LockFileModuleExtension(transitiveDigest, generatedRepoSpecs); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java new file mode 100644 index 00000000000000..4013e7bb4c5965 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java @@ -0,0 +1,36 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; + +/** + * After evaluating any module extension this event is sent from {@link SingleExtensionEvalFunction} + * holding the extension id and the resolution data LockFileModuleExtension. It will be received in + * {@link BazelLockFileModule} to be used to update the lockfile content + */ +@AutoValue +public abstract class ModuleExtensionResolutionEvent implements Postable { + + public static ModuleExtensionResolutionEvent create( + ModuleExtensionId extensionId, LockFileModuleExtension lockfileModuleExtension) { + return new AutoValue_ModuleExtensionResolutionEvent(extensionId, lockfileModuleExtension); + } + + public abstract ModuleExtensionId getExtensionId(); + + public abstract LockFileModuleExtension getModuleExtension(); +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java index b76131c9761da0..29a32ef988ec17 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java @@ -18,10 +18,15 @@ import static com.google.common.collect.ImmutableBiMap.toImmutableBiMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableTable; import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.cmdline.BazelModuleContext; +import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -29,6 +34,7 @@ import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException; import com.google.devtools.build.lib.runtime.ProcessWrapper; import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor; +import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; import com.google.devtools.build.lib.skyframe.BzlLoadFunction; import com.google.devtools.build.lib.skyframe.BzlLoadFunction.BzlLoadFailedException; @@ -42,6 +48,7 @@ import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Map; import java.util.Map.Entry; import java.util.function.Function; @@ -94,7 +101,7 @@ public void setRepositoryRemoteExecutor(RepositoryRemoteExecutor repositoryRemot @Nullable @Override public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { + throws SingleExtensionEvalFunctionException, InterruptedException { StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env); if (starlarkSemantics == null) { return null; @@ -108,11 +115,163 @@ public SkyValue compute(SkyKey skyKey, Environment env) } Location sampleUsageLocation = usagesValue.getExtensionUsages().values().iterator().next().getLocation(); + BzlLoadValue bzlLoadValue = + loadBzlFile(extensionId.getBzlFileLabel(), sampleUsageLocation, starlarkSemantics, env); + if (bzlLoadValue == null) { + return null; + } + // TODO(wyv): Consider whether there's a need to check .bzl load visibility + // (BzlLoadFunction#checkLoadVisibilities). + // TODO(wyv): Consider refactoring to use PackageFunction#loadBzlModules, or the simpler API + // that may be created by b/237658764. + + // Check that the .bzl file actually exports a module extension by our name. + Object exported = bzlLoadValue.getModule().getGlobal(extensionId.getExtensionName()); + if (!(exported instanceof ModuleExtension)) { + ImmutableSet exportedExtensions = + bzlLoadValue.getModule().getGlobals().entrySet().stream() + .filter(e -> e.getValue() instanceof ModuleExtension) + .map(Entry::getKey) + .collect(toImmutableSet()); + throw new SingleExtensionEvalFunctionException( + ExternalDepsException.withMessage( + ExternalDeps.Code.BAD_MODULE, + "%s does not export a module extension called %s, yet its use is requested at %s%s", + extensionId.getBzlFileLabel(), + extensionId.getExtensionName(), + sampleUsageLocation, + SpellChecker.didYouMean(extensionId.getExtensionName(), exportedExtensions)), + Transience.PERSISTENT); + } + + // Check the lockfile first for that module extension + byte[] bzlTransitiveDigest = + BazelModuleContext.of(bzlLoadValue.getModule()).bzlTransitiveDigest(); + LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env); + if (!lockfileMode.equals(LockfileMode.OFF)) { + BazelLockFileValue lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); + if (lockfile == null) { + return null; + } + SingleExtensionEvalValue singleExtensionEvalValue = + tryGettingValueFromLockFile( + extensionId, usagesValue, bzlTransitiveDigest, lockfileMode, lockfile); + if (singleExtensionEvalValue != null) { + return singleExtensionEvalValue; + } + } + + // Run that extension! + ModuleExtension extension = (ModuleExtension) exported; + ImmutableMap generatedRepoSpecs = + runModuleExtension( + extensionId, extension, usagesValue, bzlLoadValue.getModule(), starlarkSemantics, env); + if (generatedRepoSpecs == null) { + return null; + } + // Check that all imported repos have been actually generated + validateAllImportsAreGenerated(generatedRepoSpecs, usagesValue, extensionId); + + if (lockfileMode.equals(LockfileMode.UPDATE)) { + env.getListener() + .post( + ModuleExtensionResolutionEvent.create( + extensionId, + LockFileModuleExtension.create(bzlTransitiveDigest, generatedRepoSpecs))); + } + return createSingleExtentionValue(generatedRepoSpecs, usagesValue); + } + + @Nullable + private SingleExtensionEvalValue tryGettingValueFromLockFile( + ModuleExtensionId extensionId, + SingleExtensionUsagesValue usagesValue, + byte[] bzlTransitiveDigest, + LockfileMode lockfileMode, + BazelLockFileValue lockfile) + throws SingleExtensionEvalFunctionException { + LockFileModuleExtension lockedExtension = lockfile.getModuleExtensions().get(extensionId); + ImmutableMap lockedExtensionUsages; + try { + // TODO(salmasamy) might be nicer to precompute this table when we construct + // BazelLockFileValue, without adding it to the json file + ImmutableTable extensionUsagesById = + BazelDepGraphFunction.getExtensionUsagesById(lockfile.getModuleDepGraph()); + lockedExtensionUsages = extensionUsagesById.row(extensionId); + } catch (ExternalDepsException e) { + throw new SingleExtensionEvalFunctionException(e, Transience.PERSISTENT); + } + + // If we have the extension, check if the implementation and usage haven't changed + if (lockedExtension != null + && Arrays.equals(bzlTransitiveDigest, lockedExtension.getBzlTransitiveDigest()) + && usagesValue.getExtensionUsages().equals(lockedExtensionUsages)) { + return createSingleExtentionValue(lockedExtension.getGeneratedRepoSpecs(), usagesValue); + } else if (lockfileMode.equals(LockfileMode.ERROR)) { + ImmutableList extDiff = + lockfile.getModuleExtensionDiff( + lockedExtension, + lockedExtensionUsages, + extensionId, + bzlTransitiveDigest, + usagesValue.getExtensionUsages()); + throw new SingleExtensionEvalFunctionException( + ExternalDepsException.withMessage( + Code.BAD_MODULE, + "Lock file is no longer up-to-date because: %s", + String.join(", ", extDiff)), + Transience.PERSISTENT); + } + return null; + } + private SingleExtensionEvalValue createSingleExtentionValue( + ImmutableMap generatedRepoSpecs, SingleExtensionUsagesValue usagesValue) { + return SingleExtensionEvalValue.create( + generatedRepoSpecs, + generatedRepoSpecs.keySet().stream() + .collect( + toImmutableBiMap( + e -> + RepositoryName.createUnvalidated( + usagesValue.getExtensionUniqueName() + "~" + e), + Function.identity()))); + } + + private void validateAllImportsAreGenerated( + ImmutableMap generatedRepoSpecs, + SingleExtensionUsagesValue usagesValue, + ModuleExtensionId extensionId) + throws SingleExtensionEvalFunctionException { + for (ModuleExtensionUsage usage : usagesValue.getExtensionUsages().values()) { + for (Entry repoImport : usage.getImports().entrySet()) { + if (!generatedRepoSpecs.containsKey(repoImport.getValue())) { + throw new SingleExtensionEvalFunctionException( + ExternalDepsException.withMessage( + Code.BAD_MODULE, + "module extension \"%s\" from \"%s\" does not generate repository \"%s\", yet it" + + " is imported as \"%s\" in the usage at %s%s", + extensionId.getExtensionName(), + extensionId.getBzlFileLabel(), + repoImport.getValue(), + repoImport.getKey(), + usage.getLocation(), + SpellChecker.didYouMean(repoImport.getValue(), generatedRepoSpecs.keySet())), + Transience.PERSISTENT); + } + } + } + } + + private BzlLoadValue loadBzlFile( + Label bzlFileLabel, + Location sampleUsageLocation, + StarlarkSemantics starlarkSemantics, + Environment env) + throws SingleExtensionEvalFunctionException, InterruptedException { // Check that the .bzl label isn't crazy. try { - BzlLoadFunction.checkValidLoadLabel( - extensionId.getBzlFileLabel(), /*fromBuiltinsRepo=*/ false); + BzlLoadFunction.checkValidLoadLabel(bzlFileLabel, starlarkSemantics); } catch (LabelSyntaxException e) { throw new SingleExtensionEvalFunctionException( ExternalDepsException.withCauseAndMessage( @@ -126,53 +285,35 @@ public SkyValue compute(SkyKey skyKey, Environment env) bzlLoadValue = (BzlLoadValue) env.getValueOrThrow( - BzlLoadValue.keyForBzlmod(extensionId.getBzlFileLabel()), - BzlLoadFailedException.class); + BzlLoadValue.keyForBzlmod(bzlFileLabel), BzlLoadFailedException.class); } catch (BzlLoadFailedException e) { throw new SingleExtensionEvalFunctionException( ExternalDepsException.withCauseAndMessage( Code.BAD_MODULE, e, "Error loading '%s' for module extensions, requested by %s: %s", - extensionId.getBzlFileLabel(), + bzlFileLabel, sampleUsageLocation, e.getMessage()), Transience.PERSISTENT); } - if (bzlLoadValue == null) { - return null; - } - // TODO(wyv): Consider whether there's a need to check .bzl load visibility - // (BzlLoadFunction#checkLoadVisibilities). - // TODO(wyv): Consider refactoring to use PackageFunction#loadBzlModules, or the simpler API - // that may be created by b/237658764. - - // Check that the .bzl file actually exports a module extension by our name. - Object exported = bzlLoadValue.getModule().getGlobal(extensionId.getExtensionName()); - if (!(exported instanceof ModuleExtension.InStarlark)) { - ImmutableSet exportedExtensions = - bzlLoadValue.getModule().getGlobals().entrySet().stream() - .filter(e -> e.getValue() instanceof ModuleExtension.InStarlark) - .map(Entry::getKey) - .collect(toImmutableSet()); - throw new SingleExtensionEvalFunctionException( - ExternalDepsException.withMessage( - Code.BAD_MODULE, - "%s does not export a module extension called %s, yet its use is requested at %s%s", - extensionId.getBzlFileLabel(), - extensionId.getExtensionName(), - sampleUsageLocation, - SpellChecker.didYouMean(extensionId.getExtensionName(), exportedExtensions)), - Transience.PERSISTENT); - } + return bzlLoadValue; + } - // Run that extension! - ModuleExtension extension = ((ModuleExtension.InStarlark) exported).get(); + @Nullable + private ImmutableMap runModuleExtension( + ModuleExtensionId extensionId, + ModuleExtension extension, + SingleExtensionUsagesValue usagesValue, + net.starlark.java.eval.Module module, + StarlarkSemantics starlarkSemantics, + Environment env) + throws SingleExtensionEvalFunctionException, InterruptedException { ModuleExtensionEvalStarlarkThreadContext threadContext = new ModuleExtensionEvalStarlarkThreadContext( usagesValue.getExtensionUniqueName() + "~", extensionId.getBzlFileLabel().getPackageIdentifier(), - BazelModuleContext.of(bzlLoadValue.getModule()).repoMapping(), + BazelModuleContext.of(module).repoMapping(), directories, env.getListener()); try (Mutability mu = @@ -189,7 +330,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (returnValue != Starlark.NONE && !(returnValue instanceof ModuleExtensionMetadata)) { throw new SingleExtensionEvalFunctionException( ExternalDepsException.withMessage( - Code.BAD_MODULE, + ExternalDeps.Code.BAD_MODULE, "expected module extension %s in %s to return None or extension_metadata, got %s", extensionId.getExtensionName(), extensionId.getBzlFileLabel(), @@ -210,51 +351,27 @@ public SkyValue compute(SkyKey skyKey, Environment env) moduleContext.getWorkingDirectory().deleteTree(); } } catch (IOException e1) { - throw new SingleExtensionEvalFunctionException(e1, Transience.TRANSIENT); + ExternalDepsException externalDepsException = + ExternalDepsException.withCauseAndMessage( + ExternalDeps.Code.UNRECOGNIZED, + e1, + "Failed to clean up module context directory"); + throw new SingleExtensionEvalFunctionException( + externalDepsException, Transience.TRANSIENT); } return null; } catch (EvalException e) { env.getListener().handle(Event.error(e.getMessageWithStack())); throw new SingleExtensionEvalFunctionException( ExternalDepsException.withMessage( - Code.BAD_MODULE, + ExternalDeps.Code.BAD_MODULE, "error evaluating module extension %s in %s", extensionId.getExtensionName(), extensionId.getBzlFileLabel()), Transience.TRANSIENT); } } - - // Check that all imported repos have been actually generated - for (ModuleExtensionUsage usage : usagesValue.getExtensionUsages().values()) { - for (Entry repoImport : usage.getImports().entrySet()) { - if (!threadContext.getGeneratedRepoSpecs().containsKey(repoImport.getValue())) { - throw new SingleExtensionEvalFunctionException( - ExternalDepsException.withMessage( - Code.BAD_MODULE, - "module extension \"%s\" from \"%s\" does not generate repository \"%s\", yet it" - + " is imported as \"%s\" in the usage at %s%s", - extensionId.getExtensionName(), - extensionId.getBzlFileLabel(), - repoImport.getValue(), - repoImport.getKey(), - usage.getLocation(), - SpellChecker.didYouMean( - repoImport.getValue(), threadContext.getGeneratedRepoSpecs().keySet())), - Transience.PERSISTENT); - } - } - } - - return SingleExtensionEvalValue.create( - threadContext.getGeneratedRepoSpecs(), - threadContext.getGeneratedRepoSpecs().keySet().stream() - .collect( - toImmutableBiMap( - e -> - RepositoryName.createUnvalidated( - usagesValue.getExtensionUniqueName() + "~" + e), - Function.identity()))); + return threadContext.getGeneratedRepoSpecs(); } private ModuleExtensionContext createContext( @@ -298,7 +415,7 @@ private ModuleExtensionContext createContext( static final class SingleExtensionEvalFunctionException extends SkyFunctionException { - SingleExtensionEvalFunctionException(Exception cause, Transience transience) { + SingleExtensionEvalFunctionException(ExternalDepsException cause, Transience transience) { super(cause, transience); } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index 269869f3866d92..4efcebc1ece290 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -34,6 +34,7 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule; import com.google.devtools.build.lib.clock.BlazeClock; +import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; import com.google.devtools.build.lib.rules.repository.LocalRepositoryFunction; @@ -56,6 +57,7 @@ import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; import com.google.devtools.build.lib.vfs.FileStateKey; import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; @@ -177,14 +179,18 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (localOverrideHashes == null) { return null; } + RootedPath lockfilePath = + RootedPath.toRootedPath( + Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); BazelLockFileModule.updateLockfile( - rootDirectory, + lockfilePath, BazelLockFileValue.builder() .setModuleFileHash(key.moduleHash()) .setFlags(flags) .setLocalOverrideHashes(localOverrideHashes) .setModuleDepGraph(key.depGraph()) .build()); + return new SkyValue() {}; } }) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 84e6defea0e153..d73b2d479795ae 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -282,7 +282,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); // Set up a simple repo rule. registry.addModule( diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py index f9fe557bbd1851..c91173e323914d 100644 --- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -14,6 +14,7 @@ # limitations under the License. # pylint: disable=g-long-ternary +import json import os import tempfile import unittest @@ -241,12 +242,7 @@ def testLocalOverrideWithErrorMode(self): ], ) exit_code, _, stderr = self.RunBazel( - [ - 'build', - '--nobuild', - '--lockfile_mode=error', - '//:all', - ], + ['build', '--nobuild', '--lockfile_mode=error', '//:all'], allow_failure=True, ) self.AssertExitCode(exit_code, 48, stderr) @@ -259,6 +255,241 @@ def testLocalOverrideWithErrorMode(self): stderr, ) + def testModuleExtension(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'lockfile_ext = use_extension("extension.bzl", "lockfile_ext")', + 'lockfile_ext.dep(name = "bmbm", versions = ["v1", "v2"])', + 'use_repo(lockfile_ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + '', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + '', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other side!")', + ' repo_rule(name="hello")', + ' for mod in ctx.modules:', + ' for dep in mod.tags.dep:', + ' print("Name:", dep.name, ", Versions:", dep.versions)', + '', + ( + '_dep = tag_class(attrs={"name": attr.string(), "versions":' + ' attr.string_list()})' + ), + 'lockfile_ext = module_extension(', + ' implementation=_module_ext_impl,', + ' tag_classes={"dep": _dep},', + ')', + ], + ) + + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertIn('Hello from the other side!', ''.join(stderr)) + self.assertIn('Name: bmbm , Versions: ["v1", "v2"]', ''.join(stderr)) + + self.RunBazel(['shutdown']) + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertNotIn('Hello from the other side!', ''.join(stderr)) + + def testModuleExtensionsInDifferentBuilds(self): + # Test that the module extension stays in the lockfile (as long as it's + # used in the module) even if it is not in the current build + self.ScratchFile( + 'MODULE.bazel', + [ + 'extA = use_extension("extension.bzl", "extA")', + 'extB = use_extension("extension.bzl", "extB")', + 'use_repo(extA, "hello_ext_A")', + 'use_repo(extB, "hello_ext_B")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + 'def _ext_a_impl(ctx):', + ' repo_rule(name="hello_ext_A")', + 'def _ext_b_impl(ctx):', + ' repo_rule(name="hello_ext_B")', + 'extA = module_extension(implementation=_ext_a_impl)', + 'extB = module_extension(implementation=_ext_b_impl)', + ], + ) + + self.RunBazel(['build', '@hello_ext_A//:all']) + self.RunBazel(['build', '@hello_ext_B//:all']) + + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + self.assertGreater(len(lockfile['moduleDepGraph']), 0) + self.assertEqual( + list(lockfile['moduleExtensions'].keys()), + ['//:extension.bzl%extA', '//:extension.bzl%extB'], + ) + + def testUpdateModuleExtension(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'lockfile_ext = use_extension("extension.bzl", "lockfile_ext")', + 'use_repo(lockfile_ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\\"lala\\")")', + 'repo_rule = repository_rule(implementation = _repo_rule_impl)', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other side!")', + ' repo_rule(name= "hello")', + ( + 'lockfile_ext = module_extension(implementation =' + ' _module_ext_impl)' + ), + ], + ) + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertIn('Hello from the other side!', ''.join(stderr)) + self.RunBazel(['shutdown']) + + # Update extension. Make sure that it is executed and updated in the + # lockfile without errors (since it's already in the lockfile) + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\\"lala\\")")', + 'repo_rule = repository_rule(implementation = _repo_rule_impl)', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other town!")', + ' repo_rule(name= "hello")', + ( + 'lockfile_ext = module_extension(implementation =' + ' _module_ext_impl)' + ), + ], + ) + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertIn('Hello from the other town!', ''.join(stderr)) + + def testUpdateModuleExtensionErrorMode(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'lockfile_ext = use_extension("extension.bzl", "lockfile_ext")', + 'use_repo(lockfile_ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\\"lala\\")")', + 'repo_rule = repository_rule(implementation = _repo_rule_impl)', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other side!")', + ' repo_rule(name= "hello")', + ( + 'lockfile_ext = module_extension(implementation =' + ' _module_ext_impl)' + ), + ], + ) + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertIn('Hello from the other side!', ''.join(stderr)) + self.RunBazel(['shutdown']) + + # Update extension. + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\\"lalo\\")")', + 'repo_rule = repository_rule(implementation = _repo_rule_impl)', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other town!")', + ' repo_rule(name= "hello")', + ( + 'lockfile_ext = module_extension(implementation =' + ' _module_ext_impl)' + ), + ], + ) + + exit_code, _, stderr = self.RunBazel( + ['build', '--nobuild', '--lockfile_mode=error', '@hello//:all'], + allow_failure=True, + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + ( + 'ERROR: Lock file is no longer up-to-date because: The ' + 'implementation of the extension ' + "'ModuleExtensionId{bzlFileLabel=//:extension.bzl, " + "extensionName=lockfile_ext, isolationKey=Optional.empty}' or one " + 'of its transitive .bzl files has changed' + ), + stderr, + ) + + def testRemoveModuleExtensionsNotUsed(self): + # Test that the module extension is removed from the lockfile if it is not + # used anymore + self.ScratchFile( + 'MODULE.bazel', + [ + 'ext = use_extension("extension.bzl", "ext")', + 'use_repo(ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + 'def _ext_impl(ctx):', + ' repo_rule(name="hello")', + 'ext = module_extension(implementation=_ext_impl)', + ], + ) + + self.RunBazel(['build', '@hello//:all']) + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + self.assertEqual( + list(lockfile['moduleExtensions'].keys()), ['//:extension.bzl%ext'] + ) + + self.ScratchFile('MODULE.bazel', []) + self.RunBazel(['build', '//:all']) + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + self.assertEqual(len(lockfile['moduleExtensions']), 0) + if __name__ == '__main__': unittest.main() From ee6463176e7363800ea16e13447201af8bb316e0 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Mon, 10 Jul 2023 19:45:11 +0200 Subject: [PATCH 04/20] fixes --- .../devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java | 2 +- .../build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java | 2 +- .../build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index 2852c88116e9a3..f6d863f7e0cc8c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -222,7 +222,7 @@ static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws Interrup try { moduleExtensionId = ModuleExtensionId.create( - labelConverter.convert(usage.getExtensionBzlFile()), usage.getExtensionName()); + labelConverter.convert(usage.getExtensionBzlFile()), usage.getExtensionName(), usage.getIsolationKey()); } catch (LabelSyntaxException e) { throw ExternalDepsException.withCauseAndMessage( Code.BAD_MODULE, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java index 29a32ef988ec17..6510c3698b9b5a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java @@ -271,7 +271,7 @@ private BzlLoadValue loadBzlFile( throws SingleExtensionEvalFunctionException, InterruptedException { // Check that the .bzl label isn't crazy. try { - BzlLoadFunction.checkValidLoadLabel(bzlFileLabel, starlarkSemantics); + BzlLoadFunction.checkValidLoadLabel(bzlFileLabel, /*fromBuiltinsRepo=*/ false); } catch (LabelSyntaxException e) { throw new SingleExtensionEvalFunctionException( ExternalDepsException.withCauseAndMessage( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index 4efcebc1ece290..5a50de0353be14 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -96,7 +96,7 @@ public void setup() throws Exception { differencer = new SequencedRecordingDifferencer(); registryFactory = new FakeRegistry.Factory(); evaluationContext = - EvaluationContext.newBuilder().setParallelism(8).setEventHandler(reporter).build(); + EvaluationContext.newBuilder().setNumThreads(8).setEventHandler(reporter).build(); AtomicReference packageLocator = new AtomicReference<>( From acc5fa78a2dfdcc0253b0dd9d19b1521195704d8 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Tue, 11 Jul 2023 00:15:39 +0200 Subject: [PATCH 05/20] Fix tests --- .../build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java index 6510c3698b9b5a..a3ee9af7e88468 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java @@ -127,7 +127,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) // Check that the .bzl file actually exports a module extension by our name. Object exported = bzlLoadValue.getModule().getGlobal(extensionId.getExtensionName()); - if (!(exported instanceof ModuleExtension)) { + if (!(exported instanceof ModuleExtension.InStarlark)) { ImmutableSet exportedExtensions = bzlLoadValue.getModule().getGlobals().entrySet().stream() .filter(e -> e.getValue() instanceof ModuleExtension) @@ -162,7 +162,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) } // Run that extension! - ModuleExtension extension = (ModuleExtension) exported; + ModuleExtension extension = ((ModuleExtension.InStarlark) exported).get(); ImmutableMap generatedRepoSpecs = runModuleExtension( extensionId, extension, usagesValue, bzlLoadValue.getModule(), starlarkSemantics, env); From a5a6276af0fc1861ad75838858a7099a3b626c1d Mon Sep 17 00:00:00 2001 From: salma-samy Date: Thu, 6 Jul 2023 02:50:15 -0700 Subject: [PATCH 06/20] Enable lockfile by default and fix warning PiperOrigin-RevId: 545927412 Change-Id: I34e8531b8e396ccdfe0eecbee22cb68f76f969fc # Conflicts: # src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java # src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java # src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java # src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java # src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java --- .../lib/bazel/BazelRepositoryModule.java | 2 +- .../bazel/bzlmod/LockFileModuleExtension.java | 2 +- .../bazel/repository/RepositoryOptions.java | 2 +- .../RunfilesRepoMappingManifestTest.java | 13 +- .../StarlarkRuleTransitionProviderTest.java | 2 +- .../lib/analysis/util/AnalysisTestCase.java | 4 +- .../bzlmod/BazelDepGraphFunctionTest.java | 3 +- .../bzlmod/BazelLockFileFunctionTest.java | 11 +- .../BazelModuleResolutionFunctionTest.java | 8 +- .../bzlmod/BzlmodRepoRuleFunctionTest.java | 2 +- .../query2/testutil/SkyframeQueryHelper.java | 32 +- .../lib/rules/LabelBuildSettingTest.java | 2 +- .../repository/RepositoryDelegatorTest.java | 3 +- .../StarlarkDocExtractTest.java | 795 ++++++++++++++++++ .../lib/skyframe/BzlLoadFunctionTest.java | 2 +- .../PrepareDepsOfPatternsFunctionTest.java | 2 +- ...isteredExecutionPlatformsFunctionTest.java | 2 +- .../RegisteredToolchainsFunctionTest.java | 2 +- .../RepositoryMappingFunctionTest.java | 2 +- 19 files changed, 846 insertions(+), 45 deletions(-) create mode 100644 src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 1e19a99de35e2d..04b0a36d7a4a84 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -151,7 +151,7 @@ public class BazelRepositoryModule extends BlazeModule { private final AtomicBoolean ignoreDevDeps = new AtomicBoolean(false); private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING; private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR; - private LockfileMode bazelLockfileMode = LockfileMode.OFF; + private LockfileMode bazelLockfileMode = LockfileMode.UPDATE; private List allowedYankedVersions = ImmutableList.of(); private SingleExtensionEvalFunction singleExtensionEvalFunction; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java index af4da1e131ee18..5431d6cf0fabe6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java @@ -28,7 +28,7 @@ @GenerateTypeAdapter public abstract class LockFileModuleExtension implements Postable { - @SuppressWarnings("AutoValueImmutableFields") + @SuppressWarnings("mutable") public abstract byte[] getBzlTransitiveDigest(); public abstract ImmutableMap getGeneratedRepoSpecs(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java index f2cbea78e44119..d11631b4556502 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java @@ -277,8 +277,8 @@ public class RepositoryOptions extends OptionsBase { @Option( name = "lockfile_mode", - defaultValue = "off", // TODO(salmasamy) later will be changed to 'update' converter = LockfileMode.Converter.class, + defaultValue = "update", documentationCategory = OptionDocumentationCategory.BZLMOD, effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS}, help = diff --git a/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java b/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java index 377164876ce25a..e05873c05020d7 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java @@ -59,9 +59,16 @@ protected ImmutableList extraPrecomputedValues() throws Exception { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF), - PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of())); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE), + PrecomputedValue.injected(BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of())); + } + + @Override + protected SkyframeExecutorRepositoryHelpersHolder getRepositoryHelpersHolder() { + // Transitive packages are needed for RepoMappingManifestAction and are only stored when + // external repositories are enabled. + return SkyframeExecutorRepositoryHelpersHolder.create( + new RepositoryDirectoryDirtinessChecker()); } @Before diff --git a/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java b/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java index ea8cf9f5106196..4876868cb7520d 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java @@ -77,7 +77,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Override diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java index 373668e06a2c62..51b0eef293b7c2 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java @@ -245,7 +245,7 @@ protected void useRuleClassProvider(ConfiguredRuleClassProvider ruleClassProvide BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), PrecomputedValue.injected( - BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF))) + BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE))) .build(ruleClassProvider, fileSystem); useConfiguration(); skyframeExecutor = createSkyframeExecutor(pkgFactory); @@ -294,7 +294,7 @@ private void reinitializeSkyframeExecutor() { PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.WARNING), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF))); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE))); } /** Resets the SkyframeExecutor, as if a clean had been executed. */ diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index dfdab83e416e2a..7b00d9ea308fb3 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -139,7 +139,6 @@ public void setup() throws Exception { PrecomputedValue.STARLARK_SEMANTICS.set( differencer, StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build()); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); @@ -147,7 +146,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.OFF); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index 5a50de0353be14..9461583a3d57f1 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -208,7 +208,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) differencer, BazelCompatibilityMode.ERROR); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.set(differencer, ImmutableMap.of()); RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING.set( differencer, RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY); @@ -285,10 +285,9 @@ public void moduleWithFlags() throws Exception { ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of("my_dep_1", override)); BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, yankedVersions); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( - differencer, CheckDirectDepsMode.ERROR); + differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( - differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + differencer, BazelCompatibilityMode.WARNING); UpdateLockFileKey key = UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); @@ -310,9 +309,9 @@ public void moduleWithFlags() throws Exception { assertThat(value.getFlags().cmdModuleOverrides()).isEqualTo(moduleOverride); assertThat(value.getFlags().allowedYankedVersions()).isEqualTo(yankedVersions); assertThat(value.getFlags().directDependenciesMode()) - .isEqualTo(CheckDirectDepsMode.ERROR.toString()); + .isEqualTo(CheckDirectDepsMode.WARNING.toString()); assertThat(value.getFlags().compatibilityMode()) - .isEqualTo(BazelCompatibilityMode.ERROR.toString()); + .isEqualTo(BazelCompatibilityMode.WARNING.toString()); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 427450dd5cd237..8186818cd1f50d 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -130,7 +130,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.OFF); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } @@ -291,8 +291,7 @@ public void testYankedVersionCheckIgnoredByAll() throws Exception { @Test public void testYankedVersionCheckIgnoredBySpecific() throws Exception { setupModulesForYankedVersion(); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set( - differencer, ImmutableList.of("b@1.0")); + BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of("b@1.0")); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); assertThat(result.hasError()).isFalse(); @@ -301,8 +300,7 @@ public void testYankedVersionCheckIgnoredBySpecific() throws Exception { @Test public void testBadYankedVersionFormat() throws Exception { setupModulesForYankedVersion(); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set( - differencer, ImmutableList.of("b~1.0")); + BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of("b~1.0")); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); assertThat(result.hasError()).isTrue(); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index d1a4ef5b5037c2..92ff555778779d 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -145,7 +145,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java index 6e705ff8b87833..ef99119130ef3a 100644 --- a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java +++ b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java @@ -23,7 +23,6 @@ import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ServerDirectories; -import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; @@ -36,9 +35,8 @@ import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.cmdline.RepositoryName; -import com.google.devtools.build.lib.packages.ConstantRuleVisibility; import com.google.devtools.build.lib.packages.PackageFactory; -import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension; +import com.google.devtools.build.lib.packages.RuleVisibility; import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.packages.util.MockToolsConfig; @@ -59,9 +57,9 @@ import com.google.devtools.build.lib.query2.engine.QueryUtil.AggregateAllOutputFormatterCallback; import com.google.devtools.build.lib.query2.engine.ThreadSafeOutputFormatterCallback; import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; +import com.google.devtools.build.lib.runtime.QuiescingExecutorsImpl; import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; import com.google.devtools.build.lib.skyframe.IgnoredPackagePrefixesFunction; -import com.google.devtools.build.lib.skyframe.PackageValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.SkyframeExecutor; import com.google.devtools.build.lib.skyframe.SkyframeTargetPatternEvaluator; @@ -149,6 +147,11 @@ public void setUp() throws Exception { this.queryEnvironmentFactory = makeQueryEnvironmentFactory(); } + @Override + public final void cleanUp() { + skyframeExecutor.getEvaluator().cleanupInterningPools(); + } + protected abstract String getRootDirectoryNameForSetup(); protected abstract void performAdditionalClientSetup(MockToolsConfig mockToolsConfig) @@ -312,10 +315,13 @@ protected boolean enableBzlmod() { protected void initTargetPatternEvaluator(ConfiguredRuleClassProvider ruleClassProvider) { this.toolsRepository = ruleClassProvider.getToolsRepository(); + if (skyframeExecutor != null) { + cleanUp(); + } skyframeExecutor = createSkyframeExecutor(ruleClassProvider); PackageOptions packageOptions = Options.getDefaults(PackageOptions.class); - packageOptions.defaultVisibility = ConstantRuleVisibility.PRIVATE; + packageOptions.defaultVisibility = RuleVisibility.PRIVATE; packageOptions.showLoadingProgress = true; packageOptions.globbingThreads = 7; packageOptions.packagePath = ImmutableList.of(rootDirectory.getPathString()); @@ -334,6 +340,7 @@ protected void initTargetPatternEvaluator(ConfiguredRuleClassProvider ruleClassP ImmutableMap.of(), ImmutableMap.of(), new TimestampGranularityMonitor(BlazeClock.instance()), + QuiescingExecutorsImpl.forTesting(), FakeOptions.builder().put(packageOptions).put(buildLanguageOptions).build()); } catch (InterruptedException | AbruptExitException e) { throw new IllegalStateException(e); @@ -372,8 +379,7 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), PrecomputedValue.injected( - BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF))) - .setEnvironmentExtensions(getEnvironmentExtensions()) + BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE))) .build(ruleClassProvider, fileSystem); SkyframeExecutor skyframeExecutor = BazelSkyframeExecutorConstants.newBazelSkyframeExecutorBuilder() @@ -384,7 +390,7 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru .setIgnoredPackagePrefixesFunction( new IgnoredPackagePrefixesFunction(ignoredPackagePrefixesFile)) .setExtraSkyFunctions(analysisMock.getSkyFunctions(directories)) - .setPerCommandSyscallCache(delegatingSyscallCache) + .setSyscallCache(delegatingSyscallCache) .build(); skyframeExecutor.injectExtraPrecomputedValues( ImmutableList.builder() @@ -415,21 +421,17 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR)) - .add(PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)) + .add( + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)) .build()); SkyframeExecutorTestHelper.process(skyframeExecutor); return skyframeExecutor; } - protected abstract Iterable getEnvironmentExtensions(); - - protected abstract BuildOptions getDefaultBuildOptions( - ConfiguredRuleClassProvider ruleClassProvider); - @Override public void assertPackageNotLoaded(String packageName) throws Exception { MemoizingEvaluator evaluator = skyframeExecutor.getEvaluator(); - SkyKey key = PackageValue.key(PackageIdentifier.createInMainRepo(packageName)); + SkyKey key = PackageIdentifier.createInMainRepo(packageName); if (evaluator.getExistingValue(key) != null || evaluator.getExistingErrorForTesting(key) != null) { throw new IllegalStateException("Package was loaded: " + packageName); diff --git a/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java b/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java index 39e7aeafc023d7..7ce085431e5bb5 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java @@ -60,7 +60,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } private void writeRulesBzl(String type) throws Exception { diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java index 94615cb8fd98a4..6ca0e6fff59be6 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -239,6 +239,7 @@ public void setupDelegator() throws Exception { SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootPath, ImmutableMap.of())) .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, @@ -270,7 +271,7 @@ public void setupDelegator() throws Exception { differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java b/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java new file mode 100644 index 00000000000000..95f3023a417a6f --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java @@ -0,0 +1,795 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.starlarkdocextract; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.actions.BinaryFileWriteAction; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; +import com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil; +import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.PrecomputedValue.Injected; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.AspectInfo; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.AttributeInfo; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.AttributeType; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.FunctionParamInfo; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.ModuleInfo; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.OriginKey; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.ProviderInfo; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.ProviderNameGroup; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.RuleInfo; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.StarlarkFunctionInfo; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.TextFormat; +import java.io.IOException; +import java.util.NoSuchElementException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public final class StarlarkDocExtractTest extends BuildViewTestCase { + private Path moduleRoot; // initialized by extraPrecomputedValues + private FakeRegistry registry; + + @Override + protected ImmutableList extraPrecomputedValues() { + // TODO(b/285924565): support --enable_bzlmod in BuildViewTestCase tests without needing the + // boilerplate below. + try { + moduleRoot = scratch.dir("modules"); + } catch (IOException e) { + throw new IllegalStateException(e); + } + registry = FakeRegistry.DEFAULT_FACTORY.newFakeRegistry(moduleRoot.getPathString()); + return ImmutableList.of( + PrecomputedValue.injected( + ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), + PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), + PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()), + PrecomputedValue.injected(BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + PrecomputedValue.injected( + BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), + PrecomputedValue.injected( + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); + } + + private static ModuleInfo protoFromBinaryFileWriteAction(Action action) throws Exception { + assertThat(action).isInstanceOf(BinaryFileWriteAction.class); + return ModuleInfo.parseFrom( + ((BinaryFileWriteAction) action).getSource().openStream(), + ExtensionRegistry.getEmptyRegistry()); + } + + private static ModuleInfo protoFromTextFileWriteAction(Action action) throws Exception { + assertThat(action).isInstanceOf(FileWriteAction.class); + return TextFormat.parse( + ((FileWriteAction) action).getFileContents(), + ExtensionRegistry.getEmptyRegistry(), + ModuleInfo.class); + } + + private ModuleInfo protoFromConfiguredTarget(String targetName) throws Exception { + ConfiguredTarget target = getConfiguredTarget(targetName); + return protoFromBinaryFileWriteAction( + getGeneratingAction( + target, + Label.parseCanonicalUnchecked(targetName).toPathFragment().getPathString() + + ".binaryproto")); + } + + @Before + public void setUpBzlLibrary() throws Exception { + // TODO(https://github.com/bazelbuild/bazel/issues/18599): get rid of this when we bundle + // bzl_library with Bazel. + scratch.file( + "bzl_library.bzl", + "def _bzl_library_impl(ctx):", + " deps_files = [x.files for x in ctx.attr.deps]", + " all_files = depset(ctx.files.srcs, order = 'postorder', transitive = deps_files)", + " return DefaultInfo(files = all_files)", + "", + "bzl_library = rule(", + " implementation = _bzl_library_impl,", + " attrs = {", + " 'srcs': attr.label_list(allow_files = ['.bzl', '.scl']),", + " 'deps': attr.label_list(),", + " }", + ")"); + } + + @Test + public void basicFunctionality() throws Exception { + scratch.file( + "foo.bzl", // + "'''Module doc string'''", + "True"); + scratch.file( + "BUILD", // + "starlark_doc_extract(", + " name = 'extract',", + " src = 'foo.bzl',", + ")"); + ConfiguredTarget target = getConfiguredTarget("//:extract"); + ModuleInfo moduleInfo = + protoFromBinaryFileWriteAction(getGeneratingAction(target, "extract.binaryproto")); + assertThat(moduleInfo.getModuleDocstring()).isEqualTo("Module doc string"); + assertThat(moduleInfo.getFile()).isEqualTo("//:foo.bzl"); + } + + @Test + public void textprotoOut() throws Exception { + scratch.file( + "foo.bzl", // + "'''Module doc string'''", + "True"); + scratch.file( + "BUILD", // + "starlark_doc_extract(", + " name = 'extract',", + " src = 'foo.bzl',", + ")"); + ConfiguredTarget ruleTarget = getConfiguredTarget("//:extract"); + // Verify that we do not generate textproto output unless explicitly requested. + assertThrows( + NoSuchElementException.class, () -> getGeneratingAction(ruleTarget, "extract.textproto")); + + ConfiguredTarget textprotoOutputTarget = getConfiguredTarget("//:extract.textproto"); + ModuleInfo moduleInfo = + protoFromTextFileWriteAction( + getGeneratingAction(textprotoOutputTarget, "extract.textproto")); + assertThat(moduleInfo.getModuleDocstring()).isEqualTo("Module doc string"); + } + + @Test + public void sclDialect() throws Exception { + setBuildLanguageOptions("--experimental_enable_scl_dialect"); + scratch.file( + "foo.scl", // + "def f():", + " '''This is my function'''", + " pass"); + scratch.file( + "bar.scl", // + "'''My scl module string'''", + "load('//:foo.scl', 'f')", + "bar_f = f"); + scratch.file( + "BUILD", // + "load('bzl_library.bzl', 'bzl_library')", + "bzl_library(", + " name = 'foo_scl',", + " srcs = ['foo.scl'],", + ")", + "starlark_doc_extract(", + " name = 'extract',", + " src = 'bar.scl',", + " deps = ['foo_scl'],", + ")"); + ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract"); + assertThat(moduleInfo.getModuleDocstring()).isEqualTo("My scl module string"); + assertThat(moduleInfo.getFile()).isEqualTo("//:bar.scl"); + assertThat(moduleInfo.getFuncInfo(0).getDocString()).isEqualTo("This is my function"); + } + + @Test + public void sourceWithSyntaxError_fails() throws Exception { + scratch.file( + "error.bzl", // + "!!!"); + scratch.file( + "error_loader.bzl", // + "'''This is my module'''", + "load('error.bzl', 'x')"); + scratch.file( + "BUILD", // + "load('bzl_library.bzl', 'bzl_library')", + "bzl_library(", + " name = 'error_bzl',", + " srcs = ['error.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'error_doc',", + " src = 'error.bzl',", + ")", + "starlark_doc_extract(", + " name = 'error_loader_doc',", + " src = 'error_loader.bzl',", + " deps = ['error_bzl'],", + ")"); + + AssertionError errorDocFailure = + assertThrows(AssertionError.class, () -> getConfiguredTarget("//:error_doc")); + assertThat(errorDocFailure).hasMessageThat().contains("invalid character: '!'"); + + AssertionError errorLoaderDocFailure = + assertThrows(AssertionError.class, () -> getConfiguredTarget("//:error_loader_doc")); + assertThat(errorLoaderDocFailure).hasMessageThat().contains("invalid character: '!'"); + } + + @Test + public void symbolNames() throws Exception { + scratch.file( + "foo.bzl", // + "def func1():", + " pass", + "def func2():", + " pass", + "def _hidden():", + " pass"); + scratch.file( + "BUILD", // + "starlark_doc_extract(", + " name = 'extract_some',", + " src = 'foo.bzl',", + " symbol_names = ['func1'],", + ")", + "starlark_doc_extract(", + " name = 'extract_all',", + " src = 'foo.bzl',", + ")"); + + ModuleInfo dumpSome = protoFromConfiguredTarget("//:extract_some"); + assertThat(dumpSome.getFuncInfoList().stream().map(StarlarkFunctionInfo::getFunctionName)) + .containsExactly("func1"); + + ModuleInfo dumpAll = protoFromConfiguredTarget("//:extract_all"); + assertThat(dumpAll.getFuncInfoList().stream().map(StarlarkFunctionInfo::getFunctionName)) + .containsExactly("func1", "func2"); + } + + @Test + public void originKey() throws Exception { + scratch.file( + "origin.bzl", // + "def my_macro():", + " pass", + "MyInfo = provider()", + "MyOtherInfo = provider()", + "my_rule = rule(", + " implementation = lambda ctx: None,", + " attrs = {'a': attr.label(providers = [MyInfo, MyOtherInfo])},", + " provides = [MyInfo, MyOtherInfo],", + ")", + "my_aspect = aspect(implementation = lambda target, ctx: None)"); + scratch.file( + "renamer.bzl", // + "load(':origin.bzl', 'my_macro', 'MyInfo', 'MyOtherInfo', 'my_rule', 'my_aspect')", + "namespace = struct(", + " renamed_macro = my_macro,", + " RenamedInfo = MyInfo,", + " renamed_rule = my_rule,", + " renamed_aspect = my_aspect,", + ")", + "other_namespace = struct(", + " RenamedOtherInfo = MyOtherInfo,", + ")"); + scratch.file( + "BUILD", // + "load('bzl_library.bzl', 'bzl_library')", + "bzl_library(", + " name = 'origin_bzl',", + " srcs = ['origin.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'extract_renamed',", + " src = 'renamer.bzl',", + " deps = ['origin_bzl'],", + " symbol_names = ['namespace'],", + ")"); + + ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract_renamed"); + + assertThat(moduleInfo.getFuncInfoList().stream().map(StarlarkFunctionInfo::getFunctionName)) + .containsExactly("namespace.renamed_macro"); + assertThat(moduleInfo.getFuncInfoList().stream().map(StarlarkFunctionInfo::getOriginKey)) + .containsExactly( + OriginKey.newBuilder().setName("my_macro").setFile("//:origin.bzl").build()); + + assertThat(moduleInfo.getProviderInfoList().stream().map(ProviderInfo::getProviderName)) + .containsExactly("namespace.RenamedInfo"); + assertThat(moduleInfo.getProviderInfoList().stream().map(ProviderInfo::getOriginKey)) + .containsExactly(OriginKey.newBuilder().setName("MyInfo").setFile("//:origin.bzl").build()); + + assertThat(moduleInfo.getRuleInfoList().stream().map(RuleInfo::getRuleName)) + .containsExactly("namespace.renamed_rule"); + assertThat(moduleInfo.getRuleInfoList().stream().map(RuleInfo::getOriginKey)) + .containsExactly( + OriginKey.newBuilder().setName("my_rule").setFile("//:origin.bzl").build()); + + assertThat(moduleInfo.getRuleInfo(0).getAttributeList()) + .containsExactly( + ModuleInfoExtractor.IMPLICIT_NAME_ATTRIBUTE_INFO, + AttributeInfo.newBuilder() + .setName("a") + .setType(AttributeType.LABEL) + .setDefaultValue("None") + .addProviderNameGroup( + ProviderNameGroup.newBuilder() + .addProviderName("namespace.RenamedInfo") + .addProviderName("other_namespace.RenamedOtherInfo") + .addOriginKey( + OriginKey.newBuilder().setName("MyInfo").setFile("//:origin.bzl")) + .addOriginKey( + OriginKey.newBuilder().setName("MyOtherInfo").setFile("//:origin.bzl"))) + .build()); + assertThat(moduleInfo.getRuleInfo(0).getAdvertisedProviders()) + .isEqualTo( + ProviderNameGroup.newBuilder() + .addProviderName("namespace.RenamedInfo") + .addProviderName("other_namespace.RenamedOtherInfo") + .addOriginKey(OriginKey.newBuilder().setName("MyInfo").setFile("//:origin.bzl")) + .addOriginKey( + OriginKey.newBuilder().setName("MyOtherInfo").setFile("//:origin.bzl")) + .build()); + + assertThat(moduleInfo.getAspectInfoList().stream().map(AspectInfo::getAspectName)) + .containsExactly("namespace.renamed_aspect"); + assertThat(moduleInfo.getAspectInfoList().stream().map(AspectInfo::getOriginKey)) + .containsExactly( + OriginKey.newBuilder().setName("my_aspect").setFile("//:origin.bzl").build()); + } + + @Test + public void originKeyFileAndModuleInfoFileLabels_forBzlFileInBzlmodModule_areDisplayForm() + throws Exception { + setBuildLanguageOptions("--enable_bzlmod"); + scratch.overwriteFile("MODULE.bazel", "bazel_dep(name='origin_repo', version='0.1')"); + registry.addModule( + BzlmodTestUtil.createModuleKey("origin_repo", "0.1"), + "module(name='origin_repo', version='0.1')"); + Path originRepoPath = moduleRoot.getRelative("origin_repo~0.1"); + scratch.file(originRepoPath.getRelative("WORKSPACE").getPathString()); + scratch.file( + originRepoPath.getRelative("BUILD").getPathString(), // + "exports_files(['origin.bzl'])"); + scratch.file( + originRepoPath.getRelative("origin.bzl").getPathString(), // + "def my_macro():", + " pass", + "MyInfo = provider()", + "my_rule = rule(", + " implementation = lambda ctx: None,", + " attrs = {'a': attr.label(providers = [MyInfo])},", + " provides = [MyInfo],", + ")", + "my_aspect = aspect(implementation = lambda target, ctx: None)"); + scratch.file( + "renamer.bzl", // + "load('@origin_repo//:origin.bzl', 'my_macro', 'MyInfo', 'my_rule', 'my_aspect')", + "namespace = struct(", + " renamed_macro = my_macro,", + " RenamedInfo = MyInfo,", + " renamed_rule = my_rule,", + " renamed_aspect = my_aspect,", + ")"); + scratch.file( + "BUILD", // + "load('bzl_library.bzl', 'bzl_library')", + "bzl_library(", + " name = 'origin_bzl',", + " srcs = ['@origin_repo//:origin.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'extract_origin',", + " src = '@origin_repo//:origin.bzl',", + ")", + "starlark_doc_extract(", + " name = 'extract_renamed',", + " src = 'renamer.bzl',", + " deps = ['origin_bzl'],", + ")"); + + // verify that ModuleInfo.name for a .bzl file in another bzlmod module is in display form, i.e. + // "@origin_repo//:origin.bzl" as opposed to "@@origin_repo~0.1//:origin.bzl" + ModuleInfo originModuleInfo = protoFromConfiguredTarget("//:extract_origin"); + assertThat(originModuleInfo.getFile()).isEqualTo("@origin_repo//:origin.bzl"); + + // verify that OriginKey.name for entities defined in a .bzl file in another bzlmod module is in + // display form, i.e. "@origin_repo//:origin.bzl" as opposed to "@@origin_repo~0.1//:origin.bzl" + ModuleInfo renamedModuleInfo = protoFromConfiguredTarget("//:extract_renamed"); + assertThat(renamedModuleInfo.getFile()).isEqualTo("//:renamer.bzl"); + assertThat(renamedModuleInfo.getFuncInfo(0).getOriginKey().getFile()) + .isEqualTo("@origin_repo//:origin.bzl"); + assertThat(renamedModuleInfo.getProviderInfo(0).getOriginKey().getFile()) + .isEqualTo("@origin_repo//:origin.bzl"); + assertThat(renamedModuleInfo.getAspectInfo(0).getOriginKey().getFile()) + .isEqualTo("@origin_repo//:origin.bzl"); + assertThat(renamedModuleInfo.getRuleInfo(0).getOriginKey().getFile()) + .isEqualTo("@origin_repo//:origin.bzl"); + assertThat( + renamedModuleInfo + .getRuleInfo(0) + .getAttribute(1) // 0 is the implicit name attribute + .getProviderNameGroup(0) + .getOriginKey(0) + .getFile()) + .isEqualTo("@origin_repo//:origin.bzl"); + assertThat(renamedModuleInfo.getRuleInfo(0).getAdvertisedProviders().getOriginKey(0).getFile()) + .isEqualTo("@origin_repo//:origin.bzl"); + } + + @Test + public void exportNestedFunctionsAndLambdas() throws Exception { + scratch.file( + "origin.bzl", // + "def return_nested():", + " def nested(x):", + " '''My nested function'''", + " pass", + " return nested", + "", + "def return_lambda():", + " return lambda y: y"); + scratch.file( + "exporter.bzl", // + "load(':origin.bzl', 'return_nested', 'return_lambda')", + "exported_nested = return_nested()", + "exported_lambda = return_lambda()"); + scratch.file( + "BUILD", // + "load('bzl_library.bzl', 'bzl_library')", + "bzl_library(", + " name = 'origin_bzl',", + " srcs = ['origin.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'extract_exporter',", + " src = 'exporter.bzl',", + " deps = ['origin_bzl'],", + ")"); + + ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract_exporter"); + + assertThat(moduleInfo.getFuncInfoList()) + .containsExactly( + StarlarkFunctionInfo.newBuilder() + .setFunctionName("exported_nested") + .setDocString("My nested function") + .addParameter(FunctionParamInfo.newBuilder().setName("x").setMandatory(true)) + .setOriginKey( + // OriginKey.name for nested functions is explicitly unset + OriginKey.newBuilder().setFile("//:origin.bzl")) + .build(), + StarlarkFunctionInfo.newBuilder() + .setFunctionName("exported_lambda") + .addParameter(FunctionParamInfo.newBuilder().setName("y").setMandatory(true)) + .setOriginKey( + // OriginKey.name for lambdas is explicitly unset + OriginKey.newBuilder().setFile("//:origin.bzl")) + .build()); + } + + @Test + public void missingBzlLibraryDeps_fails() throws Exception { + scratch.file( + "dep.bzl", // + "load('//:forgotten_dep_of_dep.bzl', 'g')", + "def f(): pass"); + scratch.file( + "forgotten_dep_of_dep.bzl", // + "def g(): pass"); + scratch.file( + "forgotten_dep.bzl", // + "load('//:forgotten_dep_of_forgotten_dep.bzl', 'j')", + "def h(): pass"); + scratch.file( + "forgotten_dep2.bzl", // + "def i(): pass"); + scratch.file( + "forgotten_dep_of_forgotten_dep.bzl", // + "def j(): pass"); + scratch.file( + "foo.bzl", // + "load('//:dep.bzl', 'f')", + "load('//:forgotten_dep.bzl', 'h')", + "load('//:forgotten_dep2.bzl', 'i')"); + scratch.file( + "BUILD", // + "load('bzl_library.bzl', 'bzl_library')", + "bzl_library(", + " name = 'dep_bzl',", + " srcs = ['dep.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'extract',", + " src = 'foo.bzl',", // Note that src does not need to be part of deps + " deps = ['dep_bzl']", + ")"); + + AssertionError e = assertThrows(AssertionError.class, () -> getConfiguredTarget("//:extract")); + assertThat(e) + .hasMessageThat() + .contains( + "missing bzl_library targets for Starlark module(s) //:forgotten_dep_of_dep.bzl," + + " //:forgotten_dep.bzl, //:forgotten_dep2.bzl"); + // We do not want to log transitive deps of already missing deps in the error message - it would + // be hard to read and unnecessary, since a valid bzl_library target should bring in its + // transitive deps. + assertThat(e).hasMessageThat().doesNotContain("forgotten_dep_of_forgotten_dep.bzl"); + } + + @Test + public void depsWithDerivedFiles_onUnknownLoads_failsAndPrintsDerivedFiles() throws Exception { + scratch.file("BUILD"); + scratch.file( + "pkg/source_file_masked_by_rule_name.bzl", // + "def f(): pass"); + scratch.file( + "pkg/source_file_masked_by_rule_output_name.bzl", // + "def g(): pass"); + scratch.file( + "pkg/foo.bzl", // + "load('//pkg:source_file_masked_by_rule_name.bzl', 'f')", + "load('//pkg:source_file_masked_by_rule_output_name.bzl', 'g')"); + scratch.file( + "pkg/BUILD", // + "load('//:bzl_library.bzl', 'bzl_library')", + "genrule(", + " name = 'source_file_masked_by_rule_name.bzl',", + " outs = ['some_output.bzl'],", + " cmd = 'touch $@'", + ")", + "genrule(", + " name = 'source_file_masked_by_rule_output_name_bzl_generator',", + " outs = ['source_file_masked_by_rule_output_name.bzl'],", + " cmd = 'touch $@'", + ")", + "genrule(", + " name = 'some_rule',", + " outs = ['ordinary_generated_file.bzl'],", + " cmd = 'touch $@'", + ")", + "bzl_library(", + " name = 'deps_bzl',", + " srcs = [", + " 'source_file_masked_by_rule_name.bzl',", + " 'source_file_masked_by_rule_output_name.bzl',", + " 'ordinary_generated_file.bzl',", + " ],", + ")", + "starlark_doc_extract(", + " name = 'extract',", + " src = 'foo.bzl',", + " deps = ['deps_bzl']", + ")"); + + AssertionError error = + assertThrows(AssertionError.class, () -> getConfiguredTarget("//pkg:extract")); + assertThat(error) + .hasMessageThat() + .contains( + "missing bzl_library targets for Starlark module(s)" + + " //pkg:source_file_masked_by_rule_name.bzl," + + " //pkg:source_file_masked_by_rule_output_name.bzl\n" + + "Note the following are generated file(s) and cannot be loaded in Starlark:" + + " pkg/some_output.bzl (generated by rule" + + " //pkg:source_file_masked_by_rule_name.bzl)," + + " pkg/source_file_masked_by_rule_output_name.bzl (generated by rule" + + " //pkg:source_file_masked_by_rule_output_name_bzl_generator)," + + " pkg/ordinary_generated_file.bzl (generated by rule //pkg:some_rule)"); + } + + @Test + public void depsWithDerivedFiles_onNoUnknownLoads_succeeds() throws Exception { + scratch.file("BUILD"); + scratch.file( + "util.bzl", + "def _impl(ctx):", + " out = ctx.actions.declare_file(ctx.attr.out)", + " ctx.actions.run_shell(command = 'touch $1', arguments = [out.path], outputs = [out])", + " return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out]))", + "generate_out_without_declaring_it_as_a_target = rule(", + " attrs = {'out': attr.string()},", + " implementation = _impl,", + ")"); + scratch.file( + "pkg/source_dep.bzl", // + "def f(): pass"); + scratch.file( + "pkg/foo.bzl", // + "load('//pkg:source_dep.bzl', 'f')"); + scratch.file( + "pkg/BUILD", // + "load('//:bzl_library.bzl', 'bzl_library')", + "load('//:util.bzl', 'generate_out_without_declaring_it_as_a_target')", + "genrule(", + " name = 'some_rule',", + " outs = ['declared_derived_dep.bzl'],", + " cmd = 'touch $@'", + ")", + // //pkg:generate_source_dep_without_declaring_it_as_a_target masks the source_dep.bzl + // source artifact with a non-target, derived artifact having the same root-relative path. + "generate_out_without_declaring_it_as_a_target(", + " name = 'generate_source_dep_without_declaring_it_as_a_target',", + " out = 'source_dep.bzl'", + ")", + "bzl_library(", + " name = 'deps_bzl',", + " srcs = [", + " 'declared_derived_dep.bzl',", + " 'source_dep.bzl',", + " 'generate_source_dep_without_declaring_it_as_a_target',", + " ],", + ")", + "starlark_doc_extract(", + " name = 'extract',", + " src = 'foo.bzl',", + " deps = ['deps_bzl']", + ")"); + + getConfiguredTarget("//pkg:extract"); + assertNoEvents(); + } + + @Test + public void srcDerivedFile_fails() throws Exception { + scratch.file("BUILD"); + scratch.file("pkg/source_file_masked_by_rule_name.bzl"); + scratch.file("pkg/source_file_masked_by_rule_output_name.bzl"); + scratch.file( + "pkg/BUILD", // + "genrule(", + " name = 'source_file_masked_by_rule_name.bzl',", + " outs = ['some_output.bzl'],", + " cmd = 'touch $@'", + ")", + "genrule(", + " name = 'source_file_masked_by_rule_output_name_bzl_generator',", + " outs = ['source_file_masked_by_rule_output_name.bzl'],", + " cmd = 'touch $@'", + ")", + "genrule(", + " name = 'some_rule',", + " outs = ['ordinary_generated_file.bzl'],", + " cmd = 'touch $@'", + ")", + "starlark_doc_extract(", + " name = 'source_file_masked_by_rule_name_doc',", + " src = 'source_file_masked_by_rule_name.bzl',", + ")", + "starlark_doc_extract(", + " name = 'source_file_masked_by_rule_output_name_doc',", + " src = 'source_file_masked_by_rule_output_name.bzl',", + ")", + "starlark_doc_extract(", + " name = 'ordinary_generated_file_doc',", + " src = 'ordinary_generated_file.bzl',", + ")"); + + AssertionError maskedByRuleError = + assertThrows( + AssertionError.class, + () -> getConfiguredTarget("//pkg:source_file_masked_by_rule_name_doc")); + assertThat(maskedByRuleError) + .hasMessageThat() + .contains( + "pkg/some_output.bzl (generated by rule //pkg:source_file_masked_by_rule_name.bzl)" + + " is not a source file and cannot be loaded in Starlark"); + + AssertionError maskedByRuleOutputError = + assertThrows( + AssertionError.class, + () -> getConfiguredTarget("//pkg:source_file_masked_by_rule_output_name_doc")); + assertThat(maskedByRuleOutputError) + .hasMessageThat() + .contains( + "pkg/source_file_masked_by_rule_output_name.bzl (generated by rule" + + " //pkg:source_file_masked_by_rule_output_name_bzl_generator) is not a source" + + " file and cannot be loaded in Starlark"); + + AssertionError ordinaryGeneratedFileError = + assertThrows( + AssertionError.class, () -> getConfiguredTarget("//pkg:ordinary_generated_file_doc")); + assertThat(ordinaryGeneratedFileError) + .hasMessageThat() + .contains( + "pkg/ordinary_generated_file.bzl (generated by rule //pkg:some_rule) is not a source" + + " file and cannot be loaded in Starlark"); + } + + @Test + public void srcAlias_resolvesToActual() throws Exception { + scratch.file("alias_name.bzl"); + scratch.file("alias_actual.bzl"); + scratch.file( + "BUILD", // + "alias(", + " name = 'alias_name.bzl',", + " actual = 'alias_actual.bzl',", + ")", + "starlark_doc_extract(", + " name = 'extract',", + " src = 'alias_name.bzl',", + ")"); + + ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract"); + assertThat(moduleInfo.getFile()).isEqualTo("//:alias_actual.bzl"); + } + + @Test + public void srcFilegroup_resolvesToFilegroupSrc() throws Exception { + scratch.file("masked_by_filegroup_name.bzl"); + scratch.file("filegroup_src_actual.bzl"); + scratch.file( + "BUILD", // + "filegroup(", + " name = 'masked_by_filegroup_name.bzl',", + " srcs = ['filegroup_src_actual.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'extract',", + " src = 'masked_by_filegroup_name.bzl',", + ")"); + + ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract"); + assertThat(moduleInfo.getFile()).isEqualTo("//:filegroup_src_actual.bzl"); + } + + @Test + public void srcFilegroup_mustHaveSingleSrc() throws Exception { + scratch.file("foo.bzl"); + scratch.file("bar.bzl"); + scratch.file( + "BUILD", // + "filegroup(", + " name = 'no_files',", + " srcs = [],", + ")", + "filegroup(", + " name = 'two_files',", + " srcs = ['foo.bzl', 'bar.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'no_files_doc',", + " src = 'no_files',", + ")", + "starlark_doc_extract(", + " name = 'two_files_doc',", + " src = 'two_files',", + ")"); + + AssertionError extractNoFilesError = + assertThrows(AssertionError.class, () -> getConfiguredTarget("//:no_files_doc")); + assertThat(extractNoFilesError) + .hasMessageThat() + .contains("'//:no_files' must produce a single file"); + + AssertionError extractTwoFilesError = + assertThrows(AssertionError.class, () -> getConfiguredTarget("//:two_files_doc")); + assertThat(extractTwoFilesError) + .hasMessageThat() + .contains("'//:two_files' must produce a single file"); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java index 61230364befbd8..2b98361118813b 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java @@ -113,7 +113,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Before diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java index 2d00ff7c3ed816..aaf6bd05afee16 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java @@ -247,7 +247,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } // Helpers: diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java index cda1434744fe9c..5097e5e24982c2 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java @@ -113,7 +113,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java index 2374e2c62126cb..b7b54d9e88bd5c 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java @@ -69,7 +69,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java index 06672b18ddc308..b7dcd438e337e1 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java @@ -88,7 +88,7 @@ protected ImmutableList extraPrecomputedValues() thro BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Override From afedb3f74b6abe4607e86aa495af2a1b1c3f4a1a Mon Sep 17 00:00:00 2001 From: salma-samy Date: Thu, 6 Jul 2023 13:59:47 -0700 Subject: [PATCH 07/20] Add lockfile documentation PiperOrigin-RevId: 546085523 Change-Id: If287e6e143f8858d185f83a1630275520db65e44 # Conflicts: # site/en/_book.yaml --- site/en/_book.yaml | 15 +++ site/en/external/lockfile.md | 191 +++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 site/en/external/lockfile.md diff --git a/site/en/_book.yaml b/site/en/_book.yaml index 1b3e1d53268e02..19bdc921b9d8aa 100644 --- a/site/en/_book.yaml +++ b/site/en/_book.yaml @@ -125,6 +125,21 @@ upper_tabs: path: /run/scripts - title: Client/server implementation path: /run/client-server + - heading: External dependencies + - title: Overview + path: /external/overview + - title: Bazel modules + path: /external/module + - title: Bazel registries + path: /external/registry + - title: Module extensions + path: /external/extension + - title: Module lockfile + path: /external/lockfile + - title: Bzlmod migration guide + path: /external/migration + - title: Advanced topics + path: /external/advanced - heading: Querying your build - title: Query quickstart path: /query/quickstart diff --git a/site/en/external/lockfile.md b/site/en/external/lockfile.md new file mode 100644 index 00000000000000..39ed0e8e680a2a --- /dev/null +++ b/site/en/external/lockfile.md @@ -0,0 +1,191 @@ +Project: /_project.yaml +Book: /_book.yaml +keywords: product:Bazel,lockfile,Bzlmod + +# Bazel Lockfile + +{% include "_buttons.html" %} + +The lockfile feature in Bazel enables the recording of specific versions or +dependencies of software libraries or packages required by a project. It +achieves this by storing the result of module resolution and extension +evaluation. The lockfile promotes reproducible builds, ensuring consistent +development environments. Additionally, it enhances build efficiency by allowing +Bazel to skip the resolution process when there are no changes in project +dependencies. Furthermore, the lockfile improves stability by preventing +unexpected updates or breaking changes in external libraries, thereby reducing +the risk of introducing bugs. + +## Lockfile Generation {:#lockfile-generation} + +The lockfile is generated under the workspace root with the name +`MODULE.bazel.lock`. It is created or updated during the build process, +specifically after module resolution and extension evaluation. The lockfile +captures the current state of the project, including the MODULE file, flags, +overrides, and other relevant information. Importantly, it only includes +dependencies that are included in the current invocation of the build. + +When changes occur in the project that affect its dependencies, the lockfile is +automatically updated to reflect the new state. This ensures that the lockfile +remains focused on the specific set of dependencies required for the current +build, providing an accurate representation of the project's resolved +dependencies. + +## Lockfile Usage {:#lockfile-usage} + +The lockfile can be controlled by the flag +[`--lockfile_mode`](/reference/command-line-reference#flag--lockfile_mode) to +customize the behavior of Bazel when the project state differs from the +lockfile. The available modes are: + +* `update` (Default): If the project state matches the lockfile, the + resolution result is immediately returned from the lockfile. Otherwise, + resolution is executed, and the lockfile is updated to reflect the current + state. +* `error`: If the project state matches the lockfile, the resolution result is + returned from the lockfile. Otherwise, Bazel throws an error indicating the + variations between the project and the lockfile. This mode is particularly + useful when you want to ensure that your project's dependencies remain + unchanged, and any differences are treated as errors. +* `off`: The lockfile is not checked at all. + +## Lockfile Benefits {:#lockfile-benefits} + +The lockfile offers several benefits and can be utilized in various ways: + +- **Reproducible builds.** By capturing the specific versions or dependencies + of software libraries, the lockfile ensures that builds are reproducible + across different environments and over time. Developers can rely on + consistent and predictable results when building their projects. + +- **Efficient resolution skipping.** The lockfile enables Bazel to skip the + resolution process if there are no changes in the project dependencies since + the last build. This significantly improves build efficiency, especially in + scenarios where resolution can be time-consuming. + +- **Stability and risk reduction.** The lockfile helps maintain stability by + preventing unexpected updates or breaking changes in external libraries. By + locking the dependencies to specific versions, the risk of introducing bugs + due to incompatible or untested updates is reduced. + +## Lockfile Contents {:#lockfile-contents} + +The lockfile contains all the necessary information to determine whether the +project state has changed. It also includes the result of building the project +in the current state. The lockfile consists of two main parts: + +1. Inputs of the module resolution, such as `moduleFileHash`, `flags` and + `localOverrideHashes`, as well as the output of the resolution, which is + `moduleDepGraph`. +2. For each module extension, the lockfile includes inputs that affect it, + represented by `transitiveDigest`, and the output of running that extension + referred to as `generatedRepoSpecs` + +Here is an example that demonstrates the structure of the lockfile, along with +explanations for each section: + +```json +{ + "lockFileVersion": 1, + "moduleFileHash": "b0f47b98a67ee15f9.......8dff8721c66b721e370", + "flags": { + "cmdRegistries": [ + "https://bcr.bazel.build/" + ], + "cmdModuleOverrides": {}, + "allowedYankedVersions": [], + "envVarAllowedYankedVersions": "", + "ignoreDevDependency": false, + "directDependenciesMode": "WARNING", + "compatibilityMode": "ERROR" + }, + "localOverrideHashes": { + "bazel_tools": "b5ae1fa37632140aff8.......15c6fe84a1231d6af9" + }, + "moduleDepGraph": { + "": { + "name": "", + "version": "", + "executionPlatformsToRegister": [], + "toolchainsToRegister": [], + "extensionUsages": [ + { + "extensionBzlFile": "extension.bzl", + "extensionName": "lockfile_ext" + } + ], + ... + } + }, + "moduleExtensions": { + "//:extension.bzl%lockfile_ext": { + "transitiveDigest": "oWDzxG/aLnyY6Ubrfy....+Jp6maQvEPxn0pBM=", + "generatedRepoSpecs": { + "hello": { + "bzlFile": "@@//:extension.bzl", + ... + } + } + } + } +} +``` + +### Module File Hash {:#module-file-hash} + +The `moduleFileHash` represents the hash of the `MODULE.bazel` file contents. If +any changes occur in this file, the hash value differs. + +### Flags {:#flags} + +The `Flags` object stores all the flags that can affect the resolution result. + +### Local Override Hashes {:#local-override-hashes} + +If the root module includes `local_path_overrides`, this section stores the hash +of the `MODULE.bazel` file in the local repository. It allows tracking changes +to this dependency. + +### Module Dependency Graph {:#module-dep-graph} + +The `moduleDepGraph` represents the result of the resolution process using the +inputs mentioned above. It forms the dependency graph of all the modules +required to run the project. + +### Module Extensions {:#module-extensions} + +The `moduleExtensions` section is a map that includes only the extensions used +in the current invocation or previously invoked, while excluding any extensions +that are no longer utilized. In other words, if an extension is not being used +anymore across the dependency graph, it is removed from the `moduleExtensions` +map. + +Each entry in this map corresponds to a used extension and is identified by its +containing file and name. The corresponding value for each entry contains the +relevant information associated with that extension: + +1. The `transitiveDigest` the digest of the extension implementation and its + transitive .bzl files. +2. The `generatedRepoSpecs` the result of running that extension with the + current input. + +An additional factor that can affect the extension results is their _usages_. +Although not stored in the lockfile, the usages are considered when comparing +the current state of the extension with the one in the lockfile. + +## Best Practices {:#best-practices} + +To maximize the benefits of the lockfile feature, consider the following best +practices: + +* Regularly update the lockfile to reflect changes in project dependencies or + configuration. This ensures that subsequent builds are based on the most + up-to-date and accurate set of dependencies. + +* Include the lockfile in version control to facilitate collaboration and + ensure that all team members have access to the same lockfile, promoting + consistent development environments across the project. + +By following these best practices, you can effectively utilize the lockfile +feature in Bazel, leading to more efficient, reliable, and collaborative +software development workflows. \ No newline at end of file From 8045c0152850054f3fd93a00837b8e7edf45c805 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Mon, 10 Jul 2023 03:40:33 -0700 Subject: [PATCH 08/20] Fix null event issue If a change "only" occurred in a module extension, then the value of ModuleResolutionEvent is null -the skyvalue is cached and the event was never sent- then "combineModuleExtensions" function would crash with null pointer exception while trying to get the old usages. In this case we can just re-add the old module extensions from the lockfile because if the module resolution is the same, this means the usage didn't change. PiperOrigin-RevId: 546821911 Change-Id: Ie685cbab654d1c41403aebd31ddad91033be1d56 --- .../lib/bazel/bzlmod/BazelLockFileModule.java | 17 ++++++++++++----- src/test/py/bazel/bzlmod/bazel_lockfile_test.py | 17 ++++++++++++++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java index 521dd09a1a6dde..5bd3b40a06bd16 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java @@ -105,11 +105,18 @@ private ImmutableMap combineModuleEx ImmutableMap.Builder updatedExtensionMap = ImmutableMap.builder(); - // Add the old extensions (stored in the lockfile) only if it still has a usage somewhere - for (Map.Entry extensionEntry : - oldModuleExtensions.entrySet()) { - if (moduleResolutionEvent.getExtensionUsagesById().containsRow(extensionEntry.getKey())) { - updatedExtensionMap.put(extensionEntry); + // This event being null means that no changes occurred to the usages of the stored extensions, + // hence no changes to any module resulted in re-running resolution. So we can just add all the + // old stored extensions. Otherwise, check the usage of each one. + if (moduleResolutionEvent == null) { + updatedExtensionMap.putAll(oldModuleExtensions); + } else { + // Add the old extensions (stored in the lockfile) only if it still has a usage somewhere + for (Map.Entry extensionEntry : + oldModuleExtensions.entrySet()) { + if (moduleResolutionEvent.getExtensionUsagesById().containsRow(extensionEntry.getKey())) { + updatedExtensionMap.put(extensionEntry); + } } } diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py index c91173e323914d..03381c12e895e5 100644 --- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -367,7 +367,16 @@ def testUpdateModuleExtension(self): ) _, _, stderr = self.RunBazel(['build', '@hello//:all']) self.assertIn('Hello from the other side!', ''.join(stderr)) - self.RunBazel(['shutdown']) + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + old_impl = lockfile['moduleExtensions']['//:extension.bzl%lockfile_ext'][ + 'bzlTransitiveDigest' + ] + + # Run again to make sure the resolution value is cached. So even if module + # resolution doesn't rerun (its event is null), the lockfile is still + # updated with the newest extension eval results + self.RunBazel(['build', '@hello//:all']) # Update extension. Make sure that it is executed and updated in the # lockfile without errors (since it's already in the lockfile) @@ -389,6 +398,12 @@ def testUpdateModuleExtension(self): ) _, _, stderr = self.RunBazel(['build', '@hello//:all']) self.assertIn('Hello from the other town!', ''.join(stderr)) + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + new_impl = lockfile['moduleExtensions']['//:extension.bzl%lockfile_ext'][ + 'bzlTransitiveDigest' + ] + self.assertNotEqual(new_impl, old_impl) def testUpdateModuleExtensionErrorMode(self): self.ScratchFile( From 0dfe17dbe241652b4620f01a8cfdbf6aa05f952b Mon Sep 17 00:00:00 2001 From: salma-samy Date: Tue, 11 Jul 2023 11:57:35 +0200 Subject: [PATCH 09/20] Fix tests --- .../build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index 7b00d9ea308fb3..5dda67fd839ba2 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -290,7 +290,7 @@ public void createValue_moduleExtensions() throws Exception { maven, "rules_jvm_external~1.0~maven", pip, "rules_python~2.0~pip", myext, "dep~2.0~myext", - myext2, "dep~2.0~myext~2"); + myext2, "dep~2.0~myext2"); assertThat(value.getFullRepoMapping(ModuleKey.ROOT)) .isEqualTo( @@ -321,7 +321,7 @@ public void createValue_moduleExtensions() throws Exception { "oneext", "dep~2.0~myext~myext", "twoext", - "dep~2.0~myext~2~myext")); + "dep~2.0~myext2~myext")); } @Test From 3ad72544c4150b363b73f46be6b4da975325ae67 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Tue, 6 Jun 2023 04:45:14 -0700 Subject: [PATCH 10/20] Update lockfile function exception Add new exception for this function to handle any syntax errors or missing data within the lockfile Related: https://github.com/bazelbuild/bazel/issues/18455 PiperOrigin-RevId: 538144339 Change-Id: I82160f3bff6598c26b3f99c824fe85ea86086c1f --- .../bazel/bzlmod/BazelLockFileFunction.java | 26 +++- .../bzlmod/BazelLockFileFunctionTest.java | 132 ++++++++++++++++++ 2 files changed, 151 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index cfeb31fa0f57bb..ba6c5b6c192cdf 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.FileValue; -import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction.BazelDepGraphFunctionException; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; @@ -35,7 +34,7 @@ import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; -import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; import java.io.FileNotFoundException; import java.io.IOException; import javax.annotation.Nullable; @@ -81,8 +80,14 @@ public SkyValue compute(SkyKey skyKey, Environment env) bazelLockFileValue = LOCKFILE_GSON.fromJson(json, BazelLockFileValue.class); } catch (FileNotFoundException e) { bazelLockFileValue = EMPTY_LOCKFILE; - } catch (IOException ex) { - throw new JsonIOException("Failed to read or parse module-lock file", ex); + } catch (IOException | JsonSyntaxException | NullPointerException e) { + throw new BazelLockfileFunctionException( + ExternalDepsException.withMessage( + Code.BAD_MODULE, + "Failed to read and parse the MODULE.bazel.lock file with error: %s." + + " Try deleting it and rerun the build.", + e.getMessage()), + Transience.PERSISTENT); } return bazelLockFileValue; } @@ -99,7 +104,7 @@ public static void updateLockedModule( BzlmodFlagsAndEnvVars flags, ImmutableMap localOverrideHashes, ImmutableMap resolvedDepGraph) - throws BazelDepGraphFunctionException { + throws BazelLockfileFunctionException { RootedPath lockfilePath = RootedPath.toRootedPath(Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); @@ -113,10 +118,17 @@ public static void updateLockedModule( try { FileSystemUtils.writeContent(lockfilePath.asPath(), UTF_8, LOCKFILE_GSON.toJson(value)); } catch (IOException e) { - throw new BazelDepGraphFunctionException( + throw new BazelLockfileFunctionException( ExternalDepsException.withCauseAndMessage( - Code.BAD_MODULE, e, "Unable to update module-lock file"), + Code.BAD_MODULE, e, "Unable to update the MODULE.bazel.lock file"), Transience.PERSISTENT); } } + + static final class BazelLockfileFunctionException extends SkyFunctionException { + + BazelLockfileFunctionException(Exception cause, Transience transience) { + super(cause, transience); + } + } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index e10d52d15924fb..f1c775b201fbd7 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -67,6 +67,8 @@ import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; @@ -387,6 +389,136 @@ public void fullModule() throws Exception { assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); } + @Test + public void invalidLockfileEmptyFile() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.of( + ModuleKey.ROOT, + BazelModuleResolutionFunction.moduleFromInterimModule( + rootValue.getModule(), null, null)); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + scratch.overwriteFile(rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), "{}"); + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (!result.hasError()) { + fail("expected error about missing field in the lockfile, but succeeded"); + } + assertThat(result.getError().toString()) + .contains( + "Failed to read and parse the MODULE.bazel.lock file with error: Null moduleFileHash." + + " Try deleting it and rerun the build."); + } + + @Test + public void invalidLockfileNullFlag() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.of( + ModuleKey.ROOT, + BazelModuleResolutionFunction.moduleFromInterimModule( + rootValue.getModule(), null, null)); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + JsonObject jsonObject = + (JsonObject) JsonParser.parseString(scratch.readFile("MODULE.bazel.lock")); + jsonObject.get("flags").getAsJsonObject().remove("directDependenciesMode"); + scratch.overwriteFile( + rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), jsonObject.toString()); + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (!result.hasError()) { + fail("expected error about missing field in the lockfile, but succeeded"); + } + assertThat(result.getError().toString()) + .contains( + "Failed to read and parse the MODULE.bazel.lock file with error: Null" + + " directDependenciesMode. Try deleting it and rerun the build."); + } + + @Test + public void invalidLockfileMalformed() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.of( + ModuleKey.ROOT, + BazelModuleResolutionFunction.moduleFromInterimModule( + rootValue.getModule(), null, null)); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + JsonObject jsonObject = + (JsonObject) JsonParser.parseString(scratch.readFile("MODULE.bazel.lock")); + jsonObject.get("flags").getAsJsonObject().addProperty("allowedYankedVersions", "string!"); + scratch.overwriteFile( + rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), jsonObject.toString()); + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (!result.hasError()) { + fail("expected error about invalid field value in the lockfile, but succeeded"); + } + assertThat(result.getError().toString()) + .contains( + "Failed to read and parse the MODULE.bazel.lock file with error:" + + " java.lang.IllegalStateException: Expected BEGIN_ARRAY but was STRING at line 1" + + " column 129 path $.flags.allowedYankedVersions. Try deleting it and rerun the" + + " build."); + } + @AutoValue abstract static class UpdateLockFileKey implements SkyKey { From f31f6eadd399b0aaab5ff503f6fb23485ea75ef5 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Thu, 15 Jun 2023 05:30:52 -0700 Subject: [PATCH 11/20] Update lockfile writing logic to be event triggered. The writing will only happen at the end of resolution in the after command function (at this point, the module and all needed module extensions are resolved). Which solves the problem of reading and writing into the lockfile multiple times in one invocation. PiperOrigin-RevId: 540552139 Change-Id: I4a78412a388bde2ff7949d119831318c40d49047 # Conflicts: # src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java # src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java # src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java --- .../com/google/devtools/build/lib/bazel/BUILD | 1 + .../devtools/build/lib/bazel/Bazel.java | 1 + .../lib/bazel/BazelRepositoryModule.java | 3 +- .../devtools/build/lib/bazel/bzlmod/BUILD | 16 ++++ .../bazel/bzlmod/BazelDepGraphFunction.java | 49 ++++------- .../bazel/bzlmod/BazelLockFileFunction.java | 50 ++--------- .../lib/bazel/bzlmod/BazelLockFileModule.java | 86 +++++++++++++++++++ .../lib/bazel/bzlmod/BazelLockFileValue.java | 31 +++++-- .../repository/RepositoryResolvedModule.java | 1 - .../build/lib/analysis/util/AnalysisMock.java | 2 +- .../devtools/build/lib/bazel/bzlmod/BUILD | 1 + .../bzlmod/BazelDepGraphFunctionTest.java | 2 +- .../bzlmod/BazelLockFileFunctionTest.java | 46 ++++++---- .../BazelModuleResolutionFunctionTest.java | 2 +- .../bzlmod/BzlmodRepoRuleFunctionTest.java | 46 +--------- .../bzlmod/ModuleExtensionResolutionTest.java | 2 +- .../repository/RepositoryDelegatorTest.java | 2 +- 17 files changed, 187 insertions(+), 154 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD index 3c29ea712d2909..bd7402b7f9d77a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD @@ -142,6 +142,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info", "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:bazel_lockfile_module", "//src/main/java/com/google/devtools/build/lib/bazel/coverage", "//src/main/java/com/google/devtools/build/lib/bazel/debug:workspace-rule-module", "//src/main/java/com/google/devtools/build/lib/bazel/repository", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java index e35b11a8386f5d..7fb055d87e2c30 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java @@ -63,6 +63,7 @@ public final class Bazel { com.google.devtools.build.lib.bazel.repository.RepositoryResolvedModule.class, com.google.devtools.build.lib.bazel.repository.CacheHitReportingModule.class, com.google.devtools.build.lib.bazel.SpawnLogModule.class, + com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileModule.class, com.google.devtools.build.lib.outputfilter.OutputFilteringModule.class, com.google.devtools.build.lib.worker.WorkerModule.class, com.google.devtools.build.lib.runtime.CacheFileDigestsModule.class, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 5e87645a450f50..1e19a99de35e2d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -265,8 +265,7 @@ public ResolutionReason getResolutionReason() { .addSkyFunction( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, directories.getWorkspace(), builtinModules)) - .addSkyFunction( - SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(directories.getWorkspace())) + .addSkyFunction(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .addSkyFunction( SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace())) .addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction()) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 4b45705309b979..02967174997589 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -85,6 +85,22 @@ java_library( ], ) +java_library( + name = "bazel_lockfile_module", + srcs = ["BazelLockFileModule.java"], + deps = [ + ":resolution", + ":resolution_impl", + "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//third_party:flogger", + "//third_party:guava", + "//third_party:jsr305", + ], +) + java_library( name = "resolution", srcs = [ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index f99b33b627a371..f55e3fbc8b6b13 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -35,7 +35,6 @@ import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue; -import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; @@ -54,16 +53,12 @@ */ public class BazelDepGraphFunction implements SkyFunction { - private final Path rootDirectory; - - public BazelDepGraphFunction(Path rootDirectory) { - this.rootDirectory = rootDirectory; - } + public BazelDepGraphFunction() {} @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { + throws BazelDepGraphFunctionException, InterruptedException { RootModuleFileValue root = (RootModuleFileValue) env.getValue(ModuleFileValue.KEY_FOR_ROOT_MODULE); if (root == null) { @@ -74,11 +69,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) ImmutableMap localOverrideHashes = null; ImmutableMap depGraph = null; BzlmodFlagsAndEnvVars flags = null; + BazelLockFileValue lockFile = null; // If the module has not changed (has the same contents and flags as the lockfile), // read the dependency graph from the lock file, else run resolution and update lockfile if (!lockfileMode.equals(LockfileMode.OFF)) { - BazelLockFileValue lockFile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); + lockFile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); if (lockFile == null) { return null; } @@ -115,8 +111,14 @@ public SkyValue compute(SkyKey skyKey, Environment env) } depGraph = selectionResult.getResolvedDepGraph(); if (lockfileMode.equals(LockfileMode.UPDATE)) { - BazelLockFileFunction.updateLockedModule( - rootDirectory, root.getModuleFileHash(), flags, localOverrideHashes, depGraph); + BazelLockFileValue updatedLockFile = + lockFile.toBuilder() + .setModuleFileHash(root.getModuleFileHash()) + .setFlags(flags) + .setLocalOverrideHashes(localOverrideHashes) + .setModuleDepGraph(depGraph) + .build(); + env.getListener().post(updatedLockFile); } } @@ -214,9 +216,7 @@ private ImmutableTable getEx try { moduleExtensionId = ModuleExtensionId.create( - labelConverter.convert(usage.getExtensionBzlFile()), - usage.getExtensionName(), - usage.getIsolationKey()); + labelConverter.convert(usage.getExtensionBzlFile()), usage.getExtensionName()); } catch (LabelSyntaxException e) { throw new BazelDepGraphFunctionException( ExternalDepsException.withCauseAndMessage( @@ -250,31 +250,12 @@ private ImmutableBiMap calculateUniqueNameForUsedExte // not start with a tilde. RepositoryName repository = id.getBzlFileLabel().getRepository(); String nonEmptyRepoPart = repository.isMain() ? "_main" : repository.getName(); - // When using a namespace, prefix the extension name with "_" to distinguish the prefix from - // those generated by non-namespaced extension usages. Extension names are identified by their - // Starlark identifier, which in the case of an exported symbol cannot start with "_". - // We also include whether the isolated usage is a dev usage as well as its index in the - // MODULE.bazel file to ensure that canonical repository names don't change depending on - // whether dev dependencies are ignored. This removes potential for confusion and also - // prevents unnecessary refetches when --ignore_dev_dependency is toggled. - String bestName = - id.getIsolationKey() - .map( - namespace -> - String.format( - "%s~_%s~%s~%s~%s%d", - nonEmptyRepoPart, - id.getExtensionName(), - namespace.getModule().getName(), - namespace.getModule().getVersion(), - namespace.isDevUsage() ? "dev" : "", - namespace.getIsolatedUsageIndex())) - .orElse(nonEmptyRepoPart + "~" + id.getExtensionName()); + String bestName = nonEmptyRepoPart + "~" + id.getExtensionName(); if (extensionUniqueNames.putIfAbsent(bestName, id) == null) { continue; } int suffix = 2; - while (extensionUniqueNames.putIfAbsent(bestName + "~" + suffix, id) != null) { + while (extensionUniqueNames.putIfAbsent(bestName + suffix, id) != null) { suffix++; } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index ba6c5b6c192cdf..5200381621fc7e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -51,12 +51,13 @@ public class BazelLockFileFunction implements SkyFunction { ImmutableList.of(), ImmutableMap.of(), ImmutableList.of(), "", false, "", ""); private static final BazelLockFileValue EMPTY_LOCKFILE = - BazelLockFileValue.create( - BazelLockFileValue.LOCK_FILE_VERSION, - "", - EMPTY_FLAGS, - ImmutableMap.of(), - ImmutableMap.of()); + BazelLockFileValue.builder() + .setLockFileVersion(BazelLockFileValue.LOCK_FILE_VERSION) + .setModuleFileHash("") + .setFlags(EMPTY_FLAGS) + .setLocalOverrideHashes(ImmutableMap.of()) + .setModuleDepGraph(ImmutableMap.of()) + .build(); public BazelLockFileFunction(Path rootDirectory) { this.rootDirectory = rootDirectory; @@ -65,7 +66,7 @@ public BazelLockFileFunction(Path rootDirectory) { @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { + throws BazelLockfileFunctionException, InterruptedException { RootedPath lockfilePath = RootedPath.toRootedPath(Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); @@ -92,42 +93,9 @@ public SkyValue compute(SkyKey skyKey, Environment env) return bazelLockFileValue; } - /** - * Updates the stored module in the lock file (ModuleHash, Flags & Dependency graph) - * - * @param moduleFileHash The hash of the current module file - * @param resolvedDepGraph The resolved dependency graph from the module file - */ - public static void updateLockedModule( - Path rootDirectory, - String moduleFileHash, - BzlmodFlagsAndEnvVars flags, - ImmutableMap localOverrideHashes, - ImmutableMap resolvedDepGraph) - throws BazelLockfileFunctionException { - RootedPath lockfilePath = - RootedPath.toRootedPath(Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); - - BazelLockFileValue value = - BazelLockFileValue.create( - BazelLockFileValue.LOCK_FILE_VERSION, - moduleFileHash, - flags, - localOverrideHashes, - resolvedDepGraph); - try { - FileSystemUtils.writeContent(lockfilePath.asPath(), UTF_8, LOCKFILE_GSON.toJson(value)); - } catch (IOException e) { - throw new BazelLockfileFunctionException( - ExternalDepsException.withCauseAndMessage( - Code.BAD_MODULE, e, "Unable to update the MODULE.bazel.lock file"), - Transience.PERSISTENT); - } - } - static final class BazelLockfileFunctionException extends SkyFunctionException { - BazelLockfileFunctionException(Exception cause, Transience transience) { + BazelLockfileFunctionException(ExternalDepsException cause, Transience transience) { super(cause, transience); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java new file mode 100644 index 00000000000000..962c70a8f98d9c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java @@ -0,0 +1,86 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static com.google.devtools.build.lib.bazel.bzlmod.GsonTypeAdapterUtil.LOCKFILE_GSON; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.eventbus.Subscribe; +import com.google.common.flogger.GoogleLogger; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; +import com.google.devtools.build.lib.cmdline.LabelConstants; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; +import java.io.IOException; +import javax.annotation.Nullable; + +/** + * Module collecting Bazel module and module extensions resolution results and updating the + * lockfile. + */ +public class BazelLockFileModule extends BlazeModule { + + private Path workspaceRoot; + + @Nullable private BazelLockFileValue lockfileValue; + + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + @Override + public void beforeCommand(CommandEnvironment env) { + workspaceRoot = env.getWorkspace(); + RepositoryOptions options = env.getOptions().getOptions(RepositoryOptions.class); + if (options.lockfileMode.equals(LockfileMode.UPDATE)) { + env.getEventBus().register(this); + } + } + + @Override + public void afterCommand() { + if (lockfileValue == null) { // module didn't change --> no event was posted + return; + } + updateLockfile(workspaceRoot, lockfileValue); + this.lockfileValue = null; + } + + /** + * Updates the data stored in the lockfile (MODULE.bazel.lock) + * + * @param workspaceRoot Where to update the lockfile + * @param updatedLockfile The updated lockfile data to save + */ + public static void updateLockfile(Path workspaceRoot, BazelLockFileValue updatedLockfile) { + RootedPath lockfilePath = + RootedPath.toRootedPath(Root.fromPath(workspaceRoot), LabelConstants.MODULE_LOCKFILE_NAME); + try { + FileSystemUtils.writeContent( + lockfilePath.asPath(), UTF_8, LOCKFILE_GSON.toJson(updatedLockfile)); + } catch (IOException e) { + logger.atSevere().withCause(e).log( + "Error while updating MODULE.bazel.lock file: %s", e.getMessage()); + } + } + + @Subscribe + public void bazelModuleResolved(BazelLockFileValue lockfileValue) { + this.lockfileValue = lockfileValue; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java index 6f32405f61dd2c..fa93c4f39874e4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java @@ -18,6 +18,7 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.skyframe.SkyKey; @@ -33,20 +34,14 @@ */ @AutoValue @GenerateTypeAdapter -public abstract class BazelLockFileValue implements SkyValue { +public abstract class BazelLockFileValue implements SkyValue, Postable { public static final int LOCK_FILE_VERSION = 1; @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_LOCK_FILE; - public static BazelLockFileValue create( - int lockFileVersion, - String moduleFileHash, - BzlmodFlagsAndEnvVars flags, - ImmutableMap localOverrideHashes, - ImmutableMap moduleDepGraph) { - return new AutoValue_BazelLockFileValue( - lockFileVersion, moduleFileHash, flags, localOverrideHashes, moduleDepGraph); + static Builder builder() { + return new AutoValue_BazelLockFileValue.Builder().setLockFileVersion(LOCK_FILE_VERSION); } /** Current version of the lock file */ @@ -64,6 +59,24 @@ public static BazelLockFileValue create( /** The post-selection dep graph retrieved from the lock file. */ public abstract ImmutableMap getModuleDepGraph(); + public abstract Builder toBuilder(); + + /** Builder type for {@link BazelLockFileValue}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setLockFileVersion(int value); + + public abstract Builder setModuleFileHash(String value); + + public abstract Builder setFlags(BzlmodFlagsAndEnvVars value); + + public abstract Builder setLocalOverrideHashes(ImmutableMap value); + + public abstract Builder setModuleDepGraph(ImmutableMap value); + + public abstract BazelLockFileValue build(); + } + /** Returns the difference between the lockfile and the current module & flags */ public ArrayList getDiffLockfile( String moduleFileHash, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java index 33469c7367e421..8c79a470d6f76e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java @@ -83,7 +83,6 @@ public void afterCommand() { } try (Writer writer = Files.newWriter(new File(resolvedFile), StandardCharsets.UTF_8)) { writer.write(EXPORTED_NAME + " = " + new ValuePrinter().repr(resultBuilder.build())); - writer.close(); } catch (IOException e) { logger.atWarning().withCause(e).log("IO Error writing to file %s", resolvedFile); } diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java index a511ce2d7d0c8e..047c2aa1741b70 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java @@ -143,7 +143,7 @@ public ImmutableMap getSkyFunctions(BlazeDirectori directories.getWorkspace(), getBuiltinModules(directories)), SkyFunctions.BAZEL_DEP_GRAPH, - new BazelDepGraphFunction(directories.getWorkspace()), + new BazelDepGraphFunction(), SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace()), SkyFunctions.BAZEL_MODULE_RESOLUTION, diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 8c2c76bf5bc1d1..50de8c6f8cb7ef 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -30,6 +30,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info", "//src/main/java/com/google/devtools/build/lib/analysis:server_directories", "//src/main/java/com/google/devtools/build/lib/authandtls", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:bazel_lockfile_module", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:exception", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index 6894c77a50fe03..dfdab83e416e2a 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -127,7 +127,7 @@ public void setup() throws Exception { new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, resolutionFunctionMock) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index f1c775b201fbd7..269869f3866d92 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -27,6 +27,7 @@ import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ServerDirectories; import com.google.devtools.build.lib.analysis.util.AnalysisMock; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction.BazelLockfileFunctionException; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; @@ -63,7 +64,6 @@ import com.google.devtools.build.skyframe.RecordingDifferencer; import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; import com.google.devtools.build.skyframe.SkyFunction; -import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; @@ -71,6 +71,7 @@ import com.google.gson.JsonParser; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; import javax.annotation.Nullable; import net.starlark.java.eval.StarlarkSemantics; import org.junit.Before; @@ -93,7 +94,7 @@ public void setup() throws Exception { differencer = new SequencedRecordingDifferencer(); registryFactory = new FakeRegistry.Factory(); evaluationContext = - EvaluationContext.newBuilder().setNumThreads(8).setEventHandler(reporter).build(); + EvaluationContext.newBuilder().setParallelism(8).setEventHandler(reporter).build(); AtomicReference packageLocator = new AtomicReference<>( @@ -163,7 +164,7 @@ public void setup() throws Exception { @Nullable @Override public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { + throws BazelLockfileFunctionException, InterruptedException { UpdateLockFileKey key = (UpdateLockFileKey) skyKey; BzlmodFlagsAndEnvVars flags = BazelDepGraphFunction.getFlagsAndEnvVars(env); @@ -176,12 +177,14 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (localOverrideHashes == null) { return null; } - BazelLockFileFunction.updateLockedModule( + BazelLockFileModule.updateLockfile( rootDirectory, - key.moduleHash(), - flags, - localOverrideHashes, - key.depGraph()); + BazelLockFileValue.builder() + .setModuleFileHash(key.moduleHash()) + .setFlags(flags) + .setLocalOverrideHashes(localOverrideHashes) + .setModuleDepGraph(key.depGraph()) + .build()); return new SkyValue() {}; } }) @@ -268,10 +271,12 @@ public void moduleWithFlags() throws Exception { ImmutableList yankedVersions = ImmutableList.of("2.4", "2.3"); LocalPathOverride override = LocalPathOverride.create("override_path"); + ImmutableList registries = ImmutableList.of("registry1", "registry2"); + ImmutableMap moduleOverride = ImmutableMap.of("my_dep_1", override.getPath()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); + ModuleFileFunction.REGISTRIES.set(differencer, registries); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of("my_dep_1", override)); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, yankedVersions); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.ERROR); @@ -295,6 +300,8 @@ public void moduleWithFlags() throws Exception { BazelLockFileValue value = result.get(BazelLockFileValue.KEY); assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); assertThat(value.getFlags().ignoreDevDependency()).isTrue(); + assertThat(value.getFlags().cmdRegistries()).isEqualTo(registries); + assertThat(value.getFlags().cmdModuleOverrides()).isEqualTo(moduleOverride); assertThat(value.getFlags().allowedYankedVersions()).isEqualTo(yankedVersions); assertThat(value.getFlags().directDependenciesMode()) .isEqualTo(CheckDirectDepsMode.ERROR.toString()); @@ -425,8 +432,9 @@ public void invalidLockfileEmptyFile() throws Exception { } assertThat(result.getError().toString()) .contains( - "Failed to read and parse the MODULE.bazel.lock file with error: Null moduleFileHash." - + " Try deleting it and rerun the build."); + "Failed to read and parse the MODULE.bazel.lock file with error: " + + "java.lang.IllegalStateException: Missing required properties: moduleFileHash " + + "flags localOverrideHashes moduleDepGraph. Try deleting it and rerun the build."); } @Test @@ -511,12 +519,16 @@ public void invalidLockfileMalformed() throws Exception { if (!result.hasError()) { fail("expected error about invalid field value in the lockfile, but succeeded"); } - assertThat(result.getError().toString()) - .contains( - "Failed to read and parse the MODULE.bazel.lock file with error:" - + " java.lang.IllegalStateException: Expected BEGIN_ARRAY but was STRING at line 1" - + " column 129 path $.flags.allowedYankedVersions. Try deleting it and rerun the" - + " build."); + Pattern expectedExceptionMessage = + Pattern.compile( + Pattern.quote( + "Failed to read and parse the MODULE.bazel.lock file with error:" + + " java.lang.IllegalStateException: Expected BEGIN_ARRAY but was STRING at" + + " line 1 column 129 path $.flags.allowedYankedVersions") + + ".*" + + Pattern.quote("Try deleting it and rerun the build."), + Pattern.DOTALL); + assertThat(result.getError().toString()).containsMatch(expectedExceptionMessage); } @AutoValue diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 03fdbacf5e170f..427450dd5cd237 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -112,7 +112,7 @@ public void setup() throws Exception { SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 212bf5a2a2214b..d1a4ef5b5037c2 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -124,7 +124,7 @@ public void setup() throws Exception { .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( @@ -178,50 +178,6 @@ public void testRepoSpec_bazelModule() throws Exception { assertThat(repoRule.getAttr("path", Type.STRING)).isEqualTo("/usr/local/modules/ccc~2.0"); } - @Test - public void testRepoSpec_lockfile() throws Exception { - scratch.file( - workspaceRoot.getRelative("MODULE.bazel").getPathString(), - "bazel_dep(name='bbb',version='2.0')"); - - FakeRegistry registry = - registryFactory - .newFakeRegistry("/usr/local/modules") - .addModule(createModuleKey("bbb", "2.0"), "module(name='bbb', version='2.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); - - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); - - RepositoryName repo = RepositoryName.create("bbb~2.0"); - EvaluationResult result = - evaluator.evaluate(ImmutableList.of(BzlmodRepoRuleValue.key(repo)), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - BzlmodRepoRuleValue bzlmodRepoRuleValue = result.get(BzlmodRepoRuleValue.key(repo)); - Rule repoRule = bzlmodRepoRuleValue.getRule(); - assertThat(repoRule.getName()).isEqualTo("bbb~2.0"); - - /* Rerun the setup to: - 1.Reset evaluator to remove the sky values cache. - 2.Reset registry factory to be empty (now "bbb" doesn't exist at all) - without the lockfile "bbb" should not be found and this should fail - */ - setup(); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); - registry = registryFactory.newFakeRegistry("/usr/local/modules"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); - - // Calling again should read from lockfile and still find ccc - result = evaluator.evaluate(ImmutableList.of(BzlmodRepoRuleValue.key(repo)), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - bzlmodRepoRuleValue = result.get(BzlmodRepoRuleValue.key(repo)); - repoRule = bzlmodRepoRuleValue.getRule(); - assertThat(repoRule.getName()).isEqualTo("bbb~2.0"); - } - @Test public void testRepoSpec_nonRegistryOverride() throws Exception { scratch.file( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 31fce676ac7875..84e6defea0e153 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -251,7 +251,7 @@ public void setup() throws Exception { BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction()) .put(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction) diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java index b33273dc392f3e..94615cb8fd98a4 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -238,7 +238,7 @@ public void setupDelegator() throws Exception { .put( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootPath, ImmutableMap.of())) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, From ae5aa6053eb12fb9787395f9e86a1b56add4d036 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Mon, 26 Jun 2023 08:24:33 -0700 Subject: [PATCH 12/20] Add module extension to lockfile PiperOrigin-RevId: 543445157 Change-Id: Ib32a2c9fdc22c5b228d78141fc9648b3ef5edf7d # Conflicts: # src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java # src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java --- .../devtools/build/lib/bazel/bzlmod/BUILD | 6 + .../bazel/bzlmod/BazelDepGraphFunction.java | 80 +++--- .../bazel/bzlmod/BazelLockFileFunction.java | 18 +- .../lib/bazel/bzlmod/BazelLockFileModule.java | 88 +++++- .../lib/bazel/bzlmod/BazelLockFileValue.java | 47 +++- .../bzlmod/BazelModuleResolutionEvent.java | 39 +++ .../bazel/bzlmod/BzlmodFlagsAndEnvVars.java | 8 +- .../lib/bazel/bzlmod/GsonTypeAdapterUtil.java | 46 ++++ .../bazel/bzlmod/LockFileModuleExtension.java | 40 +++ .../ModuleExtensionResolutionEvent.java | 36 +++ .../bzlmod/SingleExtensionEvalFunction.java | 257 +++++++++++++----- .../bzlmod/BazelLockFileFunctionTest.java | 8 +- .../bzlmod/ModuleExtensionResolutionTest.java | 2 +- .../py/bazel/bzlmod/bazel_lockfile_test.py | 243 ++++++++++++++++- 14 files changed, 770 insertions(+), 148 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 02967174997589..61015909ecdbb1 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -92,10 +92,13 @@ java_library( ":resolution", ":resolution_impl", "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception", "//src/main/java/com/google/devtools/build/lib/vfs", "//third_party:flogger", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", ], @@ -108,14 +111,17 @@ java_library( "ArchiveOverride.java", "BazelDepGraphValue.java", "BazelLockFileValue.java", + "BazelModuleResolutionEvent.java", "BazelModuleResolutionValue.java", "BzlmodFlagsAndEnvVars.java", "GitOverride.java", "InterimModule.java", "LocalPathOverride.java", + "LockFileModuleExtension.java", "Module.java", "ModuleBase.java", "ModuleExtensionEvalStarlarkThreadContext.java", + "ModuleExtensionResolutionEvent.java", "ModuleFileValue.java", "ModuleOverride.java", "MultipleVersionOverride.java", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index f55e3fbc8b6b13..2852c88116e9a3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -41,7 +41,6 @@ import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; -import java.util.List; import java.util.Map; import java.util.Map.Entry; import javax.annotation.Nullable; @@ -69,13 +68,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) ImmutableMap localOverrideHashes = null; ImmutableMap depGraph = null; BzlmodFlagsAndEnvVars flags = null; - BazelLockFileValue lockFile = null; + BazelLockFileValue lockfile = null; // If the module has not changed (has the same contents and flags as the lockfile), // read the dependency graph from the lock file, else run resolution and update lockfile if (!lockfileMode.equals(LockfileMode.OFF)) { - lockFile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); - if (lockFile == null) { + lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); + if (lockfile == null) { return null; } flags = getFlagsAndEnvVars(env); @@ -83,17 +82,17 @@ public SkyValue compute(SkyKey skyKey, Environment env) return null; } localOverrideHashes = getLocalOverridesHashes(root.getOverrides(), env); - if (localOverrideHashes == null) { // trying to read override module + if (localOverrideHashes == null) { // still reading an override "module" return null; } - if (root.getModuleFileHash().equals(lockFile.getModuleFileHash()) - && flags.equals(lockFile.getFlags()) - && localOverrideHashes.equals(lockFile.getLocalOverrideHashes())) { - depGraph = lockFile.getModuleDepGraph(); + if (root.getModuleFileHash().equals(lockfile.getModuleFileHash()) + && flags.equals(lockfile.getFlags()) + && localOverrideHashes.equals(lockfile.getLocalOverrideHashes())) { + depGraph = lockfile.getModuleDepGraph(); } else if (lockfileMode.equals(LockfileMode.ERROR)) { - List diffLockfile = - lockFile.getDiffLockfile(root.getModuleFileHash(), localOverrideHashes, flags); + ImmutableList diffLockfile = + lockfile.getModuleAndFlagsDiff(root.getModuleFileHash(), localOverrideHashes, flags); throw new BazelDepGraphFunctionException( ExternalDepsException.withMessage( Code.BAD_MODULE, @@ -110,28 +109,34 @@ public SkyValue compute(SkyKey skyKey, Environment env) return null; } depGraph = selectionResult.getResolvedDepGraph(); - if (lockfileMode.equals(LockfileMode.UPDATE)) { - BazelLockFileValue updatedLockFile = - lockFile.toBuilder() - .setModuleFileHash(root.getModuleFileHash()) - .setFlags(flags) - .setLocalOverrideHashes(localOverrideHashes) - .setModuleDepGraph(depGraph) - .build(); - env.getListener().post(updatedLockFile); - } } ImmutableMap canonicalRepoNameLookup = depGraph.keySet().stream() .collect(toImmutableMap(ModuleKey::getCanonicalRepoName, key -> key)); - ImmutableTable extensionUsagesById = - getExtensionUsagesById(depGraph); + ImmutableTable extensionUsagesById; + try { + extensionUsagesById = getExtensionUsagesById(depGraph); + } catch (ExternalDepsException e) { + throw new BazelDepGraphFunctionException(e, Transience.PERSISTENT); + } ImmutableBiMap extensionUniqueNames = calculateUniqueNameForUsedExtensionId(extensionUsagesById); + if (!lockfileMode.equals(LockfileMode.OFF)) { + BazelLockFileValue updateLockfile = + lockfile.toBuilder() + .setModuleFileHash(root.getModuleFileHash()) + .setFlags(flags) + .setLocalOverrideHashes(localOverrideHashes) + .setModuleDepGraph(depGraph) + .build(); + env.getListener() + .post(BazelModuleResolutionEvent.create(updateLockfile, extensionUsagesById)); + } + return BazelDepGraphValue.create( depGraph, canonicalRepoNameLookup, @@ -202,8 +207,9 @@ static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws Interrup * For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can group * all usages by the label + name (the ModuleExtensionId). */ - private ImmutableTable getExtensionUsagesById( - ImmutableMap depGraph) throws BazelDepGraphFunctionException { + public static ImmutableTable + getExtensionUsagesById(ImmutableMap depGraph) + throws ExternalDepsException { ImmutableTable.Builder extensionUsagesTableBuilder = ImmutableTable.builder(); for (Module module : depGraph.values()) { @@ -218,22 +224,18 @@ private ImmutableTable getEx ModuleExtensionId.create( labelConverter.convert(usage.getExtensionBzlFile()), usage.getExtensionName()); } catch (LabelSyntaxException e) { - throw new BazelDepGraphFunctionException( - ExternalDepsException.withCauseAndMessage( - Code.BAD_MODULE, - e, - "invalid label for module extension found at %s", - usage.getLocation()), - Transience.PERSISTENT); + throw ExternalDepsException.withCauseAndMessage( + Code.BAD_MODULE, + e, + "invalid label for module extension found at %s", + usage.getLocation()); } if (!moduleExtensionId.getBzlFileLabel().getRepository().isVisible()) { - throw new BazelDepGraphFunctionException( - ExternalDepsException.withMessage( - Code.BAD_MODULE, - "invalid label for module extension found at %s: no repo visible as '@%s' here", - usage.getLocation(), - moduleExtensionId.getBzlFileLabel().getRepository().getName()), - Transience.PERSISTENT); + throw ExternalDepsException.withMessage( + Code.BAD_MODULE, + "invalid label for module extension found at %s: no repo visible as '@%s' here", + usage.getLocation(), + moduleExtensionId.getBzlFileLabel().getRepository().getName()); } extensionUsagesTableBuilder.put(moduleExtensionId, module.getKey(), usage); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index 5200381621fc7e..41cff393000ac1 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -57,6 +57,7 @@ public class BazelLockFileFunction implements SkyFunction { .setFlags(EMPTY_FLAGS) .setLocalOverrideHashes(ImmutableMap.of()) .setModuleDepGraph(ImmutableMap.of()) + .setModuleExtensions(ImmutableMap.of()) .build(); public BazelLockFileFunction(Path rootDirectory) { @@ -75,12 +76,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) return null; } - BazelLockFileValue bazelLockFileValue; try { - String json = FileSystemUtils.readContent(lockfilePath.asPath(), UTF_8); - bazelLockFileValue = LOCKFILE_GSON.fromJson(json, BazelLockFileValue.class); - } catch (FileNotFoundException e) { - bazelLockFileValue = EMPTY_LOCKFILE; + return getLockfileValue(lockfilePath); } catch (IOException | JsonSyntaxException | NullPointerException e) { throw new BazelLockfileFunctionException( ExternalDepsException.withMessage( @@ -90,9 +87,20 @@ public SkyValue compute(SkyKey skyKey, Environment env) e.getMessage()), Transience.PERSISTENT); } + } + + public static BazelLockFileValue getLockfileValue(RootedPath lockfilePath) throws IOException { + BazelLockFileValue bazelLockFileValue; + try { + String json = FileSystemUtils.readContent(lockfilePath.asPath(), UTF_8); + bazelLockFileValue = LOCKFILE_GSON.fromJson(json, BazelLockFileValue.class); + } catch (FileNotFoundException e) { + bazelLockFileValue = EMPTY_LOCKFILE; + } return bazelLockFileValue; } + static final class BazelLockfileFunctionException extends SkyFunctionException { BazelLockfileFunctionException(ExternalDepsException cause, Transience transience) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java index 962c70a8f98d9c..521dd09a1a6dde 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java @@ -17,6 +17,7 @@ import static com.google.devtools.build.lib.bazel.bzlmod.GsonTypeAdapterUtil.LOCKFILE_GSON; import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.collect.ImmutableMap; import com.google.common.eventbus.Subscribe; import com.google.common.flogger.GoogleLogger; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; @@ -24,11 +25,16 @@ import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.Root; import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.gson.JsonSyntaxException; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import javax.annotation.Nullable; /** @@ -38,8 +44,8 @@ public class BazelLockFileModule extends BlazeModule { private Path workspaceRoot; - - @Nullable private BazelLockFileValue lockfileValue; + @Nullable private BazelModuleResolutionEvent moduleResolutionEvent; + private final List extensionResolutionEvents = new ArrayList<>(); private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); @@ -53,23 +59,74 @@ public void beforeCommand(CommandEnvironment env) { } @Override - public void afterCommand() { - if (lockfileValue == null) { // module didn't change --> no event was posted - return; + public void afterCommand() throws AbruptExitException { + if (moduleResolutionEvent == null && extensionResolutionEvents.isEmpty()) { + return; // nothing changed, do nothing! + } + + RootedPath lockfilePath = + RootedPath.toRootedPath(Root.fromPath(workspaceRoot), LabelConstants.MODULE_LOCKFILE_NAME); + + // Create an updated version of the lockfile with the events updates + BazelLockFileValue lockfile; + if (moduleResolutionEvent != null) { + lockfile = moduleResolutionEvent.getLockfileValue(); + } else { + // Read the existing lockfile (if none exists, will get an empty lockfile value) + try { + lockfile = BazelLockFileFunction.getLockfileValue(lockfilePath); + } catch (IOException | JsonSyntaxException | NullPointerException e) { + logger.atSevere().withCause(e).log( + "Failed to read and parse the MODULE.bazel.lock file with error: %s." + + " Try deleting it and rerun the build.", + e.getMessage()); + return; + } } - updateLockfile(workspaceRoot, lockfileValue); - this.lockfileValue = null; + lockfile = + lockfile.toBuilder() + .setModuleExtensions(combineModuleExtensions(lockfile.getModuleExtensions())) + .build(); + + // Write the new value to the file + updateLockfile(lockfilePath, lockfile); + this.moduleResolutionEvent = null; + this.extensionResolutionEvents.clear(); + } + + /** + * Combines the old extensions stored in the lockfile -that are still used- with the new + * extensions from the events (if any) + * + * @param oldModuleExtensions Module extensions stored in the current lockfile + */ + private ImmutableMap combineModuleExtensions( + ImmutableMap oldModuleExtensions) { + ImmutableMap.Builder updatedExtensionMap = + ImmutableMap.builder(); + + // Add the old extensions (stored in the lockfile) only if it still has a usage somewhere + for (Map.Entry extensionEntry : + oldModuleExtensions.entrySet()) { + if (moduleResolutionEvent.getExtensionUsagesById().containsRow(extensionEntry.getKey())) { + updatedExtensionMap.put(extensionEntry); + } + } + + // Add the new resolved extensions + for (ModuleExtensionResolutionEvent extensionEvent : extensionResolutionEvents) { + updatedExtensionMap.put(extensionEvent.getExtensionId(), extensionEvent.getModuleExtension()); + } + return updatedExtensionMap.buildKeepingLast(); } /** * Updates the data stored in the lockfile (MODULE.bazel.lock) * - * @param workspaceRoot Where to update the lockfile + * @param lockfilePath Rooted path to lockfile * @param updatedLockfile The updated lockfile data to save */ - public static void updateLockfile(Path workspaceRoot, BazelLockFileValue updatedLockfile) { - RootedPath lockfilePath = - RootedPath.toRootedPath(Root.fromPath(workspaceRoot), LabelConstants.MODULE_LOCKFILE_NAME); + public static void updateLockfile(RootedPath lockfilePath, BazelLockFileValue updatedLockfile) { try { FileSystemUtils.writeContent( lockfilePath.asPath(), UTF_8, LOCKFILE_GSON.toJson(updatedLockfile)); @@ -80,7 +137,12 @@ public static void updateLockfile(Path workspaceRoot, BazelLockFileValue updated } @Subscribe - public void bazelModuleResolved(BazelLockFileValue lockfileValue) { - this.lockfileValue = lockfileValue; + public void bazelModuleResolved(BazelModuleResolutionEvent moduleResolutionEvent) { + this.moduleResolutionEvent = moduleResolutionEvent; + } + + @Subscribe + public void moduleExtensionResolved(ModuleExtensionResolutionEvent extensionResolutionEvent) { + this.extensionResolutionEvents.add(extensionResolutionEvent); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java index fa93c4f39874e4..c0d0c06aed41e4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java @@ -17,6 +17,7 @@ import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; import com.google.devtools.build.lib.skyframe.SkyFunctions; @@ -24,7 +25,7 @@ import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Map; /** @@ -41,7 +42,9 @@ public abstract class BazelLockFileValue implements SkyValue, Postable { @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_LOCK_FILE; static Builder builder() { - return new AutoValue_BazelLockFileValue.Builder().setLockFileVersion(LOCK_FILE_VERSION); + return new AutoValue_BazelLockFileValue.Builder() + .setLockFileVersion(LOCK_FILE_VERSION) + .setModuleExtensions(ImmutableMap.of()); } /** Current version of the lock file */ @@ -59,6 +62,9 @@ static Builder builder() { /** The post-selection dep graph retrieved from the lock file. */ public abstract ImmutableMap getModuleDepGraph(); + /** Mapping the extension id to the module extension data */ + public abstract ImmutableMap getModuleExtensions(); + public abstract Builder toBuilder(); /** Builder type for {@link BazelLockFileValue}. */ @@ -74,30 +80,55 @@ public abstract static class Builder { public abstract Builder setModuleDepGraph(ImmutableMap value); + public abstract Builder setModuleExtensions( + ImmutableMap value); + public abstract BazelLockFileValue build(); } /** Returns the difference between the lockfile and the current module & flags */ - public ArrayList getDiffLockfile( + public ImmutableList getModuleAndFlagsDiff( String moduleFileHash, ImmutableMap localOverrideHashes, BzlmodFlagsAndEnvVars flags) { - ArrayList diffLockfile = new ArrayList<>(); + ImmutableList.Builder moduleDiff = new ImmutableList.Builder<>(); if (!moduleFileHash.equals(getModuleFileHash())) { - diffLockfile.add("the root MODULE.bazel has been modified"); + moduleDiff.add("the root MODULE.bazel has been modified"); } - diffLockfile.addAll(getFlags().getDiffFlags(flags)); + moduleDiff.addAll(getFlags().getDiffFlags(flags)); for (Map.Entry entry : localOverrideHashes.entrySet()) { String currentValue = entry.getValue(); String lockfileValue = getLocalOverrideHashes().get(entry.getKey()); // If the lockfile value is null, the module hash would be different anyway if (lockfileValue != null && !currentValue.equals(lockfileValue)) { - diffLockfile.add( + moduleDiff.add( "The MODULE.bazel file has changed for the overriden module: " + entry.getKey()); } } + return moduleDiff.build(); + } - return diffLockfile; + public ImmutableList getModuleExtensionDiff( + LockFileModuleExtension lockedExtension, + ImmutableMap lockedExtensionUsages, + ModuleExtensionId extensionId, + byte[] transitiveDigest, + ImmutableMap extensionUsages) { + ImmutableList.Builder extDiff = new ImmutableList.Builder<>(); + if (lockedExtension == null) { + extDiff.add("The module extension '" + extensionId + "' does not exist in the lockfile"); + } else { + if (!Arrays.equals(transitiveDigest, lockedExtension.getBzlTransitiveDigest())) { + extDiff.add( + "The implementation of the extension '" + + extensionId + + "' or one of its transitive .bzl files has changed"); + } + if (!extensionUsages.equals(lockedExtensionUsages)) { + extDiff.add("The usages of the extension named '" + extensionId + "' has changed"); + } + } + return extDiff.build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java new file mode 100644 index 00000000000000..81a0dbaa144b9c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java @@ -0,0 +1,39 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableTable; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; + +/** + * After resolving bazel module this event is sent from {@link BazelDepGraphFunction} holding the + * lockfile value with module updates, and the module extension usages. It will be received in + * {@link BazelLockFileModule} to be used to update the lockfile content + */ +@AutoValue +public abstract class BazelModuleResolutionEvent implements Postable { + + public static BazelModuleResolutionEvent create( + BazelLockFileValue lockFileValue, + ImmutableTable extensionUsagesById) { + return new AutoValue_BazelModuleResolutionEvent(lockFileValue, extensionUsagesById); + } + + public abstract BazelLockFileValue getLockfileValue(); + + public abstract ImmutableTable + getExtensionUsagesById(); +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java index f5f483f716c04b..a69e53791f247f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// package com.google.devtools.build.lib.bazel.bzlmod; @@ -19,7 +18,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; -import java.util.ArrayList; /** Stores the values of flags and environment variables that affect the resolution */ @AutoValue @@ -65,8 +63,8 @@ public static BzlmodFlagsAndEnvVars create( /** Error level of bazel compatability check */ public abstract String compatibilityMode(); - public ArrayList getDiffFlags(BzlmodFlagsAndEnvVars flags) { - ArrayList diffFlags = new ArrayList<>(); + public ImmutableList getDiffFlags(BzlmodFlagsAndEnvVars flags) { + ImmutableList.Builder diffFlags = new ImmutableList.Builder<>(); if (!flags.cmdRegistries().equals(cmdRegistries())) { diffFlags.add("the value of --registry flag has been modified"); } @@ -89,6 +87,6 @@ public ArrayList getDiffFlags(BzlmodFlagsAndEnvVars flags) { if (!flags.compatibilityMode().equals(compatibilityMode())) { diffFlags.add("the value of --check_bazel_compatibility flag has been modified"); } - return diffFlags; + return diffFlags.build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java index 15518989781133..c558e806e9d6d3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java @@ -23,6 +23,8 @@ import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParseException; @@ -36,6 +38,7 @@ import java.io.IOException; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; +import java.util.Base64; import java.util.List; import java.util.Optional; import javax.annotation.Nullable; @@ -96,6 +99,45 @@ public ModuleKey read(JsonReader jsonReader) throws IOException { } }; + // TODO(salmasamy) need to handle "isolated" in module extensions when it is stable + public static final TypeAdapter MODULE_EXTENSION_ID_TYPE_ADAPTER = + new TypeAdapter<>() { + @Override + public void write(JsonWriter jsonWriter, ModuleExtensionId moduleExtId) throws IOException { + jsonWriter.value(moduleExtId.getBzlFileLabel() + "%" + moduleExtId.getExtensionName()); + } + + @Override + public ModuleExtensionId read(JsonReader jsonReader) throws IOException { + String jsonString = jsonReader.nextString(); + // [0] is labelString, [1] is extensionName + List extIdParts = Splitter.on("%").splitToList(jsonString); + try { + return ModuleExtensionId.create( + Label.parseCanonical(extIdParts.get(0)), extIdParts.get(1), Optional.empty()); + } catch (LabelSyntaxException e) { + throw new JsonParseException( + String.format( + "Unable to parse ModuleExtensionID bzl file label: '%s' from the lockfile", + extIdParts.get(0)), + e); + } + } + }; + + public static final TypeAdapter BYTE_ARRAY_TYPE_ADAPTER = + new TypeAdapter<>() { + @Override + public void write(JsonWriter jsonWriter, byte[] value) throws IOException { + jsonWriter.value(Base64.getEncoder().encodeToString(value)); + } + + @Override + public byte[] read(JsonReader jsonReader) throws IOException { + return Base64.getDecoder().decode(jsonReader.nextString()); + } + }; + public static final TypeAdapterFactory OPTIONAL = new TypeAdapterFactory() { @Nullable @@ -149,6 +191,8 @@ public Optional read(JsonReader jsonReader) throws IOException { public static final Gson LOCKFILE_GSON = new GsonBuilder() .setPrettyPrinting() + .disableHtmlEscaping() + .enableComplexMapKeySerialization() .registerTypeAdapterFactory(GenerateTypeAdapter.FACTORY) .registerTypeAdapterFactory(DICT) .registerTypeAdapterFactory(IMMUTABLE_MAP) @@ -158,7 +202,9 @@ public Optional read(JsonReader jsonReader) throws IOException { .registerTypeAdapterFactory(OPTIONAL) .registerTypeAdapter(Version.class, VERSION_TYPE_ADAPTER) .registerTypeAdapter(ModuleKey.class, MODULE_KEY_TYPE_ADAPTER) + .registerTypeAdapter(ModuleExtensionId.class, MODULE_EXTENSION_ID_TYPE_ADAPTER) .registerTypeAdapter(AttributeValues.class, new AttributeValuesAdapter()) + .registerTypeAdapter(byte[].class, BYTE_ARRAY_TYPE_ADAPTER) .create(); private GsonTypeAdapterUtil() {} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java new file mode 100644 index 00000000000000..af4da1e131ee18 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java @@ -0,0 +1,40 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; + +/** + * This object serves as a container for the transitive digest (obtained from transitive .bzl files) + * and the generated repositories from evaluating a module extension. Its purpose is to store this + * information within the lockfile. + */ +@AutoValue +@GenerateTypeAdapter +public abstract class LockFileModuleExtension implements Postable { + + @SuppressWarnings("AutoValueImmutableFields") + public abstract byte[] getBzlTransitiveDigest(); + + public abstract ImmutableMap getGeneratedRepoSpecs(); + + public static LockFileModuleExtension create( + byte[] transitiveDigest, ImmutableMap generatedRepoSpecs) { + return new AutoValue_LockFileModuleExtension(transitiveDigest, generatedRepoSpecs); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java new file mode 100644 index 00000000000000..4013e7bb4c5965 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java @@ -0,0 +1,36 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; + +/** + * After evaluating any module extension this event is sent from {@link SingleExtensionEvalFunction} + * holding the extension id and the resolution data LockFileModuleExtension. It will be received in + * {@link BazelLockFileModule} to be used to update the lockfile content + */ +@AutoValue +public abstract class ModuleExtensionResolutionEvent implements Postable { + + public static ModuleExtensionResolutionEvent create( + ModuleExtensionId extensionId, LockFileModuleExtension lockfileModuleExtension) { + return new AutoValue_ModuleExtensionResolutionEvent(extensionId, lockfileModuleExtension); + } + + public abstract ModuleExtensionId getExtensionId(); + + public abstract LockFileModuleExtension getModuleExtension(); +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java index b76131c9761da0..29a32ef988ec17 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java @@ -18,10 +18,15 @@ import static com.google.common.collect.ImmutableBiMap.toImmutableBiMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableTable; import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.cmdline.BazelModuleContext; +import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -29,6 +34,7 @@ import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException; import com.google.devtools.build.lib.runtime.ProcessWrapper; import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor; +import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; import com.google.devtools.build.lib.skyframe.BzlLoadFunction; import com.google.devtools.build.lib.skyframe.BzlLoadFunction.BzlLoadFailedException; @@ -42,6 +48,7 @@ import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Map; import java.util.Map.Entry; import java.util.function.Function; @@ -94,7 +101,7 @@ public void setRepositoryRemoteExecutor(RepositoryRemoteExecutor repositoryRemot @Nullable @Override public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { + throws SingleExtensionEvalFunctionException, InterruptedException { StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env); if (starlarkSemantics == null) { return null; @@ -108,11 +115,163 @@ public SkyValue compute(SkyKey skyKey, Environment env) } Location sampleUsageLocation = usagesValue.getExtensionUsages().values().iterator().next().getLocation(); + BzlLoadValue bzlLoadValue = + loadBzlFile(extensionId.getBzlFileLabel(), sampleUsageLocation, starlarkSemantics, env); + if (bzlLoadValue == null) { + return null; + } + // TODO(wyv): Consider whether there's a need to check .bzl load visibility + // (BzlLoadFunction#checkLoadVisibilities). + // TODO(wyv): Consider refactoring to use PackageFunction#loadBzlModules, or the simpler API + // that may be created by b/237658764. + + // Check that the .bzl file actually exports a module extension by our name. + Object exported = bzlLoadValue.getModule().getGlobal(extensionId.getExtensionName()); + if (!(exported instanceof ModuleExtension)) { + ImmutableSet exportedExtensions = + bzlLoadValue.getModule().getGlobals().entrySet().stream() + .filter(e -> e.getValue() instanceof ModuleExtension) + .map(Entry::getKey) + .collect(toImmutableSet()); + throw new SingleExtensionEvalFunctionException( + ExternalDepsException.withMessage( + ExternalDeps.Code.BAD_MODULE, + "%s does not export a module extension called %s, yet its use is requested at %s%s", + extensionId.getBzlFileLabel(), + extensionId.getExtensionName(), + sampleUsageLocation, + SpellChecker.didYouMean(extensionId.getExtensionName(), exportedExtensions)), + Transience.PERSISTENT); + } + + // Check the lockfile first for that module extension + byte[] bzlTransitiveDigest = + BazelModuleContext.of(bzlLoadValue.getModule()).bzlTransitiveDigest(); + LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env); + if (!lockfileMode.equals(LockfileMode.OFF)) { + BazelLockFileValue lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); + if (lockfile == null) { + return null; + } + SingleExtensionEvalValue singleExtensionEvalValue = + tryGettingValueFromLockFile( + extensionId, usagesValue, bzlTransitiveDigest, lockfileMode, lockfile); + if (singleExtensionEvalValue != null) { + return singleExtensionEvalValue; + } + } + + // Run that extension! + ModuleExtension extension = (ModuleExtension) exported; + ImmutableMap generatedRepoSpecs = + runModuleExtension( + extensionId, extension, usagesValue, bzlLoadValue.getModule(), starlarkSemantics, env); + if (generatedRepoSpecs == null) { + return null; + } + // Check that all imported repos have been actually generated + validateAllImportsAreGenerated(generatedRepoSpecs, usagesValue, extensionId); + + if (lockfileMode.equals(LockfileMode.UPDATE)) { + env.getListener() + .post( + ModuleExtensionResolutionEvent.create( + extensionId, + LockFileModuleExtension.create(bzlTransitiveDigest, generatedRepoSpecs))); + } + return createSingleExtentionValue(generatedRepoSpecs, usagesValue); + } + + @Nullable + private SingleExtensionEvalValue tryGettingValueFromLockFile( + ModuleExtensionId extensionId, + SingleExtensionUsagesValue usagesValue, + byte[] bzlTransitiveDigest, + LockfileMode lockfileMode, + BazelLockFileValue lockfile) + throws SingleExtensionEvalFunctionException { + LockFileModuleExtension lockedExtension = lockfile.getModuleExtensions().get(extensionId); + ImmutableMap lockedExtensionUsages; + try { + // TODO(salmasamy) might be nicer to precompute this table when we construct + // BazelLockFileValue, without adding it to the json file + ImmutableTable extensionUsagesById = + BazelDepGraphFunction.getExtensionUsagesById(lockfile.getModuleDepGraph()); + lockedExtensionUsages = extensionUsagesById.row(extensionId); + } catch (ExternalDepsException e) { + throw new SingleExtensionEvalFunctionException(e, Transience.PERSISTENT); + } + + // If we have the extension, check if the implementation and usage haven't changed + if (lockedExtension != null + && Arrays.equals(bzlTransitiveDigest, lockedExtension.getBzlTransitiveDigest()) + && usagesValue.getExtensionUsages().equals(lockedExtensionUsages)) { + return createSingleExtentionValue(lockedExtension.getGeneratedRepoSpecs(), usagesValue); + } else if (lockfileMode.equals(LockfileMode.ERROR)) { + ImmutableList extDiff = + lockfile.getModuleExtensionDiff( + lockedExtension, + lockedExtensionUsages, + extensionId, + bzlTransitiveDigest, + usagesValue.getExtensionUsages()); + throw new SingleExtensionEvalFunctionException( + ExternalDepsException.withMessage( + Code.BAD_MODULE, + "Lock file is no longer up-to-date because: %s", + String.join(", ", extDiff)), + Transience.PERSISTENT); + } + return null; + } + private SingleExtensionEvalValue createSingleExtentionValue( + ImmutableMap generatedRepoSpecs, SingleExtensionUsagesValue usagesValue) { + return SingleExtensionEvalValue.create( + generatedRepoSpecs, + generatedRepoSpecs.keySet().stream() + .collect( + toImmutableBiMap( + e -> + RepositoryName.createUnvalidated( + usagesValue.getExtensionUniqueName() + "~" + e), + Function.identity()))); + } + + private void validateAllImportsAreGenerated( + ImmutableMap generatedRepoSpecs, + SingleExtensionUsagesValue usagesValue, + ModuleExtensionId extensionId) + throws SingleExtensionEvalFunctionException { + for (ModuleExtensionUsage usage : usagesValue.getExtensionUsages().values()) { + for (Entry repoImport : usage.getImports().entrySet()) { + if (!generatedRepoSpecs.containsKey(repoImport.getValue())) { + throw new SingleExtensionEvalFunctionException( + ExternalDepsException.withMessage( + Code.BAD_MODULE, + "module extension \"%s\" from \"%s\" does not generate repository \"%s\", yet it" + + " is imported as \"%s\" in the usage at %s%s", + extensionId.getExtensionName(), + extensionId.getBzlFileLabel(), + repoImport.getValue(), + repoImport.getKey(), + usage.getLocation(), + SpellChecker.didYouMean(repoImport.getValue(), generatedRepoSpecs.keySet())), + Transience.PERSISTENT); + } + } + } + } + + private BzlLoadValue loadBzlFile( + Label bzlFileLabel, + Location sampleUsageLocation, + StarlarkSemantics starlarkSemantics, + Environment env) + throws SingleExtensionEvalFunctionException, InterruptedException { // Check that the .bzl label isn't crazy. try { - BzlLoadFunction.checkValidLoadLabel( - extensionId.getBzlFileLabel(), /*fromBuiltinsRepo=*/ false); + BzlLoadFunction.checkValidLoadLabel(bzlFileLabel, starlarkSemantics); } catch (LabelSyntaxException e) { throw new SingleExtensionEvalFunctionException( ExternalDepsException.withCauseAndMessage( @@ -126,53 +285,35 @@ public SkyValue compute(SkyKey skyKey, Environment env) bzlLoadValue = (BzlLoadValue) env.getValueOrThrow( - BzlLoadValue.keyForBzlmod(extensionId.getBzlFileLabel()), - BzlLoadFailedException.class); + BzlLoadValue.keyForBzlmod(bzlFileLabel), BzlLoadFailedException.class); } catch (BzlLoadFailedException e) { throw new SingleExtensionEvalFunctionException( ExternalDepsException.withCauseAndMessage( Code.BAD_MODULE, e, "Error loading '%s' for module extensions, requested by %s: %s", - extensionId.getBzlFileLabel(), + bzlFileLabel, sampleUsageLocation, e.getMessage()), Transience.PERSISTENT); } - if (bzlLoadValue == null) { - return null; - } - // TODO(wyv): Consider whether there's a need to check .bzl load visibility - // (BzlLoadFunction#checkLoadVisibilities). - // TODO(wyv): Consider refactoring to use PackageFunction#loadBzlModules, or the simpler API - // that may be created by b/237658764. - - // Check that the .bzl file actually exports a module extension by our name. - Object exported = bzlLoadValue.getModule().getGlobal(extensionId.getExtensionName()); - if (!(exported instanceof ModuleExtension.InStarlark)) { - ImmutableSet exportedExtensions = - bzlLoadValue.getModule().getGlobals().entrySet().stream() - .filter(e -> e.getValue() instanceof ModuleExtension.InStarlark) - .map(Entry::getKey) - .collect(toImmutableSet()); - throw new SingleExtensionEvalFunctionException( - ExternalDepsException.withMessage( - Code.BAD_MODULE, - "%s does not export a module extension called %s, yet its use is requested at %s%s", - extensionId.getBzlFileLabel(), - extensionId.getExtensionName(), - sampleUsageLocation, - SpellChecker.didYouMean(extensionId.getExtensionName(), exportedExtensions)), - Transience.PERSISTENT); - } + return bzlLoadValue; + } - // Run that extension! - ModuleExtension extension = ((ModuleExtension.InStarlark) exported).get(); + @Nullable + private ImmutableMap runModuleExtension( + ModuleExtensionId extensionId, + ModuleExtension extension, + SingleExtensionUsagesValue usagesValue, + net.starlark.java.eval.Module module, + StarlarkSemantics starlarkSemantics, + Environment env) + throws SingleExtensionEvalFunctionException, InterruptedException { ModuleExtensionEvalStarlarkThreadContext threadContext = new ModuleExtensionEvalStarlarkThreadContext( usagesValue.getExtensionUniqueName() + "~", extensionId.getBzlFileLabel().getPackageIdentifier(), - BazelModuleContext.of(bzlLoadValue.getModule()).repoMapping(), + BazelModuleContext.of(module).repoMapping(), directories, env.getListener()); try (Mutability mu = @@ -189,7 +330,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (returnValue != Starlark.NONE && !(returnValue instanceof ModuleExtensionMetadata)) { throw new SingleExtensionEvalFunctionException( ExternalDepsException.withMessage( - Code.BAD_MODULE, + ExternalDeps.Code.BAD_MODULE, "expected module extension %s in %s to return None or extension_metadata, got %s", extensionId.getExtensionName(), extensionId.getBzlFileLabel(), @@ -210,51 +351,27 @@ public SkyValue compute(SkyKey skyKey, Environment env) moduleContext.getWorkingDirectory().deleteTree(); } } catch (IOException e1) { - throw new SingleExtensionEvalFunctionException(e1, Transience.TRANSIENT); + ExternalDepsException externalDepsException = + ExternalDepsException.withCauseAndMessage( + ExternalDeps.Code.UNRECOGNIZED, + e1, + "Failed to clean up module context directory"); + throw new SingleExtensionEvalFunctionException( + externalDepsException, Transience.TRANSIENT); } return null; } catch (EvalException e) { env.getListener().handle(Event.error(e.getMessageWithStack())); throw new SingleExtensionEvalFunctionException( ExternalDepsException.withMessage( - Code.BAD_MODULE, + ExternalDeps.Code.BAD_MODULE, "error evaluating module extension %s in %s", extensionId.getExtensionName(), extensionId.getBzlFileLabel()), Transience.TRANSIENT); } } - - // Check that all imported repos have been actually generated - for (ModuleExtensionUsage usage : usagesValue.getExtensionUsages().values()) { - for (Entry repoImport : usage.getImports().entrySet()) { - if (!threadContext.getGeneratedRepoSpecs().containsKey(repoImport.getValue())) { - throw new SingleExtensionEvalFunctionException( - ExternalDepsException.withMessage( - Code.BAD_MODULE, - "module extension \"%s\" from \"%s\" does not generate repository \"%s\", yet it" - + " is imported as \"%s\" in the usage at %s%s", - extensionId.getExtensionName(), - extensionId.getBzlFileLabel(), - repoImport.getValue(), - repoImport.getKey(), - usage.getLocation(), - SpellChecker.didYouMean( - repoImport.getValue(), threadContext.getGeneratedRepoSpecs().keySet())), - Transience.PERSISTENT); - } - } - } - - return SingleExtensionEvalValue.create( - threadContext.getGeneratedRepoSpecs(), - threadContext.getGeneratedRepoSpecs().keySet().stream() - .collect( - toImmutableBiMap( - e -> - RepositoryName.createUnvalidated( - usagesValue.getExtensionUniqueName() + "~" + e), - Function.identity()))); + return threadContext.getGeneratedRepoSpecs(); } private ModuleExtensionContext createContext( @@ -298,7 +415,7 @@ private ModuleExtensionContext createContext( static final class SingleExtensionEvalFunctionException extends SkyFunctionException { - SingleExtensionEvalFunctionException(Exception cause, Transience transience) { + SingleExtensionEvalFunctionException(ExternalDepsException cause, Transience transience) { super(cause, transience); } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index 269869f3866d92..4efcebc1ece290 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -34,6 +34,7 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule; import com.google.devtools.build.lib.clock.BlazeClock; +import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; import com.google.devtools.build.lib.rules.repository.LocalRepositoryFunction; @@ -56,6 +57,7 @@ import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; import com.google.devtools.build.lib.vfs.FileStateKey; import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; @@ -177,14 +179,18 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (localOverrideHashes == null) { return null; } + RootedPath lockfilePath = + RootedPath.toRootedPath( + Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); BazelLockFileModule.updateLockfile( - rootDirectory, + lockfilePath, BazelLockFileValue.builder() .setModuleFileHash(key.moduleHash()) .setFlags(flags) .setLocalOverrideHashes(localOverrideHashes) .setModuleDepGraph(key.depGraph()) .build()); + return new SkyValue() {}; } }) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 84e6defea0e153..d73b2d479795ae 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -282,7 +282,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); // Set up a simple repo rule. registry.addModule( diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py index f9fe557bbd1851..c91173e323914d 100644 --- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -14,6 +14,7 @@ # limitations under the License. # pylint: disable=g-long-ternary +import json import os import tempfile import unittest @@ -241,12 +242,7 @@ def testLocalOverrideWithErrorMode(self): ], ) exit_code, _, stderr = self.RunBazel( - [ - 'build', - '--nobuild', - '--lockfile_mode=error', - '//:all', - ], + ['build', '--nobuild', '--lockfile_mode=error', '//:all'], allow_failure=True, ) self.AssertExitCode(exit_code, 48, stderr) @@ -259,6 +255,241 @@ def testLocalOverrideWithErrorMode(self): stderr, ) + def testModuleExtension(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'lockfile_ext = use_extension("extension.bzl", "lockfile_ext")', + 'lockfile_ext.dep(name = "bmbm", versions = ["v1", "v2"])', + 'use_repo(lockfile_ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + '', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + '', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other side!")', + ' repo_rule(name="hello")', + ' for mod in ctx.modules:', + ' for dep in mod.tags.dep:', + ' print("Name:", dep.name, ", Versions:", dep.versions)', + '', + ( + '_dep = tag_class(attrs={"name": attr.string(), "versions":' + ' attr.string_list()})' + ), + 'lockfile_ext = module_extension(', + ' implementation=_module_ext_impl,', + ' tag_classes={"dep": _dep},', + ')', + ], + ) + + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertIn('Hello from the other side!', ''.join(stderr)) + self.assertIn('Name: bmbm , Versions: ["v1", "v2"]', ''.join(stderr)) + + self.RunBazel(['shutdown']) + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertNotIn('Hello from the other side!', ''.join(stderr)) + + def testModuleExtensionsInDifferentBuilds(self): + # Test that the module extension stays in the lockfile (as long as it's + # used in the module) even if it is not in the current build + self.ScratchFile( + 'MODULE.bazel', + [ + 'extA = use_extension("extension.bzl", "extA")', + 'extB = use_extension("extension.bzl", "extB")', + 'use_repo(extA, "hello_ext_A")', + 'use_repo(extB, "hello_ext_B")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + 'def _ext_a_impl(ctx):', + ' repo_rule(name="hello_ext_A")', + 'def _ext_b_impl(ctx):', + ' repo_rule(name="hello_ext_B")', + 'extA = module_extension(implementation=_ext_a_impl)', + 'extB = module_extension(implementation=_ext_b_impl)', + ], + ) + + self.RunBazel(['build', '@hello_ext_A//:all']) + self.RunBazel(['build', '@hello_ext_B//:all']) + + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + self.assertGreater(len(lockfile['moduleDepGraph']), 0) + self.assertEqual( + list(lockfile['moduleExtensions'].keys()), + ['//:extension.bzl%extA', '//:extension.bzl%extB'], + ) + + def testUpdateModuleExtension(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'lockfile_ext = use_extension("extension.bzl", "lockfile_ext")', + 'use_repo(lockfile_ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\\"lala\\")")', + 'repo_rule = repository_rule(implementation = _repo_rule_impl)', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other side!")', + ' repo_rule(name= "hello")', + ( + 'lockfile_ext = module_extension(implementation =' + ' _module_ext_impl)' + ), + ], + ) + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertIn('Hello from the other side!', ''.join(stderr)) + self.RunBazel(['shutdown']) + + # Update extension. Make sure that it is executed and updated in the + # lockfile without errors (since it's already in the lockfile) + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\\"lala\\")")', + 'repo_rule = repository_rule(implementation = _repo_rule_impl)', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other town!")', + ' repo_rule(name= "hello")', + ( + 'lockfile_ext = module_extension(implementation =' + ' _module_ext_impl)' + ), + ], + ) + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertIn('Hello from the other town!', ''.join(stderr)) + + def testUpdateModuleExtensionErrorMode(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'lockfile_ext = use_extension("extension.bzl", "lockfile_ext")', + 'use_repo(lockfile_ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\\"lala\\")")', + 'repo_rule = repository_rule(implementation = _repo_rule_impl)', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other side!")', + ' repo_rule(name= "hello")', + ( + 'lockfile_ext = module_extension(implementation =' + ' _module_ext_impl)' + ), + ], + ) + _, _, stderr = self.RunBazel(['build', '@hello//:all']) + self.assertIn('Hello from the other side!', ''.join(stderr)) + self.RunBazel(['shutdown']) + + # Update extension. + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\\"lalo\\")")', + 'repo_rule = repository_rule(implementation = _repo_rule_impl)', + 'def _module_ext_impl(ctx):', + ' print("Hello from the other town!")', + ' repo_rule(name= "hello")', + ( + 'lockfile_ext = module_extension(implementation =' + ' _module_ext_impl)' + ), + ], + ) + + exit_code, _, stderr = self.RunBazel( + ['build', '--nobuild', '--lockfile_mode=error', '@hello//:all'], + allow_failure=True, + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + ( + 'ERROR: Lock file is no longer up-to-date because: The ' + 'implementation of the extension ' + "'ModuleExtensionId{bzlFileLabel=//:extension.bzl, " + "extensionName=lockfile_ext, isolationKey=Optional.empty}' or one " + 'of its transitive .bzl files has changed' + ), + stderr, + ) + + def testRemoveModuleExtensionsNotUsed(self): + # Test that the module extension is removed from the lockfile if it is not + # used anymore + self.ScratchFile( + 'MODULE.bazel', + [ + 'ext = use_extension("extension.bzl", "ext")', + 'use_repo(ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + 'def _ext_impl(ctx):', + ' repo_rule(name="hello")', + 'ext = module_extension(implementation=_ext_impl)', + ], + ) + + self.RunBazel(['build', '@hello//:all']) + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + self.assertEqual( + list(lockfile['moduleExtensions'].keys()), ['//:extension.bzl%ext'] + ) + + self.ScratchFile('MODULE.bazel', []) + self.RunBazel(['build', '//:all']) + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + self.assertEqual(len(lockfile['moduleExtensions']), 0) + if __name__ == '__main__': unittest.main() From da5494fff49c59a9d7e819706f29eb5d962692b7 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Mon, 10 Jul 2023 19:45:11 +0200 Subject: [PATCH 13/20] fixes --- .../devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java | 2 +- .../build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java | 2 +- .../build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index 2852c88116e9a3..f6d863f7e0cc8c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -222,7 +222,7 @@ static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws Interrup try { moduleExtensionId = ModuleExtensionId.create( - labelConverter.convert(usage.getExtensionBzlFile()), usage.getExtensionName()); + labelConverter.convert(usage.getExtensionBzlFile()), usage.getExtensionName(), usage.getIsolationKey()); } catch (LabelSyntaxException e) { throw ExternalDepsException.withCauseAndMessage( Code.BAD_MODULE, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java index 29a32ef988ec17..6510c3698b9b5a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java @@ -271,7 +271,7 @@ private BzlLoadValue loadBzlFile( throws SingleExtensionEvalFunctionException, InterruptedException { // Check that the .bzl label isn't crazy. try { - BzlLoadFunction.checkValidLoadLabel(bzlFileLabel, starlarkSemantics); + BzlLoadFunction.checkValidLoadLabel(bzlFileLabel, /*fromBuiltinsRepo=*/ false); } catch (LabelSyntaxException e) { throw new SingleExtensionEvalFunctionException( ExternalDepsException.withCauseAndMessage( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index 4efcebc1ece290..5a50de0353be14 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -96,7 +96,7 @@ public void setup() throws Exception { differencer = new SequencedRecordingDifferencer(); registryFactory = new FakeRegistry.Factory(); evaluationContext = - EvaluationContext.newBuilder().setParallelism(8).setEventHandler(reporter).build(); + EvaluationContext.newBuilder().setNumThreads(8).setEventHandler(reporter).build(); AtomicReference packageLocator = new AtomicReference<>( From 667b876bb702c3e966924bfaa1fea520d746e39a Mon Sep 17 00:00:00 2001 From: salma-samy Date: Tue, 11 Jul 2023 00:15:39 +0200 Subject: [PATCH 14/20] Fix tests --- .../build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java index 6510c3698b9b5a..a3ee9af7e88468 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java @@ -127,7 +127,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) // Check that the .bzl file actually exports a module extension by our name. Object exported = bzlLoadValue.getModule().getGlobal(extensionId.getExtensionName()); - if (!(exported instanceof ModuleExtension)) { + if (!(exported instanceof ModuleExtension.InStarlark)) { ImmutableSet exportedExtensions = bzlLoadValue.getModule().getGlobals().entrySet().stream() .filter(e -> e.getValue() instanceof ModuleExtension) @@ -162,7 +162,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) } // Run that extension! - ModuleExtension extension = (ModuleExtension) exported; + ModuleExtension extension = ((ModuleExtension.InStarlark) exported).get(); ImmutableMap generatedRepoSpecs = runModuleExtension( extensionId, extension, usagesValue, bzlLoadValue.getModule(), starlarkSemantics, env); From 7c60e261652db120a762814b5f52856951d17b7f Mon Sep 17 00:00:00 2001 From: salma-samy Date: Thu, 6 Jul 2023 02:50:15 -0700 Subject: [PATCH 15/20] Enable lockfile by default and fix warning PiperOrigin-RevId: 545927412 Change-Id: I34e8531b8e396ccdfe0eecbee22cb68f76f969fc # Conflicts: # src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java # src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java # src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java # src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java # src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java --- .../lib/bazel/BazelRepositoryModule.java | 2 +- .../bazel/bzlmod/LockFileModuleExtension.java | 2 +- .../bazel/repository/RepositoryOptions.java | 2 +- .../RunfilesRepoMappingManifestTest.java | 13 +- .../StarlarkRuleTransitionProviderTest.java | 2 +- .../lib/analysis/util/AnalysisTestCase.java | 4 +- .../bzlmod/BazelDepGraphFunctionTest.java | 3 +- .../bzlmod/BazelLockFileFunctionTest.java | 11 +- .../BazelModuleResolutionFunctionTest.java | 8 +- .../bzlmod/BzlmodRepoRuleFunctionTest.java | 2 +- .../query2/testutil/SkyframeQueryHelper.java | 32 +- .../lib/rules/LabelBuildSettingTest.java | 2 +- .../repository/RepositoryDelegatorTest.java | 3 +- .../StarlarkDocExtractTest.java | 795 ++++++++++++++++++ .../lib/skyframe/BzlLoadFunctionTest.java | 2 +- .../PrepareDepsOfPatternsFunctionTest.java | 2 +- ...isteredExecutionPlatformsFunctionTest.java | 2 +- .../RegisteredToolchainsFunctionTest.java | 2 +- .../RepositoryMappingFunctionTest.java | 2 +- 19 files changed, 846 insertions(+), 45 deletions(-) create mode 100644 src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 1e19a99de35e2d..04b0a36d7a4a84 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -151,7 +151,7 @@ public class BazelRepositoryModule extends BlazeModule { private final AtomicBoolean ignoreDevDeps = new AtomicBoolean(false); private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING; private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR; - private LockfileMode bazelLockfileMode = LockfileMode.OFF; + private LockfileMode bazelLockfileMode = LockfileMode.UPDATE; private List allowedYankedVersions = ImmutableList.of(); private SingleExtensionEvalFunction singleExtensionEvalFunction; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java index af4da1e131ee18..5431d6cf0fabe6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java @@ -28,7 +28,7 @@ @GenerateTypeAdapter public abstract class LockFileModuleExtension implements Postable { - @SuppressWarnings("AutoValueImmutableFields") + @SuppressWarnings("mutable") public abstract byte[] getBzlTransitiveDigest(); public abstract ImmutableMap getGeneratedRepoSpecs(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java index f2cbea78e44119..d11631b4556502 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java @@ -277,8 +277,8 @@ public class RepositoryOptions extends OptionsBase { @Option( name = "lockfile_mode", - defaultValue = "off", // TODO(salmasamy) later will be changed to 'update' converter = LockfileMode.Converter.class, + defaultValue = "update", documentationCategory = OptionDocumentationCategory.BZLMOD, effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS}, help = diff --git a/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java b/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java index edb5955f579b64..e76ccb8e132546 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java @@ -56,9 +56,16 @@ protected ImmutableList extraPrecomputedValues() throws Exception { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF), - PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of())); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE), + PrecomputedValue.injected(BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of())); + } + + @Override + protected SkyframeExecutorRepositoryHelpersHolder getRepositoryHelpersHolder() { + // Transitive packages are needed for RepoMappingManifestAction and are only stored when + // external repositories are enabled. + return SkyframeExecutorRepositoryHelpersHolder.create( + new RepositoryDirectoryDirtinessChecker()); } @Before diff --git a/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java b/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java index ea8cf9f5106196..4876868cb7520d 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java @@ -77,7 +77,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Override diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java index 373668e06a2c62..51b0eef293b7c2 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java @@ -245,7 +245,7 @@ protected void useRuleClassProvider(ConfiguredRuleClassProvider ruleClassProvide BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), PrecomputedValue.injected( - BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF))) + BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE))) .build(ruleClassProvider, fileSystem); useConfiguration(); skyframeExecutor = createSkyframeExecutor(pkgFactory); @@ -294,7 +294,7 @@ private void reinitializeSkyframeExecutor() { PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.WARNING), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF))); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE))); } /** Resets the SkyframeExecutor, as if a clean had been executed. */ diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index dfdab83e416e2a..7b00d9ea308fb3 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -139,7 +139,6 @@ public void setup() throws Exception { PrecomputedValue.STARLARK_SEMANTICS.set( differencer, StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build()); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); @@ -147,7 +146,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.OFF); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index 5a50de0353be14..9461583a3d57f1 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -208,7 +208,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) differencer, BazelCompatibilityMode.ERROR); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.set(differencer, ImmutableMap.of()); RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING.set( differencer, RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY); @@ -285,10 +285,9 @@ public void moduleWithFlags() throws Exception { ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of("my_dep_1", override)); BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, yankedVersions); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( - differencer, CheckDirectDepsMode.ERROR); + differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( - differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + differencer, BazelCompatibilityMode.WARNING); UpdateLockFileKey key = UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); @@ -310,9 +309,9 @@ public void moduleWithFlags() throws Exception { assertThat(value.getFlags().cmdModuleOverrides()).isEqualTo(moduleOverride); assertThat(value.getFlags().allowedYankedVersions()).isEqualTo(yankedVersions); assertThat(value.getFlags().directDependenciesMode()) - .isEqualTo(CheckDirectDepsMode.ERROR.toString()); + .isEqualTo(CheckDirectDepsMode.WARNING.toString()); assertThat(value.getFlags().compatibilityMode()) - .isEqualTo(BazelCompatibilityMode.ERROR.toString()); + .isEqualTo(BazelCompatibilityMode.WARNING.toString()); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 427450dd5cd237..8186818cd1f50d 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -130,7 +130,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.OFF); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } @@ -291,8 +291,7 @@ public void testYankedVersionCheckIgnoredByAll() throws Exception { @Test public void testYankedVersionCheckIgnoredBySpecific() throws Exception { setupModulesForYankedVersion(); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set( - differencer, ImmutableList.of("b@1.0")); + BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of("b@1.0")); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); assertThat(result.hasError()).isFalse(); @@ -301,8 +300,7 @@ public void testYankedVersionCheckIgnoredBySpecific() throws Exception { @Test public void testBadYankedVersionFormat() throws Exception { setupModulesForYankedVersion(); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set( - differencer, ImmutableList.of("b~1.0")); + BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of("b~1.0")); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); assertThat(result.hasError()).isTrue(); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index d1a4ef5b5037c2..92ff555778779d 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -145,7 +145,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java index 6e705ff8b87833..ef99119130ef3a 100644 --- a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java +++ b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java @@ -23,7 +23,6 @@ import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ServerDirectories; -import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; @@ -36,9 +35,8 @@ import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.cmdline.RepositoryName; -import com.google.devtools.build.lib.packages.ConstantRuleVisibility; import com.google.devtools.build.lib.packages.PackageFactory; -import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension; +import com.google.devtools.build.lib.packages.RuleVisibility; import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.packages.util.MockToolsConfig; @@ -59,9 +57,9 @@ import com.google.devtools.build.lib.query2.engine.QueryUtil.AggregateAllOutputFormatterCallback; import com.google.devtools.build.lib.query2.engine.ThreadSafeOutputFormatterCallback; import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; +import com.google.devtools.build.lib.runtime.QuiescingExecutorsImpl; import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; import com.google.devtools.build.lib.skyframe.IgnoredPackagePrefixesFunction; -import com.google.devtools.build.lib.skyframe.PackageValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.SkyframeExecutor; import com.google.devtools.build.lib.skyframe.SkyframeTargetPatternEvaluator; @@ -149,6 +147,11 @@ public void setUp() throws Exception { this.queryEnvironmentFactory = makeQueryEnvironmentFactory(); } + @Override + public final void cleanUp() { + skyframeExecutor.getEvaluator().cleanupInterningPools(); + } + protected abstract String getRootDirectoryNameForSetup(); protected abstract void performAdditionalClientSetup(MockToolsConfig mockToolsConfig) @@ -312,10 +315,13 @@ protected boolean enableBzlmod() { protected void initTargetPatternEvaluator(ConfiguredRuleClassProvider ruleClassProvider) { this.toolsRepository = ruleClassProvider.getToolsRepository(); + if (skyframeExecutor != null) { + cleanUp(); + } skyframeExecutor = createSkyframeExecutor(ruleClassProvider); PackageOptions packageOptions = Options.getDefaults(PackageOptions.class); - packageOptions.defaultVisibility = ConstantRuleVisibility.PRIVATE; + packageOptions.defaultVisibility = RuleVisibility.PRIVATE; packageOptions.showLoadingProgress = true; packageOptions.globbingThreads = 7; packageOptions.packagePath = ImmutableList.of(rootDirectory.getPathString()); @@ -334,6 +340,7 @@ protected void initTargetPatternEvaluator(ConfiguredRuleClassProvider ruleClassP ImmutableMap.of(), ImmutableMap.of(), new TimestampGranularityMonitor(BlazeClock.instance()), + QuiescingExecutorsImpl.forTesting(), FakeOptions.builder().put(packageOptions).put(buildLanguageOptions).build()); } catch (InterruptedException | AbruptExitException e) { throw new IllegalStateException(e); @@ -372,8 +379,7 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), PrecomputedValue.injected( - BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF))) - .setEnvironmentExtensions(getEnvironmentExtensions()) + BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE))) .build(ruleClassProvider, fileSystem); SkyframeExecutor skyframeExecutor = BazelSkyframeExecutorConstants.newBazelSkyframeExecutorBuilder() @@ -384,7 +390,7 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru .setIgnoredPackagePrefixesFunction( new IgnoredPackagePrefixesFunction(ignoredPackagePrefixesFile)) .setExtraSkyFunctions(analysisMock.getSkyFunctions(directories)) - .setPerCommandSyscallCache(delegatingSyscallCache) + .setSyscallCache(delegatingSyscallCache) .build(); skyframeExecutor.injectExtraPrecomputedValues( ImmutableList.builder() @@ -415,21 +421,17 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR)) - .add(PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)) + .add( + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)) .build()); SkyframeExecutorTestHelper.process(skyframeExecutor); return skyframeExecutor; } - protected abstract Iterable getEnvironmentExtensions(); - - protected abstract BuildOptions getDefaultBuildOptions( - ConfiguredRuleClassProvider ruleClassProvider); - @Override public void assertPackageNotLoaded(String packageName) throws Exception { MemoizingEvaluator evaluator = skyframeExecutor.getEvaluator(); - SkyKey key = PackageValue.key(PackageIdentifier.createInMainRepo(packageName)); + SkyKey key = PackageIdentifier.createInMainRepo(packageName); if (evaluator.getExistingValue(key) != null || evaluator.getExistingErrorForTesting(key) != null) { throw new IllegalStateException("Package was loaded: " + packageName); diff --git a/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java b/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java index 39e7aeafc023d7..7ce085431e5bb5 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java @@ -60,7 +60,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } private void writeRulesBzl(String type) throws Exception { diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java index 94615cb8fd98a4..6ca0e6fff59be6 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -239,6 +239,7 @@ public void setupDelegator() throws Exception { SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootPath, ImmutableMap.of())) .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, @@ -270,7 +271,7 @@ public void setupDelegator() throws Exception { differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java b/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java new file mode 100644 index 00000000000000..95f3023a417a6f --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java @@ -0,0 +1,795 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.starlarkdocextract; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.actions.BinaryFileWriteAction; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; +import com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil; +import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.PrecomputedValue.Injected; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.AspectInfo; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.AttributeInfo; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.AttributeType; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.FunctionParamInfo; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.ModuleInfo; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.OriginKey; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.ProviderInfo; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.ProviderNameGroup; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.RuleInfo; +import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.StarlarkFunctionInfo; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.TextFormat; +import java.io.IOException; +import java.util.NoSuchElementException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public final class StarlarkDocExtractTest extends BuildViewTestCase { + private Path moduleRoot; // initialized by extraPrecomputedValues + private FakeRegistry registry; + + @Override + protected ImmutableList extraPrecomputedValues() { + // TODO(b/285924565): support --enable_bzlmod in BuildViewTestCase tests without needing the + // boilerplate below. + try { + moduleRoot = scratch.dir("modules"); + } catch (IOException e) { + throw new IllegalStateException(e); + } + registry = FakeRegistry.DEFAULT_FACTORY.newFakeRegistry(moduleRoot.getPathString()); + return ImmutableList.of( + PrecomputedValue.injected( + ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), + PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), + PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()), + PrecomputedValue.injected(BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + PrecomputedValue.injected( + BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), + PrecomputedValue.injected( + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); + } + + private static ModuleInfo protoFromBinaryFileWriteAction(Action action) throws Exception { + assertThat(action).isInstanceOf(BinaryFileWriteAction.class); + return ModuleInfo.parseFrom( + ((BinaryFileWriteAction) action).getSource().openStream(), + ExtensionRegistry.getEmptyRegistry()); + } + + private static ModuleInfo protoFromTextFileWriteAction(Action action) throws Exception { + assertThat(action).isInstanceOf(FileWriteAction.class); + return TextFormat.parse( + ((FileWriteAction) action).getFileContents(), + ExtensionRegistry.getEmptyRegistry(), + ModuleInfo.class); + } + + private ModuleInfo protoFromConfiguredTarget(String targetName) throws Exception { + ConfiguredTarget target = getConfiguredTarget(targetName); + return protoFromBinaryFileWriteAction( + getGeneratingAction( + target, + Label.parseCanonicalUnchecked(targetName).toPathFragment().getPathString() + + ".binaryproto")); + } + + @Before + public void setUpBzlLibrary() throws Exception { + // TODO(https://github.com/bazelbuild/bazel/issues/18599): get rid of this when we bundle + // bzl_library with Bazel. + scratch.file( + "bzl_library.bzl", + "def _bzl_library_impl(ctx):", + " deps_files = [x.files for x in ctx.attr.deps]", + " all_files = depset(ctx.files.srcs, order = 'postorder', transitive = deps_files)", + " return DefaultInfo(files = all_files)", + "", + "bzl_library = rule(", + " implementation = _bzl_library_impl,", + " attrs = {", + " 'srcs': attr.label_list(allow_files = ['.bzl', '.scl']),", + " 'deps': attr.label_list(),", + " }", + ")"); + } + + @Test + public void basicFunctionality() throws Exception { + scratch.file( + "foo.bzl", // + "'''Module doc string'''", + "True"); + scratch.file( + "BUILD", // + "starlark_doc_extract(", + " name = 'extract',", + " src = 'foo.bzl',", + ")"); + ConfiguredTarget target = getConfiguredTarget("//:extract"); + ModuleInfo moduleInfo = + protoFromBinaryFileWriteAction(getGeneratingAction(target, "extract.binaryproto")); + assertThat(moduleInfo.getModuleDocstring()).isEqualTo("Module doc string"); + assertThat(moduleInfo.getFile()).isEqualTo("//:foo.bzl"); + } + + @Test + public void textprotoOut() throws Exception { + scratch.file( + "foo.bzl", // + "'''Module doc string'''", + "True"); + scratch.file( + "BUILD", // + "starlark_doc_extract(", + " name = 'extract',", + " src = 'foo.bzl',", + ")"); + ConfiguredTarget ruleTarget = getConfiguredTarget("//:extract"); + // Verify that we do not generate textproto output unless explicitly requested. + assertThrows( + NoSuchElementException.class, () -> getGeneratingAction(ruleTarget, "extract.textproto")); + + ConfiguredTarget textprotoOutputTarget = getConfiguredTarget("//:extract.textproto"); + ModuleInfo moduleInfo = + protoFromTextFileWriteAction( + getGeneratingAction(textprotoOutputTarget, "extract.textproto")); + assertThat(moduleInfo.getModuleDocstring()).isEqualTo("Module doc string"); + } + + @Test + public void sclDialect() throws Exception { + setBuildLanguageOptions("--experimental_enable_scl_dialect"); + scratch.file( + "foo.scl", // + "def f():", + " '''This is my function'''", + " pass"); + scratch.file( + "bar.scl", // + "'''My scl module string'''", + "load('//:foo.scl', 'f')", + "bar_f = f"); + scratch.file( + "BUILD", // + "load('bzl_library.bzl', 'bzl_library')", + "bzl_library(", + " name = 'foo_scl',", + " srcs = ['foo.scl'],", + ")", + "starlark_doc_extract(", + " name = 'extract',", + " src = 'bar.scl',", + " deps = ['foo_scl'],", + ")"); + ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract"); + assertThat(moduleInfo.getModuleDocstring()).isEqualTo("My scl module string"); + assertThat(moduleInfo.getFile()).isEqualTo("//:bar.scl"); + assertThat(moduleInfo.getFuncInfo(0).getDocString()).isEqualTo("This is my function"); + } + + @Test + public void sourceWithSyntaxError_fails() throws Exception { + scratch.file( + "error.bzl", // + "!!!"); + scratch.file( + "error_loader.bzl", // + "'''This is my module'''", + "load('error.bzl', 'x')"); + scratch.file( + "BUILD", // + "load('bzl_library.bzl', 'bzl_library')", + "bzl_library(", + " name = 'error_bzl',", + " srcs = ['error.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'error_doc',", + " src = 'error.bzl',", + ")", + "starlark_doc_extract(", + " name = 'error_loader_doc',", + " src = 'error_loader.bzl',", + " deps = ['error_bzl'],", + ")"); + + AssertionError errorDocFailure = + assertThrows(AssertionError.class, () -> getConfiguredTarget("//:error_doc")); + assertThat(errorDocFailure).hasMessageThat().contains("invalid character: '!'"); + + AssertionError errorLoaderDocFailure = + assertThrows(AssertionError.class, () -> getConfiguredTarget("//:error_loader_doc")); + assertThat(errorLoaderDocFailure).hasMessageThat().contains("invalid character: '!'"); + } + + @Test + public void symbolNames() throws Exception { + scratch.file( + "foo.bzl", // + "def func1():", + " pass", + "def func2():", + " pass", + "def _hidden():", + " pass"); + scratch.file( + "BUILD", // + "starlark_doc_extract(", + " name = 'extract_some',", + " src = 'foo.bzl',", + " symbol_names = ['func1'],", + ")", + "starlark_doc_extract(", + " name = 'extract_all',", + " src = 'foo.bzl',", + ")"); + + ModuleInfo dumpSome = protoFromConfiguredTarget("//:extract_some"); + assertThat(dumpSome.getFuncInfoList().stream().map(StarlarkFunctionInfo::getFunctionName)) + .containsExactly("func1"); + + ModuleInfo dumpAll = protoFromConfiguredTarget("//:extract_all"); + assertThat(dumpAll.getFuncInfoList().stream().map(StarlarkFunctionInfo::getFunctionName)) + .containsExactly("func1", "func2"); + } + + @Test + public void originKey() throws Exception { + scratch.file( + "origin.bzl", // + "def my_macro():", + " pass", + "MyInfo = provider()", + "MyOtherInfo = provider()", + "my_rule = rule(", + " implementation = lambda ctx: None,", + " attrs = {'a': attr.label(providers = [MyInfo, MyOtherInfo])},", + " provides = [MyInfo, MyOtherInfo],", + ")", + "my_aspect = aspect(implementation = lambda target, ctx: None)"); + scratch.file( + "renamer.bzl", // + "load(':origin.bzl', 'my_macro', 'MyInfo', 'MyOtherInfo', 'my_rule', 'my_aspect')", + "namespace = struct(", + " renamed_macro = my_macro,", + " RenamedInfo = MyInfo,", + " renamed_rule = my_rule,", + " renamed_aspect = my_aspect,", + ")", + "other_namespace = struct(", + " RenamedOtherInfo = MyOtherInfo,", + ")"); + scratch.file( + "BUILD", // + "load('bzl_library.bzl', 'bzl_library')", + "bzl_library(", + " name = 'origin_bzl',", + " srcs = ['origin.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'extract_renamed',", + " src = 'renamer.bzl',", + " deps = ['origin_bzl'],", + " symbol_names = ['namespace'],", + ")"); + + ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract_renamed"); + + assertThat(moduleInfo.getFuncInfoList().stream().map(StarlarkFunctionInfo::getFunctionName)) + .containsExactly("namespace.renamed_macro"); + assertThat(moduleInfo.getFuncInfoList().stream().map(StarlarkFunctionInfo::getOriginKey)) + .containsExactly( + OriginKey.newBuilder().setName("my_macro").setFile("//:origin.bzl").build()); + + assertThat(moduleInfo.getProviderInfoList().stream().map(ProviderInfo::getProviderName)) + .containsExactly("namespace.RenamedInfo"); + assertThat(moduleInfo.getProviderInfoList().stream().map(ProviderInfo::getOriginKey)) + .containsExactly(OriginKey.newBuilder().setName("MyInfo").setFile("//:origin.bzl").build()); + + assertThat(moduleInfo.getRuleInfoList().stream().map(RuleInfo::getRuleName)) + .containsExactly("namespace.renamed_rule"); + assertThat(moduleInfo.getRuleInfoList().stream().map(RuleInfo::getOriginKey)) + .containsExactly( + OriginKey.newBuilder().setName("my_rule").setFile("//:origin.bzl").build()); + + assertThat(moduleInfo.getRuleInfo(0).getAttributeList()) + .containsExactly( + ModuleInfoExtractor.IMPLICIT_NAME_ATTRIBUTE_INFO, + AttributeInfo.newBuilder() + .setName("a") + .setType(AttributeType.LABEL) + .setDefaultValue("None") + .addProviderNameGroup( + ProviderNameGroup.newBuilder() + .addProviderName("namespace.RenamedInfo") + .addProviderName("other_namespace.RenamedOtherInfo") + .addOriginKey( + OriginKey.newBuilder().setName("MyInfo").setFile("//:origin.bzl")) + .addOriginKey( + OriginKey.newBuilder().setName("MyOtherInfo").setFile("//:origin.bzl"))) + .build()); + assertThat(moduleInfo.getRuleInfo(0).getAdvertisedProviders()) + .isEqualTo( + ProviderNameGroup.newBuilder() + .addProviderName("namespace.RenamedInfo") + .addProviderName("other_namespace.RenamedOtherInfo") + .addOriginKey(OriginKey.newBuilder().setName("MyInfo").setFile("//:origin.bzl")) + .addOriginKey( + OriginKey.newBuilder().setName("MyOtherInfo").setFile("//:origin.bzl")) + .build()); + + assertThat(moduleInfo.getAspectInfoList().stream().map(AspectInfo::getAspectName)) + .containsExactly("namespace.renamed_aspect"); + assertThat(moduleInfo.getAspectInfoList().stream().map(AspectInfo::getOriginKey)) + .containsExactly( + OriginKey.newBuilder().setName("my_aspect").setFile("//:origin.bzl").build()); + } + + @Test + public void originKeyFileAndModuleInfoFileLabels_forBzlFileInBzlmodModule_areDisplayForm() + throws Exception { + setBuildLanguageOptions("--enable_bzlmod"); + scratch.overwriteFile("MODULE.bazel", "bazel_dep(name='origin_repo', version='0.1')"); + registry.addModule( + BzlmodTestUtil.createModuleKey("origin_repo", "0.1"), + "module(name='origin_repo', version='0.1')"); + Path originRepoPath = moduleRoot.getRelative("origin_repo~0.1"); + scratch.file(originRepoPath.getRelative("WORKSPACE").getPathString()); + scratch.file( + originRepoPath.getRelative("BUILD").getPathString(), // + "exports_files(['origin.bzl'])"); + scratch.file( + originRepoPath.getRelative("origin.bzl").getPathString(), // + "def my_macro():", + " pass", + "MyInfo = provider()", + "my_rule = rule(", + " implementation = lambda ctx: None,", + " attrs = {'a': attr.label(providers = [MyInfo])},", + " provides = [MyInfo],", + ")", + "my_aspect = aspect(implementation = lambda target, ctx: None)"); + scratch.file( + "renamer.bzl", // + "load('@origin_repo//:origin.bzl', 'my_macro', 'MyInfo', 'my_rule', 'my_aspect')", + "namespace = struct(", + " renamed_macro = my_macro,", + " RenamedInfo = MyInfo,", + " renamed_rule = my_rule,", + " renamed_aspect = my_aspect,", + ")"); + scratch.file( + "BUILD", // + "load('bzl_library.bzl', 'bzl_library')", + "bzl_library(", + " name = 'origin_bzl',", + " srcs = ['@origin_repo//:origin.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'extract_origin',", + " src = '@origin_repo//:origin.bzl',", + ")", + "starlark_doc_extract(", + " name = 'extract_renamed',", + " src = 'renamer.bzl',", + " deps = ['origin_bzl'],", + ")"); + + // verify that ModuleInfo.name for a .bzl file in another bzlmod module is in display form, i.e. + // "@origin_repo//:origin.bzl" as opposed to "@@origin_repo~0.1//:origin.bzl" + ModuleInfo originModuleInfo = protoFromConfiguredTarget("//:extract_origin"); + assertThat(originModuleInfo.getFile()).isEqualTo("@origin_repo//:origin.bzl"); + + // verify that OriginKey.name for entities defined in a .bzl file in another bzlmod module is in + // display form, i.e. "@origin_repo//:origin.bzl" as opposed to "@@origin_repo~0.1//:origin.bzl" + ModuleInfo renamedModuleInfo = protoFromConfiguredTarget("//:extract_renamed"); + assertThat(renamedModuleInfo.getFile()).isEqualTo("//:renamer.bzl"); + assertThat(renamedModuleInfo.getFuncInfo(0).getOriginKey().getFile()) + .isEqualTo("@origin_repo//:origin.bzl"); + assertThat(renamedModuleInfo.getProviderInfo(0).getOriginKey().getFile()) + .isEqualTo("@origin_repo//:origin.bzl"); + assertThat(renamedModuleInfo.getAspectInfo(0).getOriginKey().getFile()) + .isEqualTo("@origin_repo//:origin.bzl"); + assertThat(renamedModuleInfo.getRuleInfo(0).getOriginKey().getFile()) + .isEqualTo("@origin_repo//:origin.bzl"); + assertThat( + renamedModuleInfo + .getRuleInfo(0) + .getAttribute(1) // 0 is the implicit name attribute + .getProviderNameGroup(0) + .getOriginKey(0) + .getFile()) + .isEqualTo("@origin_repo//:origin.bzl"); + assertThat(renamedModuleInfo.getRuleInfo(0).getAdvertisedProviders().getOriginKey(0).getFile()) + .isEqualTo("@origin_repo//:origin.bzl"); + } + + @Test + public void exportNestedFunctionsAndLambdas() throws Exception { + scratch.file( + "origin.bzl", // + "def return_nested():", + " def nested(x):", + " '''My nested function'''", + " pass", + " return nested", + "", + "def return_lambda():", + " return lambda y: y"); + scratch.file( + "exporter.bzl", // + "load(':origin.bzl', 'return_nested', 'return_lambda')", + "exported_nested = return_nested()", + "exported_lambda = return_lambda()"); + scratch.file( + "BUILD", // + "load('bzl_library.bzl', 'bzl_library')", + "bzl_library(", + " name = 'origin_bzl',", + " srcs = ['origin.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'extract_exporter',", + " src = 'exporter.bzl',", + " deps = ['origin_bzl'],", + ")"); + + ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract_exporter"); + + assertThat(moduleInfo.getFuncInfoList()) + .containsExactly( + StarlarkFunctionInfo.newBuilder() + .setFunctionName("exported_nested") + .setDocString("My nested function") + .addParameter(FunctionParamInfo.newBuilder().setName("x").setMandatory(true)) + .setOriginKey( + // OriginKey.name for nested functions is explicitly unset + OriginKey.newBuilder().setFile("//:origin.bzl")) + .build(), + StarlarkFunctionInfo.newBuilder() + .setFunctionName("exported_lambda") + .addParameter(FunctionParamInfo.newBuilder().setName("y").setMandatory(true)) + .setOriginKey( + // OriginKey.name for lambdas is explicitly unset + OriginKey.newBuilder().setFile("//:origin.bzl")) + .build()); + } + + @Test + public void missingBzlLibraryDeps_fails() throws Exception { + scratch.file( + "dep.bzl", // + "load('//:forgotten_dep_of_dep.bzl', 'g')", + "def f(): pass"); + scratch.file( + "forgotten_dep_of_dep.bzl", // + "def g(): pass"); + scratch.file( + "forgotten_dep.bzl", // + "load('//:forgotten_dep_of_forgotten_dep.bzl', 'j')", + "def h(): pass"); + scratch.file( + "forgotten_dep2.bzl", // + "def i(): pass"); + scratch.file( + "forgotten_dep_of_forgotten_dep.bzl", // + "def j(): pass"); + scratch.file( + "foo.bzl", // + "load('//:dep.bzl', 'f')", + "load('//:forgotten_dep.bzl', 'h')", + "load('//:forgotten_dep2.bzl', 'i')"); + scratch.file( + "BUILD", // + "load('bzl_library.bzl', 'bzl_library')", + "bzl_library(", + " name = 'dep_bzl',", + " srcs = ['dep.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'extract',", + " src = 'foo.bzl',", // Note that src does not need to be part of deps + " deps = ['dep_bzl']", + ")"); + + AssertionError e = assertThrows(AssertionError.class, () -> getConfiguredTarget("//:extract")); + assertThat(e) + .hasMessageThat() + .contains( + "missing bzl_library targets for Starlark module(s) //:forgotten_dep_of_dep.bzl," + + " //:forgotten_dep.bzl, //:forgotten_dep2.bzl"); + // We do not want to log transitive deps of already missing deps in the error message - it would + // be hard to read and unnecessary, since a valid bzl_library target should bring in its + // transitive deps. + assertThat(e).hasMessageThat().doesNotContain("forgotten_dep_of_forgotten_dep.bzl"); + } + + @Test + public void depsWithDerivedFiles_onUnknownLoads_failsAndPrintsDerivedFiles() throws Exception { + scratch.file("BUILD"); + scratch.file( + "pkg/source_file_masked_by_rule_name.bzl", // + "def f(): pass"); + scratch.file( + "pkg/source_file_masked_by_rule_output_name.bzl", // + "def g(): pass"); + scratch.file( + "pkg/foo.bzl", // + "load('//pkg:source_file_masked_by_rule_name.bzl', 'f')", + "load('//pkg:source_file_masked_by_rule_output_name.bzl', 'g')"); + scratch.file( + "pkg/BUILD", // + "load('//:bzl_library.bzl', 'bzl_library')", + "genrule(", + " name = 'source_file_masked_by_rule_name.bzl',", + " outs = ['some_output.bzl'],", + " cmd = 'touch $@'", + ")", + "genrule(", + " name = 'source_file_masked_by_rule_output_name_bzl_generator',", + " outs = ['source_file_masked_by_rule_output_name.bzl'],", + " cmd = 'touch $@'", + ")", + "genrule(", + " name = 'some_rule',", + " outs = ['ordinary_generated_file.bzl'],", + " cmd = 'touch $@'", + ")", + "bzl_library(", + " name = 'deps_bzl',", + " srcs = [", + " 'source_file_masked_by_rule_name.bzl',", + " 'source_file_masked_by_rule_output_name.bzl',", + " 'ordinary_generated_file.bzl',", + " ],", + ")", + "starlark_doc_extract(", + " name = 'extract',", + " src = 'foo.bzl',", + " deps = ['deps_bzl']", + ")"); + + AssertionError error = + assertThrows(AssertionError.class, () -> getConfiguredTarget("//pkg:extract")); + assertThat(error) + .hasMessageThat() + .contains( + "missing bzl_library targets for Starlark module(s)" + + " //pkg:source_file_masked_by_rule_name.bzl," + + " //pkg:source_file_masked_by_rule_output_name.bzl\n" + + "Note the following are generated file(s) and cannot be loaded in Starlark:" + + " pkg/some_output.bzl (generated by rule" + + " //pkg:source_file_masked_by_rule_name.bzl)," + + " pkg/source_file_masked_by_rule_output_name.bzl (generated by rule" + + " //pkg:source_file_masked_by_rule_output_name_bzl_generator)," + + " pkg/ordinary_generated_file.bzl (generated by rule //pkg:some_rule)"); + } + + @Test + public void depsWithDerivedFiles_onNoUnknownLoads_succeeds() throws Exception { + scratch.file("BUILD"); + scratch.file( + "util.bzl", + "def _impl(ctx):", + " out = ctx.actions.declare_file(ctx.attr.out)", + " ctx.actions.run_shell(command = 'touch $1', arguments = [out.path], outputs = [out])", + " return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out]))", + "generate_out_without_declaring_it_as_a_target = rule(", + " attrs = {'out': attr.string()},", + " implementation = _impl,", + ")"); + scratch.file( + "pkg/source_dep.bzl", // + "def f(): pass"); + scratch.file( + "pkg/foo.bzl", // + "load('//pkg:source_dep.bzl', 'f')"); + scratch.file( + "pkg/BUILD", // + "load('//:bzl_library.bzl', 'bzl_library')", + "load('//:util.bzl', 'generate_out_without_declaring_it_as_a_target')", + "genrule(", + " name = 'some_rule',", + " outs = ['declared_derived_dep.bzl'],", + " cmd = 'touch $@'", + ")", + // //pkg:generate_source_dep_without_declaring_it_as_a_target masks the source_dep.bzl + // source artifact with a non-target, derived artifact having the same root-relative path. + "generate_out_without_declaring_it_as_a_target(", + " name = 'generate_source_dep_without_declaring_it_as_a_target',", + " out = 'source_dep.bzl'", + ")", + "bzl_library(", + " name = 'deps_bzl',", + " srcs = [", + " 'declared_derived_dep.bzl',", + " 'source_dep.bzl',", + " 'generate_source_dep_without_declaring_it_as_a_target',", + " ],", + ")", + "starlark_doc_extract(", + " name = 'extract',", + " src = 'foo.bzl',", + " deps = ['deps_bzl']", + ")"); + + getConfiguredTarget("//pkg:extract"); + assertNoEvents(); + } + + @Test + public void srcDerivedFile_fails() throws Exception { + scratch.file("BUILD"); + scratch.file("pkg/source_file_masked_by_rule_name.bzl"); + scratch.file("pkg/source_file_masked_by_rule_output_name.bzl"); + scratch.file( + "pkg/BUILD", // + "genrule(", + " name = 'source_file_masked_by_rule_name.bzl',", + " outs = ['some_output.bzl'],", + " cmd = 'touch $@'", + ")", + "genrule(", + " name = 'source_file_masked_by_rule_output_name_bzl_generator',", + " outs = ['source_file_masked_by_rule_output_name.bzl'],", + " cmd = 'touch $@'", + ")", + "genrule(", + " name = 'some_rule',", + " outs = ['ordinary_generated_file.bzl'],", + " cmd = 'touch $@'", + ")", + "starlark_doc_extract(", + " name = 'source_file_masked_by_rule_name_doc',", + " src = 'source_file_masked_by_rule_name.bzl',", + ")", + "starlark_doc_extract(", + " name = 'source_file_masked_by_rule_output_name_doc',", + " src = 'source_file_masked_by_rule_output_name.bzl',", + ")", + "starlark_doc_extract(", + " name = 'ordinary_generated_file_doc',", + " src = 'ordinary_generated_file.bzl',", + ")"); + + AssertionError maskedByRuleError = + assertThrows( + AssertionError.class, + () -> getConfiguredTarget("//pkg:source_file_masked_by_rule_name_doc")); + assertThat(maskedByRuleError) + .hasMessageThat() + .contains( + "pkg/some_output.bzl (generated by rule //pkg:source_file_masked_by_rule_name.bzl)" + + " is not a source file and cannot be loaded in Starlark"); + + AssertionError maskedByRuleOutputError = + assertThrows( + AssertionError.class, + () -> getConfiguredTarget("//pkg:source_file_masked_by_rule_output_name_doc")); + assertThat(maskedByRuleOutputError) + .hasMessageThat() + .contains( + "pkg/source_file_masked_by_rule_output_name.bzl (generated by rule" + + " //pkg:source_file_masked_by_rule_output_name_bzl_generator) is not a source" + + " file and cannot be loaded in Starlark"); + + AssertionError ordinaryGeneratedFileError = + assertThrows( + AssertionError.class, () -> getConfiguredTarget("//pkg:ordinary_generated_file_doc")); + assertThat(ordinaryGeneratedFileError) + .hasMessageThat() + .contains( + "pkg/ordinary_generated_file.bzl (generated by rule //pkg:some_rule) is not a source" + + " file and cannot be loaded in Starlark"); + } + + @Test + public void srcAlias_resolvesToActual() throws Exception { + scratch.file("alias_name.bzl"); + scratch.file("alias_actual.bzl"); + scratch.file( + "BUILD", // + "alias(", + " name = 'alias_name.bzl',", + " actual = 'alias_actual.bzl',", + ")", + "starlark_doc_extract(", + " name = 'extract',", + " src = 'alias_name.bzl',", + ")"); + + ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract"); + assertThat(moduleInfo.getFile()).isEqualTo("//:alias_actual.bzl"); + } + + @Test + public void srcFilegroup_resolvesToFilegroupSrc() throws Exception { + scratch.file("masked_by_filegroup_name.bzl"); + scratch.file("filegroup_src_actual.bzl"); + scratch.file( + "BUILD", // + "filegroup(", + " name = 'masked_by_filegroup_name.bzl',", + " srcs = ['filegroup_src_actual.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'extract',", + " src = 'masked_by_filegroup_name.bzl',", + ")"); + + ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract"); + assertThat(moduleInfo.getFile()).isEqualTo("//:filegroup_src_actual.bzl"); + } + + @Test + public void srcFilegroup_mustHaveSingleSrc() throws Exception { + scratch.file("foo.bzl"); + scratch.file("bar.bzl"); + scratch.file( + "BUILD", // + "filegroup(", + " name = 'no_files',", + " srcs = [],", + ")", + "filegroup(", + " name = 'two_files',", + " srcs = ['foo.bzl', 'bar.bzl'],", + ")", + "starlark_doc_extract(", + " name = 'no_files_doc',", + " src = 'no_files',", + ")", + "starlark_doc_extract(", + " name = 'two_files_doc',", + " src = 'two_files',", + ")"); + + AssertionError extractNoFilesError = + assertThrows(AssertionError.class, () -> getConfiguredTarget("//:no_files_doc")); + assertThat(extractNoFilesError) + .hasMessageThat() + .contains("'//:no_files' must produce a single file"); + + AssertionError extractTwoFilesError = + assertThrows(AssertionError.class, () -> getConfiguredTarget("//:two_files_doc")); + assertThat(extractTwoFilesError) + .hasMessageThat() + .contains("'//:two_files' must produce a single file"); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java index 61230364befbd8..2b98361118813b 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java @@ -113,7 +113,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Before diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java index 2d00ff7c3ed816..aaf6bd05afee16 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java @@ -247,7 +247,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } // Helpers: diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java index cda1434744fe9c..5097e5e24982c2 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java @@ -113,7 +113,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java index 2374e2c62126cb..b7b54d9e88bd5c 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java @@ -69,7 +69,7 @@ protected ImmutableList extraPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java index 06672b18ddc308..b7dcd438e337e1 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java @@ -88,7 +88,7 @@ protected ImmutableList extraPrecomputedValues() thro BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); } @Override From 2652ee6f4b88edef4813d10e19b171fc0e315e94 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Thu, 6 Jul 2023 13:59:47 -0700 Subject: [PATCH 16/20] Add lockfile documentation PiperOrigin-RevId: 546085523 Change-Id: If287e6e143f8858d185f83a1630275520db65e44 # Conflicts: # site/en/_book.yaml --- site/en/_book.yaml | 15 +++ site/en/external/lockfile.md | 191 +++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 site/en/external/lockfile.md diff --git a/site/en/_book.yaml b/site/en/_book.yaml index 1b3e1d53268e02..19bdc921b9d8aa 100644 --- a/site/en/_book.yaml +++ b/site/en/_book.yaml @@ -125,6 +125,21 @@ upper_tabs: path: /run/scripts - title: Client/server implementation path: /run/client-server + - heading: External dependencies + - title: Overview + path: /external/overview + - title: Bazel modules + path: /external/module + - title: Bazel registries + path: /external/registry + - title: Module extensions + path: /external/extension + - title: Module lockfile + path: /external/lockfile + - title: Bzlmod migration guide + path: /external/migration + - title: Advanced topics + path: /external/advanced - heading: Querying your build - title: Query quickstart path: /query/quickstart diff --git a/site/en/external/lockfile.md b/site/en/external/lockfile.md new file mode 100644 index 00000000000000..39ed0e8e680a2a --- /dev/null +++ b/site/en/external/lockfile.md @@ -0,0 +1,191 @@ +Project: /_project.yaml +Book: /_book.yaml +keywords: product:Bazel,lockfile,Bzlmod + +# Bazel Lockfile + +{% include "_buttons.html" %} + +The lockfile feature in Bazel enables the recording of specific versions or +dependencies of software libraries or packages required by a project. It +achieves this by storing the result of module resolution and extension +evaluation. The lockfile promotes reproducible builds, ensuring consistent +development environments. Additionally, it enhances build efficiency by allowing +Bazel to skip the resolution process when there are no changes in project +dependencies. Furthermore, the lockfile improves stability by preventing +unexpected updates or breaking changes in external libraries, thereby reducing +the risk of introducing bugs. + +## Lockfile Generation {:#lockfile-generation} + +The lockfile is generated under the workspace root with the name +`MODULE.bazel.lock`. It is created or updated during the build process, +specifically after module resolution and extension evaluation. The lockfile +captures the current state of the project, including the MODULE file, flags, +overrides, and other relevant information. Importantly, it only includes +dependencies that are included in the current invocation of the build. + +When changes occur in the project that affect its dependencies, the lockfile is +automatically updated to reflect the new state. This ensures that the lockfile +remains focused on the specific set of dependencies required for the current +build, providing an accurate representation of the project's resolved +dependencies. + +## Lockfile Usage {:#lockfile-usage} + +The lockfile can be controlled by the flag +[`--lockfile_mode`](/reference/command-line-reference#flag--lockfile_mode) to +customize the behavior of Bazel when the project state differs from the +lockfile. The available modes are: + +* `update` (Default): If the project state matches the lockfile, the + resolution result is immediately returned from the lockfile. Otherwise, + resolution is executed, and the lockfile is updated to reflect the current + state. +* `error`: If the project state matches the lockfile, the resolution result is + returned from the lockfile. Otherwise, Bazel throws an error indicating the + variations between the project and the lockfile. This mode is particularly + useful when you want to ensure that your project's dependencies remain + unchanged, and any differences are treated as errors. +* `off`: The lockfile is not checked at all. + +## Lockfile Benefits {:#lockfile-benefits} + +The lockfile offers several benefits and can be utilized in various ways: + +- **Reproducible builds.** By capturing the specific versions or dependencies + of software libraries, the lockfile ensures that builds are reproducible + across different environments and over time. Developers can rely on + consistent and predictable results when building their projects. + +- **Efficient resolution skipping.** The lockfile enables Bazel to skip the + resolution process if there are no changes in the project dependencies since + the last build. This significantly improves build efficiency, especially in + scenarios where resolution can be time-consuming. + +- **Stability and risk reduction.** The lockfile helps maintain stability by + preventing unexpected updates or breaking changes in external libraries. By + locking the dependencies to specific versions, the risk of introducing bugs + due to incompatible or untested updates is reduced. + +## Lockfile Contents {:#lockfile-contents} + +The lockfile contains all the necessary information to determine whether the +project state has changed. It also includes the result of building the project +in the current state. The lockfile consists of two main parts: + +1. Inputs of the module resolution, such as `moduleFileHash`, `flags` and + `localOverrideHashes`, as well as the output of the resolution, which is + `moduleDepGraph`. +2. For each module extension, the lockfile includes inputs that affect it, + represented by `transitiveDigest`, and the output of running that extension + referred to as `generatedRepoSpecs` + +Here is an example that demonstrates the structure of the lockfile, along with +explanations for each section: + +```json +{ + "lockFileVersion": 1, + "moduleFileHash": "b0f47b98a67ee15f9.......8dff8721c66b721e370", + "flags": { + "cmdRegistries": [ + "https://bcr.bazel.build/" + ], + "cmdModuleOverrides": {}, + "allowedYankedVersions": [], + "envVarAllowedYankedVersions": "", + "ignoreDevDependency": false, + "directDependenciesMode": "WARNING", + "compatibilityMode": "ERROR" + }, + "localOverrideHashes": { + "bazel_tools": "b5ae1fa37632140aff8.......15c6fe84a1231d6af9" + }, + "moduleDepGraph": { + "": { + "name": "", + "version": "", + "executionPlatformsToRegister": [], + "toolchainsToRegister": [], + "extensionUsages": [ + { + "extensionBzlFile": "extension.bzl", + "extensionName": "lockfile_ext" + } + ], + ... + } + }, + "moduleExtensions": { + "//:extension.bzl%lockfile_ext": { + "transitiveDigest": "oWDzxG/aLnyY6Ubrfy....+Jp6maQvEPxn0pBM=", + "generatedRepoSpecs": { + "hello": { + "bzlFile": "@@//:extension.bzl", + ... + } + } + } + } +} +``` + +### Module File Hash {:#module-file-hash} + +The `moduleFileHash` represents the hash of the `MODULE.bazel` file contents. If +any changes occur in this file, the hash value differs. + +### Flags {:#flags} + +The `Flags` object stores all the flags that can affect the resolution result. + +### Local Override Hashes {:#local-override-hashes} + +If the root module includes `local_path_overrides`, this section stores the hash +of the `MODULE.bazel` file in the local repository. It allows tracking changes +to this dependency. + +### Module Dependency Graph {:#module-dep-graph} + +The `moduleDepGraph` represents the result of the resolution process using the +inputs mentioned above. It forms the dependency graph of all the modules +required to run the project. + +### Module Extensions {:#module-extensions} + +The `moduleExtensions` section is a map that includes only the extensions used +in the current invocation or previously invoked, while excluding any extensions +that are no longer utilized. In other words, if an extension is not being used +anymore across the dependency graph, it is removed from the `moduleExtensions` +map. + +Each entry in this map corresponds to a used extension and is identified by its +containing file and name. The corresponding value for each entry contains the +relevant information associated with that extension: + +1. The `transitiveDigest` the digest of the extension implementation and its + transitive .bzl files. +2. The `generatedRepoSpecs` the result of running that extension with the + current input. + +An additional factor that can affect the extension results is their _usages_. +Although not stored in the lockfile, the usages are considered when comparing +the current state of the extension with the one in the lockfile. + +## Best Practices {:#best-practices} + +To maximize the benefits of the lockfile feature, consider the following best +practices: + +* Regularly update the lockfile to reflect changes in project dependencies or + configuration. This ensures that subsequent builds are based on the most + up-to-date and accurate set of dependencies. + +* Include the lockfile in version control to facilitate collaboration and + ensure that all team members have access to the same lockfile, promoting + consistent development environments across the project. + +By following these best practices, you can effectively utilize the lockfile +feature in Bazel, leading to more efficient, reliable, and collaborative +software development workflows. \ No newline at end of file From cfdb246e9b1b65a0dd5a5686e5c65a24336a72d4 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Mon, 10 Jul 2023 03:40:33 -0700 Subject: [PATCH 17/20] Fix null event issue If a change "only" occurred in a module extension, then the value of ModuleResolutionEvent is null -the skyvalue is cached and the event was never sent- then "combineModuleExtensions" function would crash with null pointer exception while trying to get the old usages. In this case we can just re-add the old module extensions from the lockfile because if the module resolution is the same, this means the usage didn't change. PiperOrigin-RevId: 546821911 Change-Id: Ie685cbab654d1c41403aebd31ddad91033be1d56 --- .../lib/bazel/bzlmod/BazelLockFileModule.java | 17 ++++++++++++----- src/test/py/bazel/bzlmod/bazel_lockfile_test.py | 17 ++++++++++++++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java index 521dd09a1a6dde..5bd3b40a06bd16 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java @@ -105,11 +105,18 @@ private ImmutableMap combineModuleEx ImmutableMap.Builder updatedExtensionMap = ImmutableMap.builder(); - // Add the old extensions (stored in the lockfile) only if it still has a usage somewhere - for (Map.Entry extensionEntry : - oldModuleExtensions.entrySet()) { - if (moduleResolutionEvent.getExtensionUsagesById().containsRow(extensionEntry.getKey())) { - updatedExtensionMap.put(extensionEntry); + // This event being null means that no changes occurred to the usages of the stored extensions, + // hence no changes to any module resulted in re-running resolution. So we can just add all the + // old stored extensions. Otherwise, check the usage of each one. + if (moduleResolutionEvent == null) { + updatedExtensionMap.putAll(oldModuleExtensions); + } else { + // Add the old extensions (stored in the lockfile) only if it still has a usage somewhere + for (Map.Entry extensionEntry : + oldModuleExtensions.entrySet()) { + if (moduleResolutionEvent.getExtensionUsagesById().containsRow(extensionEntry.getKey())) { + updatedExtensionMap.put(extensionEntry); + } } } diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py index c91173e323914d..03381c12e895e5 100644 --- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -367,7 +367,16 @@ def testUpdateModuleExtension(self): ) _, _, stderr = self.RunBazel(['build', '@hello//:all']) self.assertIn('Hello from the other side!', ''.join(stderr)) - self.RunBazel(['shutdown']) + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + old_impl = lockfile['moduleExtensions']['//:extension.bzl%lockfile_ext'][ + 'bzlTransitiveDigest' + ] + + # Run again to make sure the resolution value is cached. So even if module + # resolution doesn't rerun (its event is null), the lockfile is still + # updated with the newest extension eval results + self.RunBazel(['build', '@hello//:all']) # Update extension. Make sure that it is executed and updated in the # lockfile without errors (since it's already in the lockfile) @@ -389,6 +398,12 @@ def testUpdateModuleExtension(self): ) _, _, stderr = self.RunBazel(['build', '@hello//:all']) self.assertIn('Hello from the other town!', ''.join(stderr)) + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + new_impl = lockfile['moduleExtensions']['//:extension.bzl%lockfile_ext'][ + 'bzlTransitiveDigest' + ] + self.assertNotEqual(new_impl, old_impl) def testUpdateModuleExtensionErrorMode(self): self.ScratchFile( From 5c8967cc7282bcfdfa5feb9e50ab5d794b04e84f Mon Sep 17 00:00:00 2001 From: salma-samy Date: Tue, 11 Jul 2023 11:57:35 +0200 Subject: [PATCH 18/20] Fix tests --- .../build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index 7b00d9ea308fb3..5dda67fd839ba2 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -290,7 +290,7 @@ public void createValue_moduleExtensions() throws Exception { maven, "rules_jvm_external~1.0~maven", pip, "rules_python~2.0~pip", myext, "dep~2.0~myext", - myext2, "dep~2.0~myext~2"); + myext2, "dep~2.0~myext2"); assertThat(value.getFullRepoMapping(ModuleKey.ROOT)) .isEqualTo( @@ -321,7 +321,7 @@ public void createValue_moduleExtensions() throws Exception { "oneext", "dep~2.0~myext~myext", "twoext", - "dep~2.0~myext~2~myext")); + "dep~2.0~myext2~myext")); } @Test From 91c3f0b56b88594a05ad52137713089b1d9b65ed Mon Sep 17 00:00:00 2001 From: salma-samy Date: Tue, 11 Jul 2023 13:23:46 +0200 Subject: [PATCH 19/20] Remove unrelated updates --- .../RunfilesRepoMappingManifestTest.java | 8 - .../BazelModuleResolutionFunctionTest.java | 6 +- .../StarlarkDocExtractTest.java | 795 ------------------ 3 files changed, 4 insertions(+), 805 deletions(-) delete mode 100644 src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java diff --git a/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java b/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java index e76ccb8e132546..aa5894aa028452 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java @@ -60,14 +60,6 @@ protected ImmutableList extraPrecomputedValues() throws Exception { PrecomputedValue.injected(BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of())); } - @Override - protected SkyframeExecutorRepositoryHelpersHolder getRepositoryHelpersHolder() { - // Transitive packages are needed for RepoMappingManifestAction and are only stored when - // external repositories are enabled. - return SkyframeExecutorRepositoryHelpersHolder.create( - new RepositoryDirectoryDirtinessChecker()); - } - @Before public void enableBzlmod() throws Exception { setBuildLanguageOptions("--enable_bzlmod"); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 8186818cd1f50d..1a05544e9d2fd0 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -291,7 +291,8 @@ public void testYankedVersionCheckIgnoredByAll() throws Exception { @Test public void testYankedVersionCheckIgnoredBySpecific() throws Exception { setupModulesForYankedVersion(); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of("b@1.0")); + BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set( + differencer, ImmutableList.of("b@1.0")); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); assertThat(result.hasError()).isFalse(); @@ -300,7 +301,8 @@ public void testYankedVersionCheckIgnoredBySpecific() throws Exception { @Test public void testBadYankedVersionFormat() throws Exception { setupModulesForYankedVersion(); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of("b~1.0")); + BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set( + differencer, ImmutableList.of("b~1.0")); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); assertThat(result.hasError()).isTrue(); diff --git a/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java b/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java deleted file mode 100644 index 95f3023a417a6f..00000000000000 --- a/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java +++ /dev/null @@ -1,795 +0,0 @@ -// Copyright 2023 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.rules.starlarkdocextract; - -import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth8.assertThat; -import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; -import static org.junit.Assert.assertThrows; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.devtools.build.lib.actions.Action; -import com.google.devtools.build.lib.analysis.ConfiguredTarget; -import com.google.devtools.build.lib.analysis.actions.BinaryFileWriteAction; -import com.google.devtools.build.lib.analysis.actions.FileWriteAction; -import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; -import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; -import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; -import com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil; -import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; -import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; -import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; -import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; -import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; -import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.skyframe.PrecomputedValue; -import com.google.devtools.build.lib.skyframe.PrecomputedValue.Injected; -import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.AspectInfo; -import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.AttributeInfo; -import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.AttributeType; -import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.FunctionParamInfo; -import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.ModuleInfo; -import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.OriginKey; -import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.ProviderInfo; -import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.ProviderNameGroup; -import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.RuleInfo; -import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.StarlarkFunctionInfo; -import com.google.protobuf.ExtensionRegistry; -import com.google.protobuf.TextFormat; -import java.io.IOException; -import java.util.NoSuchElementException; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public final class StarlarkDocExtractTest extends BuildViewTestCase { - private Path moduleRoot; // initialized by extraPrecomputedValues - private FakeRegistry registry; - - @Override - protected ImmutableList extraPrecomputedValues() { - // TODO(b/285924565): support --enable_bzlmod in BuildViewTestCase tests without needing the - // boilerplate below. - try { - moduleRoot = scratch.dir("modules"); - } catch (IOException e) { - throw new IllegalStateException(e); - } - registry = FakeRegistry.DEFAULT_FACTORY.newFakeRegistry(moduleRoot.getPathString()); - return ImmutableList.of( - PrecomputedValue.injected( - ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), - PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), - PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()), - PrecomputedValue.injected(BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), - PrecomputedValue.injected( - BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), - PrecomputedValue.injected( - BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)); - } - - private static ModuleInfo protoFromBinaryFileWriteAction(Action action) throws Exception { - assertThat(action).isInstanceOf(BinaryFileWriteAction.class); - return ModuleInfo.parseFrom( - ((BinaryFileWriteAction) action).getSource().openStream(), - ExtensionRegistry.getEmptyRegistry()); - } - - private static ModuleInfo protoFromTextFileWriteAction(Action action) throws Exception { - assertThat(action).isInstanceOf(FileWriteAction.class); - return TextFormat.parse( - ((FileWriteAction) action).getFileContents(), - ExtensionRegistry.getEmptyRegistry(), - ModuleInfo.class); - } - - private ModuleInfo protoFromConfiguredTarget(String targetName) throws Exception { - ConfiguredTarget target = getConfiguredTarget(targetName); - return protoFromBinaryFileWriteAction( - getGeneratingAction( - target, - Label.parseCanonicalUnchecked(targetName).toPathFragment().getPathString() - + ".binaryproto")); - } - - @Before - public void setUpBzlLibrary() throws Exception { - // TODO(https://github.com/bazelbuild/bazel/issues/18599): get rid of this when we bundle - // bzl_library with Bazel. - scratch.file( - "bzl_library.bzl", - "def _bzl_library_impl(ctx):", - " deps_files = [x.files for x in ctx.attr.deps]", - " all_files = depset(ctx.files.srcs, order = 'postorder', transitive = deps_files)", - " return DefaultInfo(files = all_files)", - "", - "bzl_library = rule(", - " implementation = _bzl_library_impl,", - " attrs = {", - " 'srcs': attr.label_list(allow_files = ['.bzl', '.scl']),", - " 'deps': attr.label_list(),", - " }", - ")"); - } - - @Test - public void basicFunctionality() throws Exception { - scratch.file( - "foo.bzl", // - "'''Module doc string'''", - "True"); - scratch.file( - "BUILD", // - "starlark_doc_extract(", - " name = 'extract',", - " src = 'foo.bzl',", - ")"); - ConfiguredTarget target = getConfiguredTarget("//:extract"); - ModuleInfo moduleInfo = - protoFromBinaryFileWriteAction(getGeneratingAction(target, "extract.binaryproto")); - assertThat(moduleInfo.getModuleDocstring()).isEqualTo("Module doc string"); - assertThat(moduleInfo.getFile()).isEqualTo("//:foo.bzl"); - } - - @Test - public void textprotoOut() throws Exception { - scratch.file( - "foo.bzl", // - "'''Module doc string'''", - "True"); - scratch.file( - "BUILD", // - "starlark_doc_extract(", - " name = 'extract',", - " src = 'foo.bzl',", - ")"); - ConfiguredTarget ruleTarget = getConfiguredTarget("//:extract"); - // Verify that we do not generate textproto output unless explicitly requested. - assertThrows( - NoSuchElementException.class, () -> getGeneratingAction(ruleTarget, "extract.textproto")); - - ConfiguredTarget textprotoOutputTarget = getConfiguredTarget("//:extract.textproto"); - ModuleInfo moduleInfo = - protoFromTextFileWriteAction( - getGeneratingAction(textprotoOutputTarget, "extract.textproto")); - assertThat(moduleInfo.getModuleDocstring()).isEqualTo("Module doc string"); - } - - @Test - public void sclDialect() throws Exception { - setBuildLanguageOptions("--experimental_enable_scl_dialect"); - scratch.file( - "foo.scl", // - "def f():", - " '''This is my function'''", - " pass"); - scratch.file( - "bar.scl", // - "'''My scl module string'''", - "load('//:foo.scl', 'f')", - "bar_f = f"); - scratch.file( - "BUILD", // - "load('bzl_library.bzl', 'bzl_library')", - "bzl_library(", - " name = 'foo_scl',", - " srcs = ['foo.scl'],", - ")", - "starlark_doc_extract(", - " name = 'extract',", - " src = 'bar.scl',", - " deps = ['foo_scl'],", - ")"); - ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract"); - assertThat(moduleInfo.getModuleDocstring()).isEqualTo("My scl module string"); - assertThat(moduleInfo.getFile()).isEqualTo("//:bar.scl"); - assertThat(moduleInfo.getFuncInfo(0).getDocString()).isEqualTo("This is my function"); - } - - @Test - public void sourceWithSyntaxError_fails() throws Exception { - scratch.file( - "error.bzl", // - "!!!"); - scratch.file( - "error_loader.bzl", // - "'''This is my module'''", - "load('error.bzl', 'x')"); - scratch.file( - "BUILD", // - "load('bzl_library.bzl', 'bzl_library')", - "bzl_library(", - " name = 'error_bzl',", - " srcs = ['error.bzl'],", - ")", - "starlark_doc_extract(", - " name = 'error_doc',", - " src = 'error.bzl',", - ")", - "starlark_doc_extract(", - " name = 'error_loader_doc',", - " src = 'error_loader.bzl',", - " deps = ['error_bzl'],", - ")"); - - AssertionError errorDocFailure = - assertThrows(AssertionError.class, () -> getConfiguredTarget("//:error_doc")); - assertThat(errorDocFailure).hasMessageThat().contains("invalid character: '!'"); - - AssertionError errorLoaderDocFailure = - assertThrows(AssertionError.class, () -> getConfiguredTarget("//:error_loader_doc")); - assertThat(errorLoaderDocFailure).hasMessageThat().contains("invalid character: '!'"); - } - - @Test - public void symbolNames() throws Exception { - scratch.file( - "foo.bzl", // - "def func1():", - " pass", - "def func2():", - " pass", - "def _hidden():", - " pass"); - scratch.file( - "BUILD", // - "starlark_doc_extract(", - " name = 'extract_some',", - " src = 'foo.bzl',", - " symbol_names = ['func1'],", - ")", - "starlark_doc_extract(", - " name = 'extract_all',", - " src = 'foo.bzl',", - ")"); - - ModuleInfo dumpSome = protoFromConfiguredTarget("//:extract_some"); - assertThat(dumpSome.getFuncInfoList().stream().map(StarlarkFunctionInfo::getFunctionName)) - .containsExactly("func1"); - - ModuleInfo dumpAll = protoFromConfiguredTarget("//:extract_all"); - assertThat(dumpAll.getFuncInfoList().stream().map(StarlarkFunctionInfo::getFunctionName)) - .containsExactly("func1", "func2"); - } - - @Test - public void originKey() throws Exception { - scratch.file( - "origin.bzl", // - "def my_macro():", - " pass", - "MyInfo = provider()", - "MyOtherInfo = provider()", - "my_rule = rule(", - " implementation = lambda ctx: None,", - " attrs = {'a': attr.label(providers = [MyInfo, MyOtherInfo])},", - " provides = [MyInfo, MyOtherInfo],", - ")", - "my_aspect = aspect(implementation = lambda target, ctx: None)"); - scratch.file( - "renamer.bzl", // - "load(':origin.bzl', 'my_macro', 'MyInfo', 'MyOtherInfo', 'my_rule', 'my_aspect')", - "namespace = struct(", - " renamed_macro = my_macro,", - " RenamedInfo = MyInfo,", - " renamed_rule = my_rule,", - " renamed_aspect = my_aspect,", - ")", - "other_namespace = struct(", - " RenamedOtherInfo = MyOtherInfo,", - ")"); - scratch.file( - "BUILD", // - "load('bzl_library.bzl', 'bzl_library')", - "bzl_library(", - " name = 'origin_bzl',", - " srcs = ['origin.bzl'],", - ")", - "starlark_doc_extract(", - " name = 'extract_renamed',", - " src = 'renamer.bzl',", - " deps = ['origin_bzl'],", - " symbol_names = ['namespace'],", - ")"); - - ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract_renamed"); - - assertThat(moduleInfo.getFuncInfoList().stream().map(StarlarkFunctionInfo::getFunctionName)) - .containsExactly("namespace.renamed_macro"); - assertThat(moduleInfo.getFuncInfoList().stream().map(StarlarkFunctionInfo::getOriginKey)) - .containsExactly( - OriginKey.newBuilder().setName("my_macro").setFile("//:origin.bzl").build()); - - assertThat(moduleInfo.getProviderInfoList().stream().map(ProviderInfo::getProviderName)) - .containsExactly("namespace.RenamedInfo"); - assertThat(moduleInfo.getProviderInfoList().stream().map(ProviderInfo::getOriginKey)) - .containsExactly(OriginKey.newBuilder().setName("MyInfo").setFile("//:origin.bzl").build()); - - assertThat(moduleInfo.getRuleInfoList().stream().map(RuleInfo::getRuleName)) - .containsExactly("namespace.renamed_rule"); - assertThat(moduleInfo.getRuleInfoList().stream().map(RuleInfo::getOriginKey)) - .containsExactly( - OriginKey.newBuilder().setName("my_rule").setFile("//:origin.bzl").build()); - - assertThat(moduleInfo.getRuleInfo(0).getAttributeList()) - .containsExactly( - ModuleInfoExtractor.IMPLICIT_NAME_ATTRIBUTE_INFO, - AttributeInfo.newBuilder() - .setName("a") - .setType(AttributeType.LABEL) - .setDefaultValue("None") - .addProviderNameGroup( - ProviderNameGroup.newBuilder() - .addProviderName("namespace.RenamedInfo") - .addProviderName("other_namespace.RenamedOtherInfo") - .addOriginKey( - OriginKey.newBuilder().setName("MyInfo").setFile("//:origin.bzl")) - .addOriginKey( - OriginKey.newBuilder().setName("MyOtherInfo").setFile("//:origin.bzl"))) - .build()); - assertThat(moduleInfo.getRuleInfo(0).getAdvertisedProviders()) - .isEqualTo( - ProviderNameGroup.newBuilder() - .addProviderName("namespace.RenamedInfo") - .addProviderName("other_namespace.RenamedOtherInfo") - .addOriginKey(OriginKey.newBuilder().setName("MyInfo").setFile("//:origin.bzl")) - .addOriginKey( - OriginKey.newBuilder().setName("MyOtherInfo").setFile("//:origin.bzl")) - .build()); - - assertThat(moduleInfo.getAspectInfoList().stream().map(AspectInfo::getAspectName)) - .containsExactly("namespace.renamed_aspect"); - assertThat(moduleInfo.getAspectInfoList().stream().map(AspectInfo::getOriginKey)) - .containsExactly( - OriginKey.newBuilder().setName("my_aspect").setFile("//:origin.bzl").build()); - } - - @Test - public void originKeyFileAndModuleInfoFileLabels_forBzlFileInBzlmodModule_areDisplayForm() - throws Exception { - setBuildLanguageOptions("--enable_bzlmod"); - scratch.overwriteFile("MODULE.bazel", "bazel_dep(name='origin_repo', version='0.1')"); - registry.addModule( - BzlmodTestUtil.createModuleKey("origin_repo", "0.1"), - "module(name='origin_repo', version='0.1')"); - Path originRepoPath = moduleRoot.getRelative("origin_repo~0.1"); - scratch.file(originRepoPath.getRelative("WORKSPACE").getPathString()); - scratch.file( - originRepoPath.getRelative("BUILD").getPathString(), // - "exports_files(['origin.bzl'])"); - scratch.file( - originRepoPath.getRelative("origin.bzl").getPathString(), // - "def my_macro():", - " pass", - "MyInfo = provider()", - "my_rule = rule(", - " implementation = lambda ctx: None,", - " attrs = {'a': attr.label(providers = [MyInfo])},", - " provides = [MyInfo],", - ")", - "my_aspect = aspect(implementation = lambda target, ctx: None)"); - scratch.file( - "renamer.bzl", // - "load('@origin_repo//:origin.bzl', 'my_macro', 'MyInfo', 'my_rule', 'my_aspect')", - "namespace = struct(", - " renamed_macro = my_macro,", - " RenamedInfo = MyInfo,", - " renamed_rule = my_rule,", - " renamed_aspect = my_aspect,", - ")"); - scratch.file( - "BUILD", // - "load('bzl_library.bzl', 'bzl_library')", - "bzl_library(", - " name = 'origin_bzl',", - " srcs = ['@origin_repo//:origin.bzl'],", - ")", - "starlark_doc_extract(", - " name = 'extract_origin',", - " src = '@origin_repo//:origin.bzl',", - ")", - "starlark_doc_extract(", - " name = 'extract_renamed',", - " src = 'renamer.bzl',", - " deps = ['origin_bzl'],", - ")"); - - // verify that ModuleInfo.name for a .bzl file in another bzlmod module is in display form, i.e. - // "@origin_repo//:origin.bzl" as opposed to "@@origin_repo~0.1//:origin.bzl" - ModuleInfo originModuleInfo = protoFromConfiguredTarget("//:extract_origin"); - assertThat(originModuleInfo.getFile()).isEqualTo("@origin_repo//:origin.bzl"); - - // verify that OriginKey.name for entities defined in a .bzl file in another bzlmod module is in - // display form, i.e. "@origin_repo//:origin.bzl" as opposed to "@@origin_repo~0.1//:origin.bzl" - ModuleInfo renamedModuleInfo = protoFromConfiguredTarget("//:extract_renamed"); - assertThat(renamedModuleInfo.getFile()).isEqualTo("//:renamer.bzl"); - assertThat(renamedModuleInfo.getFuncInfo(0).getOriginKey().getFile()) - .isEqualTo("@origin_repo//:origin.bzl"); - assertThat(renamedModuleInfo.getProviderInfo(0).getOriginKey().getFile()) - .isEqualTo("@origin_repo//:origin.bzl"); - assertThat(renamedModuleInfo.getAspectInfo(0).getOriginKey().getFile()) - .isEqualTo("@origin_repo//:origin.bzl"); - assertThat(renamedModuleInfo.getRuleInfo(0).getOriginKey().getFile()) - .isEqualTo("@origin_repo//:origin.bzl"); - assertThat( - renamedModuleInfo - .getRuleInfo(0) - .getAttribute(1) // 0 is the implicit name attribute - .getProviderNameGroup(0) - .getOriginKey(0) - .getFile()) - .isEqualTo("@origin_repo//:origin.bzl"); - assertThat(renamedModuleInfo.getRuleInfo(0).getAdvertisedProviders().getOriginKey(0).getFile()) - .isEqualTo("@origin_repo//:origin.bzl"); - } - - @Test - public void exportNestedFunctionsAndLambdas() throws Exception { - scratch.file( - "origin.bzl", // - "def return_nested():", - " def nested(x):", - " '''My nested function'''", - " pass", - " return nested", - "", - "def return_lambda():", - " return lambda y: y"); - scratch.file( - "exporter.bzl", // - "load(':origin.bzl', 'return_nested', 'return_lambda')", - "exported_nested = return_nested()", - "exported_lambda = return_lambda()"); - scratch.file( - "BUILD", // - "load('bzl_library.bzl', 'bzl_library')", - "bzl_library(", - " name = 'origin_bzl',", - " srcs = ['origin.bzl'],", - ")", - "starlark_doc_extract(", - " name = 'extract_exporter',", - " src = 'exporter.bzl',", - " deps = ['origin_bzl'],", - ")"); - - ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract_exporter"); - - assertThat(moduleInfo.getFuncInfoList()) - .containsExactly( - StarlarkFunctionInfo.newBuilder() - .setFunctionName("exported_nested") - .setDocString("My nested function") - .addParameter(FunctionParamInfo.newBuilder().setName("x").setMandatory(true)) - .setOriginKey( - // OriginKey.name for nested functions is explicitly unset - OriginKey.newBuilder().setFile("//:origin.bzl")) - .build(), - StarlarkFunctionInfo.newBuilder() - .setFunctionName("exported_lambda") - .addParameter(FunctionParamInfo.newBuilder().setName("y").setMandatory(true)) - .setOriginKey( - // OriginKey.name for lambdas is explicitly unset - OriginKey.newBuilder().setFile("//:origin.bzl")) - .build()); - } - - @Test - public void missingBzlLibraryDeps_fails() throws Exception { - scratch.file( - "dep.bzl", // - "load('//:forgotten_dep_of_dep.bzl', 'g')", - "def f(): pass"); - scratch.file( - "forgotten_dep_of_dep.bzl", // - "def g(): pass"); - scratch.file( - "forgotten_dep.bzl", // - "load('//:forgotten_dep_of_forgotten_dep.bzl', 'j')", - "def h(): pass"); - scratch.file( - "forgotten_dep2.bzl", // - "def i(): pass"); - scratch.file( - "forgotten_dep_of_forgotten_dep.bzl", // - "def j(): pass"); - scratch.file( - "foo.bzl", // - "load('//:dep.bzl', 'f')", - "load('//:forgotten_dep.bzl', 'h')", - "load('//:forgotten_dep2.bzl', 'i')"); - scratch.file( - "BUILD", // - "load('bzl_library.bzl', 'bzl_library')", - "bzl_library(", - " name = 'dep_bzl',", - " srcs = ['dep.bzl'],", - ")", - "starlark_doc_extract(", - " name = 'extract',", - " src = 'foo.bzl',", // Note that src does not need to be part of deps - " deps = ['dep_bzl']", - ")"); - - AssertionError e = assertThrows(AssertionError.class, () -> getConfiguredTarget("//:extract")); - assertThat(e) - .hasMessageThat() - .contains( - "missing bzl_library targets for Starlark module(s) //:forgotten_dep_of_dep.bzl," - + " //:forgotten_dep.bzl, //:forgotten_dep2.bzl"); - // We do not want to log transitive deps of already missing deps in the error message - it would - // be hard to read and unnecessary, since a valid bzl_library target should bring in its - // transitive deps. - assertThat(e).hasMessageThat().doesNotContain("forgotten_dep_of_forgotten_dep.bzl"); - } - - @Test - public void depsWithDerivedFiles_onUnknownLoads_failsAndPrintsDerivedFiles() throws Exception { - scratch.file("BUILD"); - scratch.file( - "pkg/source_file_masked_by_rule_name.bzl", // - "def f(): pass"); - scratch.file( - "pkg/source_file_masked_by_rule_output_name.bzl", // - "def g(): pass"); - scratch.file( - "pkg/foo.bzl", // - "load('//pkg:source_file_masked_by_rule_name.bzl', 'f')", - "load('//pkg:source_file_masked_by_rule_output_name.bzl', 'g')"); - scratch.file( - "pkg/BUILD", // - "load('//:bzl_library.bzl', 'bzl_library')", - "genrule(", - " name = 'source_file_masked_by_rule_name.bzl',", - " outs = ['some_output.bzl'],", - " cmd = 'touch $@'", - ")", - "genrule(", - " name = 'source_file_masked_by_rule_output_name_bzl_generator',", - " outs = ['source_file_masked_by_rule_output_name.bzl'],", - " cmd = 'touch $@'", - ")", - "genrule(", - " name = 'some_rule',", - " outs = ['ordinary_generated_file.bzl'],", - " cmd = 'touch $@'", - ")", - "bzl_library(", - " name = 'deps_bzl',", - " srcs = [", - " 'source_file_masked_by_rule_name.bzl',", - " 'source_file_masked_by_rule_output_name.bzl',", - " 'ordinary_generated_file.bzl',", - " ],", - ")", - "starlark_doc_extract(", - " name = 'extract',", - " src = 'foo.bzl',", - " deps = ['deps_bzl']", - ")"); - - AssertionError error = - assertThrows(AssertionError.class, () -> getConfiguredTarget("//pkg:extract")); - assertThat(error) - .hasMessageThat() - .contains( - "missing bzl_library targets for Starlark module(s)" - + " //pkg:source_file_masked_by_rule_name.bzl," - + " //pkg:source_file_masked_by_rule_output_name.bzl\n" - + "Note the following are generated file(s) and cannot be loaded in Starlark:" - + " pkg/some_output.bzl (generated by rule" - + " //pkg:source_file_masked_by_rule_name.bzl)," - + " pkg/source_file_masked_by_rule_output_name.bzl (generated by rule" - + " //pkg:source_file_masked_by_rule_output_name_bzl_generator)," - + " pkg/ordinary_generated_file.bzl (generated by rule //pkg:some_rule)"); - } - - @Test - public void depsWithDerivedFiles_onNoUnknownLoads_succeeds() throws Exception { - scratch.file("BUILD"); - scratch.file( - "util.bzl", - "def _impl(ctx):", - " out = ctx.actions.declare_file(ctx.attr.out)", - " ctx.actions.run_shell(command = 'touch $1', arguments = [out.path], outputs = [out])", - " return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out]))", - "generate_out_without_declaring_it_as_a_target = rule(", - " attrs = {'out': attr.string()},", - " implementation = _impl,", - ")"); - scratch.file( - "pkg/source_dep.bzl", // - "def f(): pass"); - scratch.file( - "pkg/foo.bzl", // - "load('//pkg:source_dep.bzl', 'f')"); - scratch.file( - "pkg/BUILD", // - "load('//:bzl_library.bzl', 'bzl_library')", - "load('//:util.bzl', 'generate_out_without_declaring_it_as_a_target')", - "genrule(", - " name = 'some_rule',", - " outs = ['declared_derived_dep.bzl'],", - " cmd = 'touch $@'", - ")", - // //pkg:generate_source_dep_without_declaring_it_as_a_target masks the source_dep.bzl - // source artifact with a non-target, derived artifact having the same root-relative path. - "generate_out_without_declaring_it_as_a_target(", - " name = 'generate_source_dep_without_declaring_it_as_a_target',", - " out = 'source_dep.bzl'", - ")", - "bzl_library(", - " name = 'deps_bzl',", - " srcs = [", - " 'declared_derived_dep.bzl',", - " 'source_dep.bzl',", - " 'generate_source_dep_without_declaring_it_as_a_target',", - " ],", - ")", - "starlark_doc_extract(", - " name = 'extract',", - " src = 'foo.bzl',", - " deps = ['deps_bzl']", - ")"); - - getConfiguredTarget("//pkg:extract"); - assertNoEvents(); - } - - @Test - public void srcDerivedFile_fails() throws Exception { - scratch.file("BUILD"); - scratch.file("pkg/source_file_masked_by_rule_name.bzl"); - scratch.file("pkg/source_file_masked_by_rule_output_name.bzl"); - scratch.file( - "pkg/BUILD", // - "genrule(", - " name = 'source_file_masked_by_rule_name.bzl',", - " outs = ['some_output.bzl'],", - " cmd = 'touch $@'", - ")", - "genrule(", - " name = 'source_file_masked_by_rule_output_name_bzl_generator',", - " outs = ['source_file_masked_by_rule_output_name.bzl'],", - " cmd = 'touch $@'", - ")", - "genrule(", - " name = 'some_rule',", - " outs = ['ordinary_generated_file.bzl'],", - " cmd = 'touch $@'", - ")", - "starlark_doc_extract(", - " name = 'source_file_masked_by_rule_name_doc',", - " src = 'source_file_masked_by_rule_name.bzl',", - ")", - "starlark_doc_extract(", - " name = 'source_file_masked_by_rule_output_name_doc',", - " src = 'source_file_masked_by_rule_output_name.bzl',", - ")", - "starlark_doc_extract(", - " name = 'ordinary_generated_file_doc',", - " src = 'ordinary_generated_file.bzl',", - ")"); - - AssertionError maskedByRuleError = - assertThrows( - AssertionError.class, - () -> getConfiguredTarget("//pkg:source_file_masked_by_rule_name_doc")); - assertThat(maskedByRuleError) - .hasMessageThat() - .contains( - "pkg/some_output.bzl (generated by rule //pkg:source_file_masked_by_rule_name.bzl)" - + " is not a source file and cannot be loaded in Starlark"); - - AssertionError maskedByRuleOutputError = - assertThrows( - AssertionError.class, - () -> getConfiguredTarget("//pkg:source_file_masked_by_rule_output_name_doc")); - assertThat(maskedByRuleOutputError) - .hasMessageThat() - .contains( - "pkg/source_file_masked_by_rule_output_name.bzl (generated by rule" - + " //pkg:source_file_masked_by_rule_output_name_bzl_generator) is not a source" - + " file and cannot be loaded in Starlark"); - - AssertionError ordinaryGeneratedFileError = - assertThrows( - AssertionError.class, () -> getConfiguredTarget("//pkg:ordinary_generated_file_doc")); - assertThat(ordinaryGeneratedFileError) - .hasMessageThat() - .contains( - "pkg/ordinary_generated_file.bzl (generated by rule //pkg:some_rule) is not a source" - + " file and cannot be loaded in Starlark"); - } - - @Test - public void srcAlias_resolvesToActual() throws Exception { - scratch.file("alias_name.bzl"); - scratch.file("alias_actual.bzl"); - scratch.file( - "BUILD", // - "alias(", - " name = 'alias_name.bzl',", - " actual = 'alias_actual.bzl',", - ")", - "starlark_doc_extract(", - " name = 'extract',", - " src = 'alias_name.bzl',", - ")"); - - ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract"); - assertThat(moduleInfo.getFile()).isEqualTo("//:alias_actual.bzl"); - } - - @Test - public void srcFilegroup_resolvesToFilegroupSrc() throws Exception { - scratch.file("masked_by_filegroup_name.bzl"); - scratch.file("filegroup_src_actual.bzl"); - scratch.file( - "BUILD", // - "filegroup(", - " name = 'masked_by_filegroup_name.bzl',", - " srcs = ['filegroup_src_actual.bzl'],", - ")", - "starlark_doc_extract(", - " name = 'extract',", - " src = 'masked_by_filegroup_name.bzl',", - ")"); - - ModuleInfo moduleInfo = protoFromConfiguredTarget("//:extract"); - assertThat(moduleInfo.getFile()).isEqualTo("//:filegroup_src_actual.bzl"); - } - - @Test - public void srcFilegroup_mustHaveSingleSrc() throws Exception { - scratch.file("foo.bzl"); - scratch.file("bar.bzl"); - scratch.file( - "BUILD", // - "filegroup(", - " name = 'no_files',", - " srcs = [],", - ")", - "filegroup(", - " name = 'two_files',", - " srcs = ['foo.bzl', 'bar.bzl'],", - ")", - "starlark_doc_extract(", - " name = 'no_files_doc',", - " src = 'no_files',", - ")", - "starlark_doc_extract(", - " name = 'two_files_doc',", - " src = 'two_files',", - ")"); - - AssertionError extractNoFilesError = - assertThrows(AssertionError.class, () -> getConfiguredTarget("//:no_files_doc")); - assertThat(extractNoFilesError) - .hasMessageThat() - .contains("'//:no_files' must produce a single file"); - - AssertionError extractTwoFilesError = - assertThrows(AssertionError.class, () -> getConfiguredTarget("//:two_files_doc")); - assertThat(extractTwoFilesError) - .hasMessageThat() - .contains("'//:two_files' must produce a single file"); - } -} From 357a9ebf91403a0916d0720176d5b5e4d29d2205 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Tue, 11 Jul 2023 13:27:41 +0200 Subject: [PATCH 20/20] reverse unrelated changes --- .../query2/testutil/SkyframeQueryHelper.java | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java index ef99119130ef3a..3ce62e94f51e74 100644 --- a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java +++ b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java @@ -23,6 +23,7 @@ import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ServerDirectories; +import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; @@ -35,8 +36,9 @@ import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.packages.ConstantRuleVisibility; import com.google.devtools.build.lib.packages.PackageFactory; -import com.google.devtools.build.lib.packages.RuleVisibility; +import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension; import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.packages.util.MockToolsConfig; @@ -57,9 +59,9 @@ import com.google.devtools.build.lib.query2.engine.QueryUtil.AggregateAllOutputFormatterCallback; import com.google.devtools.build.lib.query2.engine.ThreadSafeOutputFormatterCallback; import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; -import com.google.devtools.build.lib.runtime.QuiescingExecutorsImpl; import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; import com.google.devtools.build.lib.skyframe.IgnoredPackagePrefixesFunction; +import com.google.devtools.build.lib.skyframe.PackageValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.SkyframeExecutor; import com.google.devtools.build.lib.skyframe.SkyframeTargetPatternEvaluator; @@ -147,11 +149,6 @@ public void setUp() throws Exception { this.queryEnvironmentFactory = makeQueryEnvironmentFactory(); } - @Override - public final void cleanUp() { - skyframeExecutor.getEvaluator().cleanupInterningPools(); - } - protected abstract String getRootDirectoryNameForSetup(); protected abstract void performAdditionalClientSetup(MockToolsConfig mockToolsConfig) @@ -315,13 +312,10 @@ protected boolean enableBzlmod() { protected void initTargetPatternEvaluator(ConfiguredRuleClassProvider ruleClassProvider) { this.toolsRepository = ruleClassProvider.getToolsRepository(); - if (skyframeExecutor != null) { - cleanUp(); - } skyframeExecutor = createSkyframeExecutor(ruleClassProvider); PackageOptions packageOptions = Options.getDefaults(PackageOptions.class); - packageOptions.defaultVisibility = RuleVisibility.PRIVATE; + packageOptions.defaultVisibility = ConstantRuleVisibility.PRIVATE; packageOptions.showLoadingProgress = true; packageOptions.globbingThreads = 7; packageOptions.packagePath = ImmutableList.of(rootDirectory.getPathString()); @@ -340,7 +334,6 @@ protected void initTargetPatternEvaluator(ConfiguredRuleClassProvider ruleClassP ImmutableMap.of(), ImmutableMap.of(), new TimestampGranularityMonitor(BlazeClock.instance()), - QuiescingExecutorsImpl.forTesting(), FakeOptions.builder().put(packageOptions).put(buildLanguageOptions).build()); } catch (InterruptedException | AbruptExitException e) { throw new IllegalStateException(e); @@ -361,7 +354,7 @@ public void setSyscallCache(SyscallCache syscallCache) { protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ruleClassProvider) { PackageFactory pkgFactory = ((PackageFactoryBuilderWithSkyframeForTesting) - TestPackageFactoryBuilderFactory.getInstance().builder(directories)) + TestPackageFactoryBuilderFactory.getInstance().builder(directories)) .setExtraSkyFunctions(analysisMock.getSkyFunctions(directories)) .setExtraPrecomputeValues( ImmutableList.of( @@ -379,7 +372,8 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), PrecomputedValue.injected( - BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE))) + BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF))) + .setEnvironmentExtensions(getEnvironmentExtensions()) .build(ruleClassProvider, fileSystem); SkyframeExecutor skyframeExecutor = BazelSkyframeExecutorConstants.newBazelSkyframeExecutorBuilder() @@ -390,7 +384,7 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru .setIgnoredPackagePrefixesFunction( new IgnoredPackagePrefixesFunction(ignoredPackagePrefixesFile)) .setExtraSkyFunctions(analysisMock.getSkyFunctions(directories)) - .setSyscallCache(delegatingSyscallCache) + .setPerCommandSyscallCache(delegatingSyscallCache) .build(); skyframeExecutor.injectExtraPrecomputedValues( ImmutableList.builder() @@ -421,17 +415,21 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR)) - .add( - PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE)) + .add(PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)) .build()); SkyframeExecutorTestHelper.process(skyframeExecutor); return skyframeExecutor; } + protected abstract Iterable getEnvironmentExtensions(); + + protected abstract BuildOptions getDefaultBuildOptions( + ConfiguredRuleClassProvider ruleClassProvider); + @Override public void assertPackageNotLoaded(String packageName) throws Exception { MemoizingEvaluator evaluator = skyframeExecutor.getEvaluator(); - SkyKey key = PackageIdentifier.createInMainRepo(packageName); + SkyKey key = PackageValue.key(PackageIdentifier.createInMainRepo(packageName)); if (evaluator.getExistingValue(key) != null || evaluator.getExistingErrorForTesting(key) != null) { throw new IllegalStateException("Package was loaded: " + packageName);