Module Lockfile

- Created lockfile K/F/V holding module information and its related flags
- Updated BazelDepGraph function to use the lockfile
- Created type adapters for module, related classes & immutable collections
- Updated tests with Module.bazel to work after enabling the lockfile
- Added new java test class for the lockfile
- Added a flag to enable/disable lockfile (while it's still in progress)
- Added integration tests

PiperOrigin-RevId: 520925174
Change-Id: Ib8da99aa86a1797da44dc9a899d932f58c4a84c9

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -26,6 +26,7 @@
 import com.google.devtools.build.lib.analysis.RuleDefinition;
 import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction;
+import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction;
@@ -249,7 +250,12 @@ public ResolutionReason getResolutionReason() {
         .addSkyFunction(
             SkyFunctions.MODULE_FILE,
             new ModuleFileFunction(registryFactory, directories.getWorkspace(), builtinModules))
-        .addSkyFunction(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction())
+        .addSkyFunction(
+            SkyFunctions.BAZEL_DEP_GRAPH,
+            new BazelDepGraphFunction(directories.getWorkspace(), registryFactory))
+        .addSkyFunction(
+            SkyFunctions.BAZEL_LOCK_FILE,
+            new BazelLockFileFunction(directories.getWorkspace(), registryFactory))
         .addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction())
         .addSkyFunction(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction())
         .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction)

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD

@@ -23,6 +23,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/net/starlark/java/eval",
         "//third_party:auto_value",
+        "//third_party:gson",
         "//third_party:guava",
         "//third_party:jsr305",
     ],
@@ -52,14 +53,13 @@ java_library(
     deps = [
         "//src/main/java/com/google/devtools/build/docgen/annot",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
-        "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/packages",
         "//src/main/java/net/starlark/java/annot",
         "//src/main/java/net/starlark/java/eval",
         "//src/main/java/net/starlark/java/syntax",
         "//third_party:auto_value",
+        "//third_party:gson",
         "//third_party:guava",
-        "//third_party:jsr305",
     ],
 )
 
@@ -88,7 +88,9 @@ java_library(
         "AbridgedModule.java",
         "ArchiveOverride.java",
         "BazelDepGraphValue.java",
+        "BazelLockFileValue.java",
         "BazelModuleResolutionValue.java",
+        "BzlmodFlagsAndEnvVars.java",
         "GitOverride.java",
         "LocalPathOverride.java",
         "Module.java",
@@ -110,7 +112,6 @@ java_library(
         ":repo_rule_creator",
         "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
-        "//src/main/java/com/google/devtools/build/lib/concurrent",
         "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/packages",
         "//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions",
@@ -120,6 +121,7 @@ java_library(
         "//src/main/java/net/starlark/java/eval",
         "//src/main/java/net/starlark/java/syntax",
         "//third_party:auto_value",
+        "//third_party:gson",
         "//third_party:guava",
         "//third_party:jsr305",
     ],
@@ -129,8 +131,12 @@ java_library(
     name = "resolution_impl",
     srcs = [
         "BazelDepGraphFunction.java",
+        "BazelLockFileFunction.java",
         "BazelModuleResolutionFunction.java",
+        "BzlmodFlagsAndEnvVars.java",
+        "DelegateTypeAdapterFactory.java",
         "Discovery.java",
+        "GsonTypeAdapterUtil.java",
         "ModuleExtensionContext.java",
         "ModuleFileFunction.java",
         "ModuleFileGlobals.java",
@@ -158,13 +164,15 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/packages",
+        "//src/main/java/com/google/devtools/build/lib/packages/semantics",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_directory_value",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function",
         "//src/main/java/com/google/devtools/build/lib/skyframe:bzl_load_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:client_environment_function",
         "//src/main/java/com/google/devtools/build/lib/skyframe:client_environment_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster",
+        "//src/main/java/com/google/devtools/build/lib/util",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/com/google/devtools/build/skyframe",
@@ -175,6 +183,7 @@ java_library(
         "//src/main/java/net/starlark/java/syntax",
         "//src/main/protobuf:failure_details_java_proto",
         "//third_party:auto_value",
+        "//third_party:gson",
         "//third_party:guava",
         "//third_party:jsr305",
     ],

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java

@@ -15,26 +15,36 @@
 
 package com.google.devtools.build.lib.bazel.bzlmod;
 
+import static com.google.common.base.Strings.nullToEmpty;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
 
 import com.google.common.collect.BiMap;
 import com.google.common.collect.HashBiMap;
+import com.google.common.collect.ImmutableBiMap;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableTable;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction.BazelModuleResolutionFunctionException;
+import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue;
 import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
 import com.google.devtools.build.lib.cmdline.PackageIdentifier;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.packages.LabelConverter;
+import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
 import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code;
+import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction;
+import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue;
+import com.google.devtools.build.lib.skyframe.PrecomputedValue;
+import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.build.skyframe.SkyFunction;
 import com.google.devtools.build.skyframe.SkyFunctionException;
 import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
 import com.google.devtools.build.skyframe.SkyKey;
 import com.google.devtools.build.skyframe.SkyValue;
 import javax.annotation.Nullable;
+import net.starlark.java.eval.StarlarkSemantics;
 
 /**
  * This function runs Bazel module resolution, extracts the dependency graph from it and creates a
@@ -43,24 +53,123 @@
  */
 public class BazelDepGraphFunction implements SkyFunction {
 
+  private final Path rootDirectory;
+  private final RegistryFactory registryFactory;
+
+  public BazelDepGraphFunction(Path rootDirectory, RegistryFactory registryFactory) {
+    this.rootDirectory = rootDirectory;
+    this.registryFactory = registryFactory;
+  }
+
   @Override
   @Nullable
   public SkyValue compute(SkyKey skyKey, Environment env)
       throws SkyFunctionException, InterruptedException {
 
-    BazelModuleResolutionValue selectionResult =
-        (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY);
-    if (env.valuesMissing()) {
+    RootModuleFileValue root =
+        (RootModuleFileValue) env.getValue(ModuleFileValue.KEY_FOR_ROOT_MODULE);
+    if (root == null) {
       return null;
     }
+    StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env);
+    if (starlarkSemantics == null) {
+      return null;
+    }
+
+    ImmutableMap<ModuleKey, Module> depGraph = null;
+    BzlmodFlagsAndEnvVars flags = null;
+
+    // If the module has not changed (has the same contents and flags as the lockfile),
+    // read the dependency graph from the lock file, else run resolution and update lockfile
+    if (starlarkSemantics.getBool(BuildLanguageOptions.ENABLE_LOCKFILE)) {
+      BazelLockFileValue lockFile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY);
+      if (lockFile == null) {
+        return null;
+      }
+      flags = getFlagsAndEnvVars(env);
+      if (flags == null) { // unable to read environment variables
+        return null;
+      }
+      if (root.getModuleFileHash().equals(lockFile.getModuleFileHash())
+          && flags.equals(lockFile.getFlags())) {
+        depGraph = lockFile.getModuleDepGraph();
+      }
+    }
+
+    if (depGraph == null) {
+      BazelModuleResolutionValue selectionResult =
+          (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY);
+      if (env.valuesMissing()) {
+        return null;
+      }
+      depGraph = selectionResult.getResolvedDepGraph();
+      if (starlarkSemantics.getBool(BuildLanguageOptions.ENABLE_LOCKFILE)) {
+        BazelLockFileFunction.updateLockedModule(
+            root.getModuleFileHash(), depGraph, rootDirectory, registryFactory, flags);
+      }
+    }
 
-    ImmutableMap<ModuleKey, Module> depGraph = selectionResult.getResolvedDepGraph();
     ImmutableMap<RepositoryName, ModuleKey> canonicalRepoNameLookup =
         depGraph.keySet().stream()
             .collect(toImmutableMap(ModuleKey::getCanonicalRepoName, key -> key));
 
-    // For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can
-    // group all usages by the label + name (the ModuleExtensionId).
+    ImmutableTable<ModuleExtensionId, ModuleKey, ModuleExtensionUsage> extensionUsagesById =
+        getExtensionUsagesById(depGraph);
+
+    ImmutableBiMap<String, ModuleExtensionId> extensionUniqueNames =
+        calculateUniqueNameForUsedExtensionId(extensionUsagesById);
+
+    return BazelDepGraphValue.create(
+        depGraph,
+        canonicalRepoNameLookup,
+        depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList()),
+        extensionUsagesById,
+        extensionUniqueNames.inverse());
+  }
+
+  @Nullable
+  public static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env)
+      throws InterruptedException {
+    ClientEnvironmentValue allowedYankedVersionsFromEnv =
+        (ClientEnvironmentValue)
+            env.getValue(
+                ClientEnvironmentFunction.key(
+                    BazelModuleResolutionFunction.BZLMOD_ALLOWED_YANKED_VERSIONS_ENV));
+    if (allowedYankedVersionsFromEnv == null) {
+      return null;
+    }
+
+    ImmutableList<String> registries = ImmutableList.copyOf(ModuleFileFunction.REGISTRIES.get(env));
+    ImmutableMap<String, String> moduleOverrides =
+        ModuleFileFunction.MODULE_OVERRIDES.get(env).entrySet().stream()
+            .collect(
+                toImmutableMap(e -> e.getKey(), e -> ((LocalPathOverride) e.getValue()).getPath()));
+
+    ImmutableList<String> yankedVersions =
+        ImmutableList.copyOf(BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.get(env));
+    Boolean ignoreDevDeps = ModuleFileFunction.IGNORE_DEV_DEPS.get(env);
+    String compatabilityMode =
+        BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.get(env).name();
+    String directDepsMode = BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.get(env).name();
+
+    String envYanked = allowedYankedVersionsFromEnv.getValue();
+
+    return BzlmodFlagsAndEnvVars.create(
+        registries,
+        moduleOverrides,
+        yankedVersions,
+        nullToEmpty(envYanked),
+        ignoreDevDeps,
+        directDepsMode,
+        compatabilityMode);
+  }
+
+  /**
+   * For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can group
+   * all usages by the label + name (the ModuleExtensionId).
+   */
+  private ImmutableTable<ModuleExtensionId, ModuleKey, ModuleExtensionUsage> getExtensionUsagesById(
+      ImmutableMap<ModuleKey, Module> depGraph) throws BazelModuleResolutionFunctionException {
     ImmutableTable.Builder<ModuleExtensionId, ModuleKey, ModuleExtensionUsage>
         extensionUsagesTableBuilder = ImmutableTable.builder();
     for (Module module : depGraph.values()) {
@@ -95,21 +204,18 @@ public SkyValue compute(SkyKey skyKey, Environment env)
         extensionUsagesTableBuilder.put(moduleExtensionId, module.getKey(), usage);
       }
     }
-    ImmutableTable<ModuleExtensionId, ModuleKey, ModuleExtensionUsage> extensionUsagesById =
-        extensionUsagesTableBuilder.buildOrThrow();
+    return extensionUsagesTableBuilder.buildOrThrow();
+  }
 
+  private ImmutableBiMap<String, ModuleExtensionId> calculateUniqueNameForUsedExtensionId(
+      ImmutableTable<ModuleExtensionId, ModuleKey, ModuleExtensionUsage> extensionUsagesById) {
     // Calculate a unique name for each used extension id.
     BiMap<String, ModuleExtensionId> extensionUniqueNames = HashBiMap.create();
     for (ModuleExtensionId id : extensionUsagesById.rowKeySet()) {
       // Ensure that the resulting extension name (and thus the repository names derived from it) do
       // not start with a tilde.
       RepositoryName repository = id.getBzlFileLabel().getRepository();
-      String nonEmptyRepoPart;
-      if (repository.isMain()) {
-        nonEmptyRepoPart = "_main";
-      } else {
-        nonEmptyRepoPart = repository.getName();
-      }
+      String nonEmptyRepoPart = repository.isMain() ? "_main" : repository.getName();
       String bestName = nonEmptyRepoPart + "~" + id.getExtensionName();
       if (extensionUniqueNames.putIfAbsent(bestName, id) == null) {
         continue;
@@ -119,12 +225,6 @@ public SkyValue compute(SkyKey skyKey, Environment env)
         suffix++;
       }
     }
-
-    return BazelDepGraphValue.create(
-        depGraph,
-        canonicalRepoNameLookup,
-        depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList()),
-        extensionUsagesById,
-        ImmutableMap.copyOf(extensionUniqueNames.inverse()));
+    return ImmutableBiMap.copyOf(extensionUniqueNames);
   }
 }