Add experimental_remote_cache_key_ignore_stamping to skip volatile …

…stamping files on compute shared cache key
bazelbuild · Oct 24, 2022 · db9dd38 · db9dd38
1 parent 7190f22
commit db9dd38
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 3 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java
@@ -76,6 +76,7 @@
 import com.google.devtools.build.lib.actions.Spawns;
 import com.google.devtools.build.lib.analysis.platform.PlatformUtils;
 import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.Reporter;
 import com.google.devtools.build.lib.exec.SpawnInputExpander.InputWalker;
@@ -127,6 +128,7 @@
 import io.reactivex.rxjava3.core.SingleObserver;
 import io.reactivex.rxjava3.disposables.Disposable;
 import io.reactivex.rxjava3.schedulers.Schedulers;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -343,7 +345,8 @@ public CachePolicy getWriteCachePolicy(Spawn spawn) {
   public boolean mayBeExecutedRemotely(Spawn spawn) {
     return remoteCache instanceof RemoteExecutionCache
         && remoteExecutor != null
-        && Spawns.mayBeExecutedRemotely(spawn);
+        && Spawns.mayBeExecutedRemotely(spawn)
+        && !(remoteOptions.remoteCacheKeyIgnoreStamping && hasVolatileArtifacts(spawn.getInputFiles()));
   }
 
   private SortedMap<PathFragment, ActionInput> buildOutputDirMap(Spawn spawn) {
@@ -395,7 +398,7 @@ private MerkleTree buildInputMerkleTree(
         inputMap = newInputMap;
       }
       return MerkleTree.build(
-          inputMap,
+          filterInputs(inputMap),
           toolSignature == null ? ImmutableSet.of() : toolSignature.toolInputs,
           context.getMetadataProvider(),
           execRoot,
@@ -420,14 +423,28 @@ public MerkleTree uncachedBuildMerkleTreeVisitor(
       throws IOException, ForbiddenActionInputException {
     ConcurrentLinkedQueue<MerkleTree> subMerkleTrees = new ConcurrentLinkedQueue<>();
     subMerkleTrees.add(
-        MerkleTree.build(walker.getLeavesInputMapping(), metadataProvider, execRoot, digestUtil));
+        MerkleTree.build(filterInputs(walker.getLeavesInputMapping()), metadataProvider, execRoot, digestUtil));
     walker.visitNonLeaves(
         (Object subNodeKey, InputWalker subWalker) -> {
           subMerkleTrees.add(buildMerkleTreeVisitor(subNodeKey, subWalker, metadataProvider));
         });
     return MerkleTree.merge(subMerkleTrees, digestUtil);
   }
 
+  private SortedMap<PathFragment, ActionInput> filterInputs(SortedMap<PathFragment, ActionInput> inputs) {
+    if (!remoteOptions.remoteCacheKeyIgnoreStamping) {
+      return inputs;
+    }
+    SortedMap<PathFragment, ActionInput> result = new TreeMap<>();
+    for (Entry<PathFragment, ActionInput> entry : inputs.entrySet()) {
+      ActionInput input = entry.getValue();
+      if (!isConstantMetadata(input)) {
+        result.put(entry.getKey(), input);
+      }
+    }
+    return result;
+  }
+
   @Nullable
   private static ByteString buildSalt(Spawn spawn) {
     String workspace =
@@ -1476,6 +1493,22 @@ void report(Event evt) {
     }
   }
 
+  private static boolean hasVolatileArtifacts(NestedSet<? extends ActionInput> inputFiles) {
+    for (ActionInput inputFile : inputFiles.getLeaves()) {
+      if (isConstantMetadata(inputFile)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private static boolean isConstantMetadata(ActionInput input) {
+    if (input instanceof Artifact) {
+      return ((Artifact) input).isConstantMetadata();
+    }
+    return false;
+  }
+
   /**
    * A simple value class combining a hash of the tool inputs (and their digests) as well as a set
    * of the relative paths of all tool inputs.

diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
@@ -83,6 +83,14 @@ public final class RemoteOptions extends CommonRemoteOptions {
               + "disable TLS.")
   public String remoteExecutor;
 
+  @Option(
+    name = "experimental_remote_cache_key_ignore_stamping",
+    defaultValue = "false",
+    documentationCategory = OptionDocumentationCategory.REMOTE,
+    effectTags = {OptionEffectTag.UNKNOWN},
+    help = "Don't use volatile stamping data in shared cache key. Also disable remote execution for stamping actions.")
+  public boolean remoteCacheKeyIgnoreStamping;
+
   @Option(
       name = "experimental_remote_execution_keepalive",
       defaultValue = "false",