Add experimental_remote_cache_key_ignore_stamping to skip volatile …

…stamping files on compute shared cache key
bazelbuild · Sep 30, 2022 · 3739927 · 3739927
1 parent a380a15
commit 3739927
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 3 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java
@@ -80,6 +80,7 @@
 import com.google.devtools.build.lib.actions.UserExecException;
 import com.google.devtools.build.lib.analysis.platform.PlatformUtils;
 import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.Reporter;
 import com.google.devtools.build.lib.exec.SpawnInputExpander.InputWalker;
@@ -125,6 +126,7 @@
 import io.reactivex.rxjava3.core.SingleObserver;
 import io.reactivex.rxjava3.disposables.Disposable;
 import io.reactivex.rxjava3.schedulers.Schedulers;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -365,7 +367,8 @@ public CachePolicy getWriteCachePolicy(Spawn spawn) {
   public boolean mayBeExecutedRemotely(Spawn spawn) {
     return remoteCache instanceof RemoteExecutionCache
         && remoteExecutor != null
-        && Spawns.mayBeExecutedRemotely(spawn);
+        && Spawns.mayBeExecutedRemotely(spawn)
+        && !(remoteOptions.remoteCacheKeyIgnoreStamping && hasVolatileArtifacts(spawn.getInputFiles()));
   }
 
   private SortedMap<PathFragment, ActionInput> buildOutputDirMap(Spawn spawn) {
@@ -410,7 +413,7 @@ private MerkleTree buildInputMerkleTree(Spawn spawn, SpawnExecutionContext conte
         newInputMap.putAll(outputDirMap);
         inputMap = newInputMap;
       }
-      return MerkleTree.build(inputMap, context.getMetadataProvider(), execRoot, digestUtil);
+      return MerkleTree.build(filterInputs(inputMap), context.getMetadataProvider(), execRoot, digestUtil);
     }
   }
 
@@ -431,14 +434,28 @@ public MerkleTree uncachedBuildMerkleTreeVisitor(
       throws IOException, ForbiddenActionInputException {
     ConcurrentLinkedQueue<MerkleTree> subMerkleTrees = new ConcurrentLinkedQueue<>();
     subMerkleTrees.add(
-        MerkleTree.build(walker.getLeavesInputMapping(), metadataProvider, execRoot, digestUtil));
+        MerkleTree.build(filterInputs(walker.getLeavesInputMapping()), metadataProvider, execRoot, digestUtil));
     walker.visitNonLeaves(
         (Object subNodeKey, InputWalker subWalker) -> {
           subMerkleTrees.add(buildMerkleTreeVisitor(subNodeKey, subWalker, metadataProvider));
         });
     return MerkleTree.merge(subMerkleTrees, digestUtil);
   }
 
+  private SortedMap<PathFragment, ActionInput> filterInputs(SortedMap<PathFragment, ActionInput> inputs) {
+    if (!remoteOptions.remoteCacheKeyIgnoreStamping) {
+      return inputs;
+    }
+    SortedMap<PathFragment, ActionInput> result = new TreeMap<>();
+    for (Entry<PathFragment, ActionInput> entry : inputs.entrySet()) {
+      ActionInput input = entry.getValue();
+      if (!isConstantMetadata(input)) {
+        result.put(entry.getKey(), input);
+      }
+    }
+    return result;
+  }
+
   @Nullable
   private static ByteString buildSalt(Spawn spawn) {
     String workspace =
@@ -1494,4 +1511,20 @@ void report(Event evt) {
       reporter.handle(evt);
     }
   }
+
+  private static boolean hasVolatileArtifacts(NestedSet<? extends ActionInput> inputFiles) {
+    for (ActionInput inputFile : inputFiles.getLeaves()) {
+      if (isConstantMetadata(inputFile)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private static boolean isConstantMetadata(ActionInput input) {
+    if (input instanceof Artifact) {
+      return ((Artifact) input).isConstantMetadata();
+    }
+    return false;
+  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
@@ -83,6 +83,14 @@ public final class RemoteOptions extends CommonRemoteOptions {
               + "disable TLS.")
   public String remoteExecutor;
 
+  @Option(
+    name = "experimental_remote_cache_key_ignore_stamping",
+    defaultValue = "false",
+    documentationCategory = OptionDocumentationCategory.REMOTE,
+    effectTags = {OptionEffectTag.UNKNOWN},
+    help = "Don't use volatile stamping data in shared cache key. Also disable remote execution for stamping actions.")
+  public boolean remoteCacheKeyIgnoreStamping;
+
   @Option(
       name = "experimental_remote_execution_keepalive",
       defaultValue = "false",