[6.1.0] Cleanup stale state when remote cache evicted

src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java

@@ -144,19 +144,11 @@ private ActionCache.Entry getCacheEntry(Action action) {
     if (!cacheConfig.enabled()) {
       return null; // ignore existing cache when disabled.
     }
-    for (Artifact output : action.getOutputs()) {
-      ActionCache.Entry entry = actionCache.get(output.getExecPathString());
-      if (entry != null) {
-        return entry;
-      }
-    }
-    return null;
+    return ActionCacheUtils.getCacheEntry(actionCache, action);
   }
 
   private void removeCacheEntry(Action action) {
-    for (Artifact output : action.getOutputs()) {
-      actionCache.remove(output.getExecPathString());
-    }
+    ActionCacheUtils.removeCacheEntry(actionCache, action);
   }
 
   @Nullable

src/main/java/com/google/devtools/build/lib/actions/ActionCacheUtils.java

@@ -0,0 +1,40 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.actions.cache.ActionCache;
+import javax.annotation.Nullable;
+
+/** Utility functions for {@link ActionCache}. */
+public class ActionCacheUtils {
+  private ActionCacheUtils() {}
+
+  /** Checks whether one of existing output paths is already used as a key. */
+  @Nullable
+  public static ActionCache.Entry getCacheEntry(ActionCache actionCache, Action action) {
+    for (Artifact output : action.getOutputs()) {
+      ActionCache.Entry entry = actionCache.get(output.getExecPathString());
+      if (entry != null) {
+        return entry;
+      }
+    }
+    return null;
+  }
+
+  public static void removeCacheEntry(ActionCache actionCache, Action action) {
+    for (Artifact output : action.getOutputs()) {
+      actionCache.remove(output.getExecPathString());
+    }
+  }
+}

src/main/java/com/google/devtools/build/lib/remote/AbstractActionInputPrefetcher.java

@@ -42,6 +42,7 @@
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable;
 import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
 import com.google.devtools.build.lib.remote.util.AsyncTaskCache;
 import com.google.devtools.build.lib.remote.util.RxUtils.TransferResult;
 import com.google.devtools.build.lib.remote.util.TempPathGenerator;
@@ -60,6 +61,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Pattern;
+import javax.annotation.Nullable;
 
 /**
  * Abstract implementation of {@link ActionInputPrefetcher} which implements the orchestration of
@@ -76,6 +78,8 @@ public abstract class AbstractActionInputPrefetcher implements ActionInputPrefet
   protected final Path execRoot;
   protected final ImmutableList<Pattern> patternsToDownload;
 
+  private final Set<ActionInput> missingActionInputs = Sets.newConcurrentHashSet();
+
   private static class Context {
     private final Set<Path> nonWritableDirs = Sets.newConcurrentHashSet();
 
@@ -387,7 +391,8 @@ private Completable prefetchInputFileOrSymlink(
     PathFragment prefetchExecPath = metadata.getMaterializationExecPath().orElse(execPath);
 
     Completable prefetch =
-        downloadFileNoCheckRx(context, execRoot.getRelative(prefetchExecPath), metadata, priority);
+        downloadFileNoCheckRx(
+            context, execRoot.getRelative(prefetchExecPath), input, metadata, priority);
 
     // If prefetching to a different path, plant a symlink into it.
     if (!prefetchExecPath.equals(execPath)) {
@@ -406,15 +411,23 @@ private Completable prefetchInputFileOrSymlink(
    * download finished.
    */
   private Completable downloadFileRx(
-      Context context, Path path, FileArtifactValue metadata, Priority priority) {
+      Context context,
+      Path path,
+      @Nullable ActionInput actionInput,
+      FileArtifactValue metadata,
+      Priority priority) {
     if (!canDownloadFile(path, metadata)) {
       return Completable.complete();
     }
-    return downloadFileNoCheckRx(context, path, metadata, priority);
+    return downloadFileNoCheckRx(context, path, actionInput, metadata, priority);
   }
 
   private Completable downloadFileNoCheckRx(
-      Context context, Path path, FileArtifactValue metadata, Priority priority) {
+      Context context,
+      Path path,
+      @Nullable ActionInput actionInput,
+      FileArtifactValue metadata,
+      Priority priority) {
     if (path.isSymbolicLink()) {
       try {
         path = path.getRelative(path.readSymbolicLink());
@@ -428,26 +441,32 @@ private Completable downloadFileNoCheckRx(
     AtomicBoolean completed = new AtomicBoolean(false);
     Completable download =
         Completable.using(
-            tempPathGenerator::generateTempPath,
-            tempPath ->
-                toCompletable(
-                        () ->
-                            doDownloadFile(
-                                tempPath, finalPath.relativeTo(execRoot), metadata, priority),
-                        directExecutor())
-                    .doOnComplete(
-                        () -> {
-                          finalizeDownload(context, tempPath, finalPath);
-                          completed.set(true);
-                        }),
-            tempPath -> {
-              if (!completed.get()) {
-                deletePartialDownload(tempPath);
-              }
-            },
-            // Set eager=false here because we want cleanup the download *after* upstream is
-            // disposed.
-            /* eager= */ false);
+                tempPathGenerator::generateTempPath,
+                tempPath ->
+                    toCompletable(
+                            () ->
+                                doDownloadFile(
+                                    tempPath, finalPath.relativeTo(execRoot), metadata, priority),
+                            directExecutor())
+                        .doOnComplete(
+                            () -> {
+                              finalizeDownload(context, tempPath, finalPath);
+                              completed.set(true);
+                            }),
+                tempPath -> {
+                  if (!completed.get()) {
+                    deletePartialDownload(tempPath);
+                  }
+                },
+                // Set eager=false here because we want cleanup the download *after* upstream is
+                // disposed.
+                /* eager= */ false)
+            .doOnError(
+                error -> {
+                  if (error instanceof CacheNotFoundException && actionInput != null) {
+                    missingActionInputs.add(actionInput);
+                  }
+                });
 
     return downloadCache.executeIfNot(
         finalPath,
@@ -467,19 +486,27 @@ private Completable downloadFileNoCheckRx(
    * <p>The file will be written into a temporary file and moved to the final destination after the
    * download finished.
    */
-  public void downloadFile(Path path, FileArtifactValue metadata)
+  public void downloadFile(Path path, @Nullable ActionInput actionInput, FileArtifactValue metadata)
       throws IOException, InterruptedException {
-    getFromFuture(downloadFileAsync(path.asFragment(), metadata, Priority.CRITICAL));
+    getFromFuture(downloadFileAsync(path.asFragment(), actionInput, metadata, Priority.CRITICAL));
   }
 
   protected ListenableFuture<Void> downloadFileAsync(
-      PathFragment path, FileArtifactValue metadata, Priority priority) {
+      PathFragment path,
+      @Nullable ActionInput actionInput,
+      FileArtifactValue metadata,
+      Priority priority) {
     Context context = new Context();
     return toListenableFuture(
         Completable.using(
             () -> context,
             ctx ->
-                downloadFileRx(context, execRoot.getFileSystem().getPath(path), metadata, priority),
+                downloadFileRx(
+                    context,
+                    execRoot.getFileSystem().getPath(path),
+                    actionInput,
+                    metadata,
+                    priority),
             Context::finalizeContext));
   }
 
@@ -636,4 +663,8 @@ private boolean outputMatchesPattern(Artifact output) {
   public void flushOutputTree() throws InterruptedException {
     downloadCache.awaitInProgressTasks();
   }
+
+  public ImmutableSet<ActionInput> getMissingActionInputs() {
+    return ImmutableSet.copyOf(missingActionInputs);
+  }
 }

src/main/java/com/google/devtools/build/lib/remote/BUILD

@@ -52,6 +52,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib:runtime/command_line_path_factory",
         "//src/main/java/com/google/devtools/build/lib/actions",
         "//src/main/java/com/google/devtools/build/lib/actions:action_input_helper",
+        "//src/main/java/com/google/devtools/build/lib/actions:action_lookup_data",
         "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
         "//src/main/java/com/google/devtools/build/lib/actions:execution_requirements",
         "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
@@ -183,11 +184,13 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
         "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
         "//src/main/java/com/google/devtools/build/lib/events",
+        "//src/main/java/com/google/devtools/build/lib/remote/common",
         "//src/main/java/com/google/devtools/build/lib/remote/util",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//third_party:flogger",
         "//third_party:guava",
+        "//third_party:jsr305",
         "//third_party:rxjava3",
     ],
 )

src/main/java/com/google/devtools/build/lib/remote/RemoteActionFileSystem.java

@@ -24,6 +24,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ActionInputMap;
 import com.google.devtools.build.lib.actions.Artifact;
 import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
@@ -538,6 +539,12 @@ public RemoteFileArtifactValue getRemoteMetadata() {
     };
   }
 
+  @Nullable
+  protected ActionInput getActionInput(PathFragment path) {
+    PathFragment execPath = path.relativeTo(execRoot);
+    return inputArtifactData.getInput(execPath.getPathString());
+  }
+
   @Nullable
   protected RemoteFileArtifactValue getRemoteMetadata(PathFragment path) {
     if (!isOutput(path)) {
@@ -577,7 +584,7 @@ private void downloadFileIfRemote(PathFragment path) throws IOException {
     FileArtifactValue m = getRemoteMetadata(path);
     if (m != null) {
       try {
-        inputFetcher.downloadFile(delegateFs.getPath(path), m);
+        inputFetcher.downloadFile(delegateFs.getPath(path), getActionInput(path), m);
       } catch (InterruptedException e) {
         Thread.currentThread().interrupt();
         throw new IOException(

src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java

@@ -151,15 +151,21 @@ public ListenableFuture<Void> uploadActionResult(
    */
   public ListenableFuture<Void> uploadFile(
       RemoteActionExecutionContext context, Digest digest, Path file) {
+    return uploadFile(context, digest, file, /* force= */ false);
+  }
+
+  protected ListenableFuture<Void> uploadFile(
+      RemoteActionExecutionContext context, Digest digest, Path file, boolean force) {
     if (digest.getSizeBytes() == 0) {
       return COMPLETED_SUCCESS;
     }
 
     Completable upload =
-        casUploadCache.executeIfNot(
+        casUploadCache.execute(
             digest,
             RxFutures.toCompletable(
-                () -> cacheProtocol.uploadFile(context, digest, file), directExecutor()));
+                () -> cacheProtocol.uploadFile(context, digest, file), directExecutor()),
+            force);
 
     return RxFutures.toListenableFuture(upload);
   }
@@ -176,15 +182,21 @@ public ListenableFuture<Void> uploadFile(
    */
   public ListenableFuture<Void> uploadBlob(
       RemoteActionExecutionContext context, Digest digest, ByteString data) {
+    return uploadBlob(context, digest, data, /* force= */ false);
+  }
+
+  protected ListenableFuture<Void> uploadBlob(
+      RemoteActionExecutionContext context, Digest digest, ByteString data, boolean force) {
     if (digest.getSizeBytes() == 0) {
       return COMPLETED_SUCCESS;
     }
 
     Completable upload =
-        casUploadCache.executeIfNot(
+        casUploadCache.execute(
             digest,
             RxFutures.toCompletable(
-                () -> cacheProtocol.uploadBlob(context, digest, data), directExecutor()));
+                () -> cacheProtocol.uploadBlob(context, digest, data), directExecutor()),
+            force);
 
     return RxFutures.toListenableFuture(upload);
   }

src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java

@@ -34,7 +34,9 @@
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
+import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.FileArtifactValue;
 import com.google.devtools.build.lib.analysis.AnalysisResult;
 import com.google.devtools.build.lib.analysis.BlazeDirectories;
 import com.google.devtools.build.lib.analysis.ConfiguredTarget;
@@ -58,6 +60,7 @@
 import com.google.devtools.build.lib.exec.ModuleActionContextRegistry;
 import com.google.devtools.build.lib.exec.SpawnStrategyRegistry;
 import com.google.devtools.build.lib.remote.RemoteServerCapabilities.ServerCapabilitiesRequirement;
+import com.google.devtools.build.lib.remote.ToplevelArtifactsDownloader.PathToMetadataConverter;
 import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
 import com.google.devtools.build.lib.remote.common.RemoteExecutionClient;
 import com.google.devtools.build.lib.remote.downloader.GrpcRemoteDownloader;
@@ -921,7 +924,6 @@ public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorB
               remoteOptions.useNewExitCodeForLostInputs);
       env.getEventBus().register(actionInputFetcher);
       builder.setActionInputPrefetcher(actionInputFetcher);
-      remoteOutputService.setActionInputFetcher(actionInputFetcher);
       actionContextProvider.setActionInputFetcher(actionInputFetcher);
 
       toplevelArtifactsDownloader =
@@ -930,15 +932,35 @@ public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorB
               remoteOutputsMode.downloadToplevelOutputsOnly(),
               env.getSkyframeExecutor().getEvaluator(),
               actionInputFetcher,
-              (path) -> {
-                FileSystem fileSystem = path.getFileSystem();
-                if (fileSystem instanceof RemoteActionFileSystem) {
-                  return ((RemoteActionFileSystem) path.getFileSystem())
-                      .getRemoteMetadata(path.asFragment());
+              new PathToMetadataConverter() {
+                @Nullable
+                @Override
+                public FileArtifactValue getMetadata(Path path) {
+                  FileSystem fileSystem = path.getFileSystem();
+                  if (fileSystem instanceof RemoteActionFileSystem) {
+                    return ((RemoteActionFileSystem) path.getFileSystem())
+                        .getRemoteMetadata(path.asFragment());
+                  }
+                  return null;
+                }
+
+                @Nullable
+                @Override
+                public ActionInput getActionInput(Path path) {
+                  FileSystem fileSystem = path.getFileSystem();
+                  if (fileSystem instanceof RemoteActionFileSystem) {
+                    return ((RemoteActionFileSystem) path.getFileSystem())
+                        .getActionInput(path.asFragment());
+                  }
+                  return null;
                 }
-                return null;
               });
       env.getEventBus().register(toplevelArtifactsDownloader);
+
+      remoteOutputService.setActionInputFetcher(actionInputFetcher);
+      remoteOutputService.setMemoizingEvaluator(env.getSkyframeExecutor().getEvaluator());
+      remoteOutputService.setActionCache(env.getBlazeWorkspace().getPersistentActionCache());
+      env.getEventBus().register(remoteOutputService);
     }
   }