Merge branch 'release-7.1.0' into cp20557-7.1.0

src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java

@@ -169,11 +169,19 @@ protected boolean couldBeModifiedByMetadata(FileArtifactValue lastKnown) {
   /**
    * Optional materialization path.
    *
-   * <p>If present, this artifact is a copy of another artifact. It is still tracked as a
-   * non-symlink by Bazel, but materialized in the local filesystem as a symlink to the original
-   * artifact, whose contents live at this location. This is used by {@link
-   * com.google.devtools.build.lib.remote.AbstractActionInputPrefetcher} to implement zero-cost
-   * copies of remotely stored artifacts.
+   * <p>If present, this artifact is a copy of another artifact whose contents live at this path.
+   * This can happen when it is declared as a file and not as an unresolved symlink but the action
+   * that creates it materializes it in the filesystem as a symlink to another output artifact. This
+   * information is useful in two situations:
+   *
+   * <ol>
+   *   <li>When the symlink target is a remotely stored artifact, we can avoid downloading it
+   *       multiple times when building without the bytes (see AbstractActionInputPrefetcher).
+   *   <li>When the symlink target is inaccessible from the sandboxed environment an action runs
+   *       under, we can rewrite it accordingly (see SandboxHelpers).
+   * </ol>
+   *
+   * @see com.google.devtools.build.lib.skyframe.TreeArtifactValue#getMaterializationExecPath().
    */
   public Optional<PathFragment> getMaterializationExecPath() {
     return Optional.empty();
@@ -214,6 +222,12 @@ public static FileArtifactValue createForSourceArtifact(
         xattrProvider);
   }
 
+  public static FileArtifactValue createForResolvedSymlink(
+      PathFragment realPath, FileArtifactValue metadata, @Nullable byte[] digest) {
+    return new ResolvedSymlinkFileArtifactValue(
+        realPath, digest, metadata.getContentsProxy(), metadata.getSize());
+  }
+
   public static FileArtifactValue createFromInjectedDigest(
       FileArtifactValue metadata, @Nullable byte[] digest) {
     return createForNormalFile(digest, metadata.getContentsProxy(), metadata.getSize());
@@ -439,7 +453,25 @@ public String toString() {
     }
   }
 
-  private static final class RegularFileArtifactValue extends FileArtifactValue {
+  private static final class ResolvedSymlinkFileArtifactValue extends RegularFileArtifactValue {
+    private final PathFragment realPath;
+
+    private ResolvedSymlinkFileArtifactValue(
+        PathFragment realPath,
+        @Nullable byte[] digest,
+        @Nullable FileContentsProxy proxy,
+        long size) {
+      super(digest, proxy, size);
+      this.realPath = realPath;
+    }
+
+    @Override
+    public Optional<PathFragment> getMaterializationExecPath() {
+      return Optional.of(realPath);
+    }
+  }
+
+  private static class RegularFileArtifactValue extends FileArtifactValue {
     private final byte[] digest;
     @Nullable private final FileContentsProxy proxy;
     private final long size;
@@ -462,7 +494,8 @@ public boolean equals(Object o) {
       RegularFileArtifactValue that = (RegularFileArtifactValue) o;
       return Arrays.equals(digest, that.digest)
           && Objects.equals(proxy, that.proxy)
-          && size == that.size;
+          && size == that.size
+          && Objects.equals(getMaterializationExecPath(), that.getMaterializationExecPath());
     }
 
     @Override

src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java

@@ -15,6 +15,7 @@
 package com.google.devtools.build.lib.sandbox;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.stream.Collectors.joining;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
@@ -59,7 +60,8 @@
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Map;
-import javax.annotation.Nullable;
+import java.util.Optional;
+import java.util.stream.Stream;
 
 /** Abstract common ancestor for sandbox spawn runners implementing the common parts. */
 abstract class AbstractSandboxSpawnRunner implements SpawnRunner {
@@ -224,7 +226,7 @@ private final SpawnResult run(
       StringBuilder msg = new StringBuilder("Action failed to execute: java.io.IOException: ");
       msg.append(exceptionMsg);
       msg.append("\n");
-      if (sandboxDebugOutput != null) {
+      if (!sandboxDebugOutput.isEmpty()) {
         msg.append("Sandbox debug output:\n");
         msg.append(sandboxDebugOutput);
         msg.append("\n");
@@ -294,12 +296,12 @@ private final SpawnResult run(
     }
 
     String sandboxDebugOutput = getSandboxDebugOutput(sandbox);
-    if (sandboxDebugOutput != null) {
+    if (!sandboxDebugOutput.isEmpty()) {
       reporter.handle(
           Event.of(
               EventKind.DEBUG,
               String.format(
-                  "Sandbox debug output for %s %s: %s",
+                  "Sandbox debug output for %s %s:\n%s",
                   originalSpawn.getMnemonic(),
                   originalSpawn.getTargetLabel(),
                   sandboxDebugOutput)));
@@ -334,18 +336,21 @@ private final SpawnResult run(
     return spawnResultBuilder.build();
   }
 
-  @Nullable
   private String getSandboxDebugOutput(SandboxedSpawn sandbox) throws IOException {
+    Optional<String> sandboxDebugOutput = Optional.empty();
     Path sandboxDebugPath = sandbox.getSandboxDebugPath();
     if (sandboxDebugPath != null && sandboxDebugPath.exists()) {
       try (InputStream inputStream = sandboxDebugPath.getInputStream()) {
         String msg = new String(inputStream.readAllBytes(), UTF_8);
         if (!msg.isEmpty()) {
-          return msg;
+          sandboxDebugOutput = Optional.of(msg);
         }
       }
     }
-    return null;
+    Optional<String> interactiveDebugInstructions = sandbox.getInteractiveDebugInstructions();
+    return Stream.of(sandboxDebugOutput, interactiveDebugInstructions)
+        .flatMap(Optional::stream)
+        .collect(joining("\n"));
   }
 
   private boolean wasTimeout(Duration timeout, Duration wallTime) {

src/main/java/com/google/devtools/build/lib/sandbox/BUILD

@@ -17,6 +17,7 @@ java_library(
     deps = [
         "//src/main/java/com/google/devtools/build/lib/actions",
         "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
+        "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
         "//src/main/java/com/google/devtools/build/lib/analysis:test/test_configuration",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/vfs",
@@ -72,6 +73,7 @@ java_library(
         ":sandbox_helpers",
         ":sandbox_options",
         "//src/main/java/com/google/devtools/build/lib/exec:tree_deleter",
+        "//src/main/java/com/google/devtools/build/lib/util:command",
         "//src/main/java/com/google/devtools/build/lib/util:describable_execution_unit",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",

src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java

@@ -228,6 +228,7 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
     SandboxInputs inputs =
         helpers.processInputFiles(
             context.getInputMapping(PathFragment.EMPTY_FRAGMENT, /* willAccessRepeatedly= */ true),
+            context.getInputMetadataProvider(),
             execRoot,
             execRoot,
             packageRoots,
@@ -258,29 +259,30 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
         allowNetwork
             || Spawns.requiresNetwork(spawn, getSandboxOptions().defaultSandboxAllowNetwork);
 
-      return new SymlinkedSandboxedSpawn(
-          sandboxPath,
-          sandboxExecRoot,
-          commandLine,
-          environment,
-          inputs,
-          outputs,
-          writableDirs,
-          treeDeleter,
-          /* sandboxDebugPath= */ null,
-          statisticsPath,
-          spawn.getMnemonic()) {
-        @Override
-        public void createFileSystem() throws IOException, InterruptedException {
-          super.createFileSystem();
-          writeConfig(
-              sandboxConfigPath,
-              writableDirs,
-              getInaccessiblePaths(),
-              allowNetworkForThisSpawn,
-              statisticsPath);
-        }
-      };
+    return new SymlinkedSandboxedSpawn(
+        sandboxPath,
+        sandboxExecRoot,
+        commandLine,
+        environment,
+        inputs,
+        outputs,
+        writableDirs,
+        treeDeleter,
+        /* sandboxDebugPath= */ null,
+        statisticsPath,
+        /* interactiveDebugArguments= */ null,
+        spawn.getMnemonic()) {
+      @Override
+      public void createFileSystem() throws IOException, InterruptedException {
+        super.createFileSystem();
+        writeConfig(
+            sandboxConfigPath,
+            writableDirs,
+            getInaccessiblePaths(),
+            allowNetworkForThisSpawn,
+            statisticsPath);
+      }
+    };
   }
 
   private void writeConfig(

src/main/java/com/google/devtools/build/lib/sandbox/DockerSandboxedSpawnRunner.java

@@ -226,6 +226,7 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
     SandboxInputs inputs =
         helpers.processInputFiles(
             context.getInputMapping(PathFragment.EMPTY_FRAGMENT, /* willAccessRepeatedly= */ true),
+            context.getInputMetadataProvider(),
             execRoot,
             execRoot,
             packageRoots,

src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxCommandLineBuilder.java

@@ -51,7 +51,6 @@ public static BindMount of(Path mountPoint, Path source) {
   }
 
   private final Path linuxSandboxPath;
-  private final List<String> commandArguments;
   private Path hermeticSandboxPath;
   private Path workingDirectory;
   private Duration timeout;
@@ -72,15 +71,13 @@ public static BindMount of(Path mountPoint, Path source) {
   private boolean sigintSendsSigterm = false;
   private String cgroupsDir;
 
-  private LinuxSandboxCommandLineBuilder(Path linuxSandboxPath, List<String> commandArguments) {
+  private LinuxSandboxCommandLineBuilder(Path linuxSandboxPath) {
     this.linuxSandboxPath = linuxSandboxPath;
-    this.commandArguments = commandArguments;
   }
 
   /** Returns a new command line builder for the {@code linux-sandbox} tool. */
-  public static LinuxSandboxCommandLineBuilder commandLineBuilder(
-      Path linuxSandboxPath, List<String> commandArguments) {
-    return new LinuxSandboxCommandLineBuilder(linuxSandboxPath, commandArguments);
+  public static LinuxSandboxCommandLineBuilder commandLineBuilder(Path linuxSandboxPath) {
+    return new LinuxSandboxCommandLineBuilder(linuxSandboxPath);
   }
 
   /**
@@ -247,7 +244,7 @@ public LinuxSandboxCommandLineBuilder addExecutionInfo(Map<String, String> execu
   }
 
   /** Builds the command line to invoke a specific command using the {@code linux-sandbox} tool. */
-  public ImmutableList<String> build() {
+  public ImmutableList<String> buildForCommand(List<String> commandArguments) {
     Preconditions.checkState(
         !(this.useFakeUsername && this.useFakeRoot),
         "useFakeUsername and useFakeRoot are exclusive");

src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java

@@ -106,10 +106,9 @@ private static boolean computeIsSupported(CommandEnvironment cmdEnv, Path linuxS
       throws InterruptedException {
     LocalExecutionOptions options = cmdEnv.getOptions().getOptions(LocalExecutionOptions.class);
     ImmutableList<String> linuxSandboxArgv =
-        LinuxSandboxCommandLineBuilder.commandLineBuilder(
-                linuxSandbox, ImmutableList.of("/bin/true"))
+        LinuxSandboxCommandLineBuilder.commandLineBuilder(linuxSandbox)
             .setTimeout(options.getLocalSigkillGraceSeconds())
-            .build();
+            .buildForCommand(ImmutableList.of("/bin/true"));
     ImmutableMap<String, String> env = ImmutableMap.of();
     Path execRoot = cmdEnv.getExecRoot();
     File cwd = execRoot.getPathFile();
@@ -281,6 +280,7 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
     SandboxInputs inputs =
         helpers.processInputFiles(
             context.getInputMapping(PathFragment.EMPTY_FRAGMENT, /* willAccessRepeatedly= */ true),
+            context.getInputMetadataProvider(),
             execRoot,
             withinSandboxExecRoot,
             packageRoots,
@@ -309,7 +309,7 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
     boolean createNetworkNamespace =
         !(allowNetwork || Spawns.requiresNetwork(spawn, sandboxOptions.defaultSandboxAllowNetwork));
     LinuxSandboxCommandLineBuilder commandLineBuilder =
-        LinuxSandboxCommandLineBuilder.commandLineBuilder(linuxSandbox, spawn.getArguments())
+        LinuxSandboxCommandLineBuilder.commandLineBuilder(linuxSandbox)
             .addExecutionInfo(spawn.getExecutionInfo())
             .setWritableFilesAndDirectories(writableDirs)
             .setTmpfsDirectories(ImmutableSet.copyOf(getSandboxOptions().sandboxTmpfsPath))
@@ -354,7 +354,7 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
       return new HardlinkedSandboxedSpawn(
           sandboxPath,
           sandboxExecRoot,
-          commandLineBuilder.build(),
+          commandLineBuilder.buildForCommand(spawn.getArguments()),
           environment,
           inputs,
           outputs,
@@ -368,14 +368,15 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
       return new SymlinkedSandboxedSpawn(
           sandboxPath,
           sandboxExecRoot,
-          commandLineBuilder.build(),
+          commandLineBuilder.buildForCommand(spawn.getArguments()),
           environment,
           inputs,
           outputs,
           writableDirs,
           treeDeleter,
           sandboxDebugPath,
           statisticsPath,
+          makeInteractiveDebugArguments(commandLineBuilder, sandboxOptions),
           spawn.getMnemonic());
     }
   }
@@ -538,4 +539,13 @@ public void cleanupSandboxBase(Path sandboxBase, TreeDeleter treeDeleter) throws
 
     super.cleanupSandboxBase(sandboxBase, treeDeleter);
   }
+
+  @Nullable
+  private ImmutableList<String> makeInteractiveDebugArguments(
+      LinuxSandboxCommandLineBuilder commandLineBuilder, SandboxOptions sandboxOptions) {
+    if (!sandboxOptions.sandboxDebug) {
+      return null;
+    }
+    return commandLineBuilder.buildForCommand(ImmutableList.of("/bin/sh", "-i"));
+  }
 }

src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunner.java

@@ -100,24 +100,26 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
     SandboxInputs inputs =
         helpers.processInputFiles(
             context.getInputMapping(PathFragment.EMPTY_FRAGMENT, /* willAccessRepeatedly= */ true),
+            context.getInputMetadataProvider(),
             execRoot,
             execRoot,
             packageRoots,
             null);
     SandboxOutputs outputs = helpers.getOutputs(spawn);
 
-      return new SymlinkedSandboxedSpawn(
-          sandboxPath,
-          sandboxExecRoot,
-          commandLineBuilder.build(),
-          environment,
-          inputs,
-          outputs,
-          getWritableDirs(sandboxExecRoot, sandboxExecRoot, environment),
-          treeDeleter,
-          /* sandboxDebugPath= */ null,
-          statisticsPath,
-          spawn.getMnemonic());
+    return new SymlinkedSandboxedSpawn(
+        sandboxPath,
+        sandboxExecRoot,
+        commandLineBuilder.build(),
+        environment,
+        inputs,
+        outputs,
+        getWritableDirs(sandboxExecRoot, sandboxExecRoot, environment),
+        treeDeleter,
+        /* sandboxDebugPath= */ null,
+        statisticsPath,
+        /* interactiveDebugArguments= */ null,
+        spawn.getMnemonic());
   }
 
   @Override