StandaloneTestStrategy.java

// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.exec;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Verify;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.io.ByteStreams;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputHelper;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
import com.google.devtools.build.lib.actions.ArtifactPathResolver;
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.ExecutionRequirements;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.SimpleSpawn;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnContinuation;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.TestExecException;
import com.google.devtools.build.lib.actions.cache.MetadataHandler;
import com.google.devtools.build.lib.analysis.actions.SpawnAction;
import com.google.devtools.build.lib.analysis.test.TestAttempt;
import com.google.devtools.build.lib.analysis.test.TestResult;
import com.google.devtools.build.lib.analysis.test.TestRunnerAction;
import com.google.devtools.build.lib.analysis.test.TestRunnerAction.ResolvedPaths;
import com.google.devtools.build.lib.analysis.test.TestStrategy;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.TestResult.ExecutionInfo;
import com.google.devtools.build.lib.buildeventstream.TestFileNameConstants;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.server.FailureDetails.Execution.Code;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.TestAction;
import com.google.devtools.build.lib.skyframe.TreeArtifactValue;
import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
import com.google.devtools.build.lib.view.test.TestStatus.TestResultData;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import javax.annotation.Nullable;

/** Runs TestRunnerAction actions. */
// TODO(bazel-team): add tests for this strategy.
public class StandaloneTestStrategy extends TestStrategy {
  private static final ImmutableMap<String, String> ENV_VARS =
      ImmutableMap.<String, String>builder()
          .put("TZ", "UTC")
          .put("TEST_SRCDIR", TestPolicy.RUNFILES_DIR)
          // TODO(lberki): Remove JAVA_RUNFILES and PYTHON_RUNFILES.
          .put("JAVA_RUNFILES", TestPolicy.RUNFILES_DIR)
          .put("PYTHON_RUNFILES", TestPolicy.RUNFILES_DIR)
          .put("RUNFILES_DIR", TestPolicy.RUNFILES_DIR)
          .put("TEST_TMPDIR", TestPolicy.TEST_TMP_DIR)
          .put("RUN_UNDER_RUNFILES", "1")
          .build();

  public static final TestPolicy DEFAULT_LOCAL_POLICY = new TestPolicy(ENV_VARS);

  protected final Path tmpDirRoot;

  public StandaloneTestStrategy(
      ExecutionOptions executionOptions, BinTools binTools, Path tmpDirRoot) {
    super(executionOptions, binTools);
    this.tmpDirRoot = tmpDirRoot;
  }

  @Override
  public TestRunnerSpawn createTestRunnerSpawn(
      TestRunnerAction action, ActionExecutionContext actionExecutionContext)
      throws ExecException, InterruptedException {
    if (action.getExecutionSettings().getInputManifest() == null) {
      String errorMessage = "cannot run local tests with --nobuild_runfile_manifests";
      throw new TestExecException(
          errorMessage,
          FailureDetail.newBuilder()
              .setTestAction(
                  TestAction.newBuilder().setCode(TestAction.Code.LOCAL_TEST_PREREQ_UNMET))
              .setMessage(errorMessage)
              .build());
    }
    Map<String, String> testEnvironment =
        createEnvironment(
            actionExecutionContext, action, tmpDirRoot, executionOptions.splitXmlGeneration);

    Map<String, String> executionInfo =
        new TreeMap<>(action.getTestProperties().getExecutionInfo());
    if (!action.shouldCacheResult()) {
      executionInfo.put(ExecutionRequirements.NO_CACHE, "");
    }
    executionInfo.put(ExecutionRequirements.TIMEOUT, "" + getTimeout(action).getSeconds());

    SimpleSpawn.LocalResourcesSupplier localResourcesSupplier =
        () ->
            action
                .getTestProperties()
                .getLocalResourceUsage(
                    action.getOwner().getLabel(), executionOptions.usingLocalTestJobs());

    Spawn spawn =
        new SimpleSpawn(
            action,
            getArgs(action),
            ImmutableMap.copyOf(testEnvironment),
            ImmutableMap.copyOf(executionInfo),
            action.getRunfilesSupplier(),
            ImmutableMap.of(),
            /*inputs=*/ action.getInputs(),
            NestedSetBuilder.emptySet(Order.STABLE_ORDER),
            createSpawnOutputs(action),
            /*mandatoryOutputs=*/ ImmutableSet.of(),
            localResourcesSupplier);
    Path execRoot = actionExecutionContext.getExecRoot();
    ArtifactPathResolver pathResolver = actionExecutionContext.getPathResolver();
    Path runfilesDir = pathResolver.convertPath(action.getExecutionSettings().getRunfilesDir());
    Path tmpDir = pathResolver.convertPath(tmpDirRoot.getChild(TestStrategy.getTmpDirName(action)));
    Path workingDirectory = runfilesDir.getRelative(action.getRunfilesPrefix());
    return new StandaloneTestRunnerSpawn(
        action, actionExecutionContext, spawn, tmpDir, workingDirectory, execRoot);
  }

  private ImmutableSet<ActionInput> createSpawnOutputs(TestRunnerAction action) {
    ImmutableSet.Builder<ActionInput> builder = ImmutableSet.builder();
    for (ActionInput output : action.getSpawnOutputs()) {
      if (output.getExecPath().equals(action.getXmlOutputPath())) {
        // HACK: Convert type of test.xml from BasicActionInput to DerivedArtifact. We want to
        // inject metadata of test.xml if it is generated remotely and it's currently only possible
        // to inject Artifact.
        builder.add(createArtifactOutput(action, output.getExecPath()));
      } else {
        builder.add(output);
      }
    }
    return builder.build();
  }

  private static ImmutableList<Pair<String, Path>> renameOutputs(
      ActionExecutionContext actionExecutionContext,
      TestRunnerAction action,
      ImmutableList<Pair<String, Path>> testOutputs,
      int attemptId)
      throws IOException {
    // Rename outputs
    String namePrefix =
        FileSystemUtils.removeExtension(action.getTestLog().getExecPath().getBaseName());
    Path testRoot = actionExecutionContext.getInputPath(action.getTestLog()).getParentDirectory();
    Path attemptsDir = testRoot.getChild(namePrefix + "_attempts");
    attemptsDir.createDirectory();
    String attemptPrefix = "attempt_" + attemptId;
    Path testLog = attemptsDir.getChild(attemptPrefix + ".log");

    // Get the normal test output paths, and then update them to use "attempt_N" names, and
    // attemptDir, before adding them to the outputs.
    ImmutableList.Builder<Pair<String, Path>> testOutputsBuilder = new ImmutableList.Builder<>();
    for (Pair<String, Path> testOutput : testOutputs) {
      // e.g. /testRoot/test.dir/file, an example we follow throughout this loop's comments.
      Path testOutputPath = testOutput.getSecond();
      Path destinationPath;
      if (testOutput.getFirst().equals(TestFileNameConstants.TEST_LOG)) {
        // The rename rules for the test log are different than for all the other files.
        destinationPath = testLog;
      } else {
        // e.g. test.dir/file
        PathFragment relativeToTestDirectory = testOutputPath.relativeTo(testRoot);

        // e.g. attempt_1.dir/file
        String destinationPathFragmentStr =
            relativeToTestDirectory.getSafePathString().replaceFirst("test", attemptPrefix);
        PathFragment destinationPathFragment = PathFragment.create(destinationPathFragmentStr);

        // e.g. /attemptsDir/attempt_1.dir/file
        destinationPath = attemptsDir.getRelative(destinationPathFragment);
        destinationPath.getParentDirectory().createDirectory();
      }

      // Move to the destination.
      testOutputPath.renameTo(destinationPath);

      testOutputsBuilder.add(Pair.of(testOutput.getFirst(), destinationPath));
    }
    return testOutputsBuilder.build();
  }

  private StandaloneFailedAttemptResult processFailedTestAttempt(
      int attemptId,
      ActionExecutionContext actionExecutionContext,
      TestRunnerAction action,
      StandaloneTestResult result)
      throws IOException {
    return processTestAttempt(
        attemptId, /*isLastAttempt=*/ false, actionExecutionContext, action, result);
  }

  private void finalizeTest(
      TestRunnerAction action,
      ActionExecutionContext actionExecutionContext,
      StandaloneTestResult standaloneTestResult,
      List<FailedAttemptResult> failedAttempts)
      throws IOException {
    processTestAttempt(
        failedAttempts.size() + 1,
        /*isLastAttempt=*/ true,
        actionExecutionContext,
        action,
        standaloneTestResult);

    TestResultData.Builder dataBuilder = standaloneTestResult.testResultDataBuilder();
    for (FailedAttemptResult failedAttempt : failedAttempts) {
      TestResultData failedAttemptData =
          ((StandaloneFailedAttemptResult) failedAttempt).testResultData;
      dataBuilder.addAllFailedLogs(failedAttemptData.getFailedLogsList());
      dataBuilder.addTestTimes(failedAttemptData.getTestTimes(0));
      dataBuilder.addAllTestProcessTimes(failedAttemptData.getTestProcessTimesList());
    }
    if (dataBuilder.getStatus() == BlazeTestStatus.PASSED && !failedAttempts.isEmpty()) {
      dataBuilder.setStatus(BlazeTestStatus.FLAKY);
    }
    TestResultData data = dataBuilder.build();
    TestResult result =
        new TestResult(action, data, false, standaloneTestResult.primarySystemFailure());
    postTestResult(actionExecutionContext, result);
  }

  private StandaloneFailedAttemptResult processTestAttempt(
      int attemptId,
      boolean isLastAttempt,
      ActionExecutionContext actionExecutionContext,
      TestRunnerAction action,
      StandaloneTestResult result)
      throws IOException {
    ImmutableList<Pair<String, Path>> testOutputs =
        action.getTestOutputsMapping(
            actionExecutionContext.getPathResolver(), actionExecutionContext.getExecRoot());
    if (!isLastAttempt) {
      testOutputs = renameOutputs(actionExecutionContext, action, testOutputs, attemptId);
    }

    // Recover the test log path, which may have been renamed, and add it to the data builder.
    Path renamedTestLog = null;
    for (Pair<String, Path> pair : testOutputs) {
      if (TestFileNameConstants.TEST_LOG.equals(pair.getFirst())) {
        Preconditions.checkState(renamedTestLog == null, "multiple test_log matches");
        renamedTestLog = pair.getSecond();
      }
    }

    TestResultData.Builder dataBuilder = result.testResultDataBuilder();
    // If the test log path does not exist, mark the test as incomplete
    if (renamedTestLog == null) {
      dataBuilder.setStatus(BlazeTestStatus.INCOMPLETE);
    }

    if (dataBuilder.getStatus() == BlazeTestStatus.PASSED) {
      dataBuilder.setPassedLog(renamedTestLog.toString());
    } else if (dataBuilder.getStatus() == BlazeTestStatus.INCOMPLETE) {
      // Incomplete (cancelled) test runs don't have a log.
      Preconditions.checkState(renamedTestLog == null);
    } else {
      dataBuilder.addFailedLogs(renamedTestLog.toString());
    }

    // Add the test log to the output
    TestResultData data = dataBuilder.build();
    actionExecutionContext
        .getEventHandler()
        .post(
            TestAttempt.forExecutedTestResult(
                action, data, attemptId, testOutputs, result.executionInfo(), isLastAttempt));
    processTestOutput(actionExecutionContext, data, action.getTestName(), renamedTestLog);
    return new StandaloneFailedAttemptResult(data);
  }

  private static Map<String, String> setupEnvironment(
      TestRunnerAction action,
      Map<String, String> clientEnv,
      Path execRoot,
      Path runfilesDir,
      Path tmpDir) {
    PathFragment relativeTmpDir;
    if (tmpDir.startsWith(execRoot)) {
      relativeTmpDir = tmpDir.relativeTo(execRoot);
    } else {
      relativeTmpDir = tmpDir.asFragment();
    }
    return DEFAULT_LOCAL_POLICY.computeTestEnvironment(
        action, clientEnv, getTimeout(action), runfilesDir.relativeTo(execRoot), relativeTmpDir);
  }

  static class TestMetadataHandler implements MetadataHandler {
    private final MetadataHandler metadataHandler;
    private final ImmutableSet<Artifact> outputs;
    private final ConcurrentMap<Artifact, FileArtifactValue> fileMetadataMap =
        new ConcurrentHashMap<>();

    TestMetadataHandler(MetadataHandler metadataHandler, ImmutableSet<Artifact> outputs) {
      this.metadataHandler = metadataHandler;
      this.outputs = outputs;
    }

    @Nullable
    @Override
    public ActionInput getInput(String execPath) {
      return metadataHandler.getInput(execPath);
    }

    @Nullable
    @Override
    public FileArtifactValue getMetadata(ActionInput input) throws IOException {
      return metadataHandler.getMetadata(input);
    }

    @Override
    public void setDigestForVirtualArtifact(Artifact artifact, byte[] digest) {
      metadataHandler.setDigestForVirtualArtifact(artifact, digest);
    }

    @Override
    public FileArtifactValue constructMetadataForDigest(
        Artifact output, FileStatus statNoFollow, byte[] injectedDigest) throws IOException {
      return metadataHandler.constructMetadataForDigest(output, statNoFollow, injectedDigest);
    }

    @Override
    public ImmutableSet<TreeFileArtifact> getTreeArtifactChildren(SpecialArtifact treeArtifact) {
      return metadataHandler.getTreeArtifactChildren(treeArtifact);
    }

    @Override
    public TreeArtifactValue getTreeArtifactValue(SpecialArtifact treeArtifact) throws IOException {
      return metadataHandler.getTreeArtifactValue(treeArtifact);
    }

    @Override
    public void markOmitted(Artifact output) {
      metadataHandler.markOmitted(output);
    }

    @Override
    public boolean artifactOmitted(Artifact artifact) {
      return metadataHandler.artifactOmitted(artifact);
    }

    @Override
    public void resetOutputs(Iterable<? extends Artifact> outputs) {
      metadataHandler.resetOutputs(outputs);
    }

    @Override
    public void injectFile(Artifact output, FileArtifactValue metadata) {
      if (outputs.contains(output)) {
        metadataHandler.injectFile(output, metadata);
      }
      fileMetadataMap.put(output, metadata);
    }

    @Override
    public void injectTree(SpecialArtifact output, TreeArtifactValue tree) {
      metadataHandler.injectTree(output, tree);
    }

    public boolean fileInjected(Artifact output) {
      return fileMetadataMap.containsKey(output);
    }
  }

  private TestAttemptContinuation beginTestAttempt(
      TestRunnerAction testAction,
      Spawn spawn,
      ActionExecutionContext actionExecutionContext,
      Path execRoot)
      throws IOException, InterruptedException {
    ResolvedPaths resolvedPaths = testAction.resolve(execRoot);
    Path out = actionExecutionContext.getInputPath(testAction.getTestLog());
    Path err = resolvedPaths.getTestStderr();
    FileOutErr testOutErr = new FileOutErr(out, err);
    Closeable streamed = null;
    if (executionOptions.testOutput.equals(ExecutionOptions.TestOutputFormat.STREAMED)) {
      streamed =
          createStreamedTestOutput(
              Reporter.outErrForReporter(actionExecutionContext.getEventHandler()), out);
    }

    // We use TestMetadataHandler here mainly because the one provided by actionExecutionContext
    // doesn't allow to inject undeclared outputs and test.xml is undeclared by the test action.
    TestMetadataHandler testMetadataHandler = null;
    if (actionExecutionContext.getMetadataHandler() != null) {
      testMetadataHandler =
          new TestMetadataHandler(
              actionExecutionContext.getMetadataHandler(), testAction.getOutputs());
    }

    long startTimeMillis = actionExecutionContext.getClock().currentTimeMillis();
    SpawnStrategyResolver resolver = actionExecutionContext.getContext(SpawnStrategyResolver.class);
    SpawnContinuation spawnContinuation;
    try {
      spawnContinuation =
          resolver.beginExecution(
              spawn,
              actionExecutionContext
                  .withFileOutErr(testOutErr)
                  .withMetadataHandler(testMetadataHandler));
    } catch (InterruptedException e) {
      if (streamed != null) {
        streamed.close();
      }
      testOutErr.close();
      throw e;
    }
    return new BazelTestAttemptContinuation(
        testAction,
        testMetadataHandler,
        actionExecutionContext,
        spawn,
        resolvedPaths,
        testOutErr,
        streamed,
        startTimeMillis,
        spawnContinuation,
        /* testResultDataBuilder= */ null,
        /* spawnResults= */ null);
  }

  private static void appendCoverageLog(FileOutErr coverageOutErr, FileOutErr outErr)
      throws IOException {
    writeOutFile(coverageOutErr.getErrorPath(), outErr.getOutputPath());
    writeOutFile(coverageOutErr.getOutputPath(), outErr.getOutputPath());
  }

  private static void writeOutFile(Path inFilePath, Path outFilePath) throws IOException {
    FileStatus stat = inFilePath.statNullable();
    if (stat != null) {
      try {
        if (stat.getSize() > 0) {
          if (outFilePath.exists()) {
            outFilePath.setWritable(true);
          }
          try (OutputStream out = outFilePath.getOutputStream(true);
              InputStream in = inFilePath.getInputStream()) {
            ByteStreams.copy(in, out);
          }
        }
      } finally {
        inFilePath.delete();
      }
    }
  }

  private static BuildEventStreamProtos.TestResult.ExecutionInfo extractExecutionInfo(
      SpawnResult spawnResult, TestResultData.Builder result) {
    BuildEventStreamProtos.TestResult.ExecutionInfo.Builder executionInfo =
        BuildEventStreamProtos.TestResult.ExecutionInfo.newBuilder();
    if (spawnResult.isCacheHit()) {
      result.setRemotelyCached(true);
      executionInfo.setCachedRemotely(true);
    }
    String strategy = spawnResult.getRunnerName();
    if (strategy != null) {
      executionInfo.setStrategy(strategy);
      result.setIsRemoteStrategy(strategy.equals("remote"));
    }
    if (spawnResult.getExecutorHostName() != null) {
      executionInfo.setHostname(spawnResult.getExecutorHostName());
    }
    return executionInfo.build();
  }

  private static Artifact.DerivedArtifact createArtifactOutput(
      TestRunnerAction action, PathFragment outputPath) {
    Artifact.DerivedArtifact testLog = (Artifact.DerivedArtifact) action.getTestLog();
    return DerivedArtifact.create(testLog.getRoot(), outputPath, testLog.getArtifactOwner());
  }

  /**
   * A spawn to generate a test.xml file from the test log. This is only used if the test does not
   * generate a test.xml file itself.
   */
  private static Spawn createXmlGeneratingSpawn(
      TestRunnerAction action, ImmutableMap<String, String> testEnv, SpawnResult result) {
    ImmutableList<String> args =
        ImmutableList.of(
            action.getTestXmlGeneratorScript().getExecPath().getCallablePathString(),
            action.getTestLog().getExecPathString(),
            action.getXmlOutputPath().getPathString(),
            Long.toString(result.getWallTime().orElse(Duration.ZERO).getSeconds()),
            Integer.toString(result.exitCode()));
    ImmutableMap.Builder<String, String> envBuilder = ImmutableMap.builder();
    // "PATH" and "TEST_BINARY" are also required, they should always be set in testEnv.
    Preconditions.checkArgument(testEnv.containsKey("PATH"));
    Preconditions.checkArgument(testEnv.containsKey("TEST_BINARY"));
    envBuilder.putAll(testEnv).put("TEST_NAME", action.getTestName());
    // testEnv only contains TEST_SHARD_INDEX and TEST_TOTAL_SHARDS if the test action is sharded,
    // we need to set the default value when the action isn't sharded.
    if (!action.isSharded()) {
      envBuilder.put("TEST_SHARD_INDEX", "0");
      envBuilder.put("TEST_TOTAL_SHARDS", "0");
    }
    Map<String, String> executionInfo =
        Maps.newHashMapWithExpectedSize(action.getExecutionInfo().size() + 1);
    executionInfo.putAll(action.getExecutionInfo());
    if (result.exitCode() != 0) {
      // If the test is failed, the spawn shouldn't use remote cache since the test.xml file is
      // renamed immediately after the spawn execution. If there is another test attempt, the async
      // upload will fail because it cannot read the file at original position.
      executionInfo.put(ExecutionRequirements.NO_REMOTE_CACHE, "");
    }
    return new SimpleSpawn(
        action,
        args,
        envBuilder.buildOrThrow(),
        // Pass the execution info of the action which is identical to the supported tags set on the
        // test target. In particular, this does not set the test timeout on the spawn.
        ImmutableMap.copyOf(executionInfo),
        null,
        ImmutableMap.of(),
        /*inputs=*/ NestedSetBuilder.create(
            Order.STABLE_ORDER, action.getTestXmlGeneratorScript(), action.getTestLog()),
        /*tools=*/ NestedSetBuilder.emptySet(Order.STABLE_ORDER),
        /*outputs=*/ ImmutableSet.of(createArtifactOutput(action, action.getXmlOutputPath())),
        /*mandatoryOutputs=*/ null,
        SpawnAction.DEFAULT_RESOURCE_SET);
  }

  private static Spawn createCoveragePostProcessingSpawn(
      ActionExecutionContext actionExecutionContext,
      TestRunnerAction action,
      List<ActionInput> expandedCoverageDir,
      Path tmpDirRoot,
      boolean splitXmlGeneration) {
    ImmutableList<String> args =
        ImmutableList.of(action.getCollectCoverageScript().getExecPathString());

    Map<String, String> testEnvironment =
        createEnvironment(actionExecutionContext, action, tmpDirRoot, splitXmlGeneration);

    testEnvironment.put("TEST_SHARD_INDEX", Integer.toString(action.getShardNum()));
    testEnvironment.put(
        "TEST_TOTAL_SHARDS", Integer.toString(action.getExecutionSettings().getTotalShards()));
    testEnvironment.put("TEST_NAME", action.getTestName());
    testEnvironment.put("IS_COVERAGE_SPAWN", "1");
    return new SimpleSpawn(
        action,
        args,
        ImmutableMap.copyOf(testEnvironment),
        action.getExecutionInfo(),
        action.getLcovMergerRunfilesSupplier(),
        /*filesetMappings=*/ ImmutableMap.of(),
        /*inputs=*/ NestedSetBuilder.<ActionInput>compileOrder()
            .addTransitive(action.getInputs())
            .addAll(expandedCoverageDir)
            .add(action.getCollectCoverageScript())
            .add(action.getCoverageDirectoryTreeArtifact())
            .add(action.getCoverageManifest())
            .addTransitive(action.getLcovMergerFilesToRun().build())
            .build(),
        /*tools=*/ NestedSetBuilder.emptySet(Order.STABLE_ORDER),
        /*outputs=*/ ImmutableSet.of(
            ActionInputHelper.fromPath(action.getCoverageData().getExecPath())),
        /*mandatoryOutputs=*/ null,
        SpawnAction.DEFAULT_RESOURCE_SET);
  }

  private static Map<String, String> createEnvironment(
      ActionExecutionContext actionExecutionContext,
      TestRunnerAction action,
      Path tmpDirRoot,
      boolean splitXmlGeneration) {
    Path execRoot = actionExecutionContext.getExecRoot();
    ArtifactPathResolver pathResolver = actionExecutionContext.getPathResolver();
    Path runfilesDir = pathResolver.convertPath(action.getExecutionSettings().getRunfilesDir());
    Path tmpDir = pathResolver.convertPath(tmpDirRoot.getChild(TestStrategy.getTmpDirName(action)));
    Map<String, String> testEnvironment =
        setupEnvironment(
            action, actionExecutionContext.getClientEnv(), execRoot, runfilesDir, tmpDir);
    if (splitXmlGeneration) {
      testEnvironment.put("EXPERIMENTAL_SPLIT_XML_GENERATION", "1");
    }
    return testEnvironment;
  }

  @Override
  public TestResult newCachedTestResult(
      Path execRoot, TestRunnerAction action, TestResultData data) {
    return new TestResult(action, data, /*cached*/ true, execRoot, /*systemFailure=*/ null);
  }

  @VisibleForTesting
  static final class StandaloneFailedAttemptResult implements FailedAttemptResult {
    private final TestResultData testResultData;

    StandaloneFailedAttemptResult(TestResultData testResultData) {
      this.testResultData = testResultData;
    }

    TestResultData testResultData() {
      return testResultData;
    }
  }

  private final class StandaloneTestRunnerSpawn implements TestRunnerSpawn {
    private final TestRunnerAction testAction;
    private final ActionExecutionContext actionExecutionContext;
    private final Spawn spawn;
    private final Path tmpDir;
    private final Path workingDirectory;
    private final Path execRoot;

    StandaloneTestRunnerSpawn(
        TestRunnerAction testAction,
        ActionExecutionContext actionExecutionContext,
        Spawn spawn,
        Path tmpDir,
        Path workingDirectory,
        Path execRoot) {
      this.testAction = testAction;
      this.actionExecutionContext = actionExecutionContext;
      this.spawn = spawn;
      this.tmpDir = tmpDir;
      this.workingDirectory = workingDirectory;
      this.execRoot = execRoot;
    }

    @Override
    public ActionExecutionContext getActionExecutionContext() {
      return actionExecutionContext;
    }

    @Override
    public TestAttemptContinuation beginExecution() throws InterruptedException, IOException {
      prepareFileSystem(testAction, execRoot, tmpDir, workingDirectory);
      return beginTestAttempt(testAction, spawn, actionExecutionContext, execRoot);
    }

    @Override
    public int getMaxAttempts(TestAttemptResult firstTestAttemptResult) {
      return getTestAttempts(testAction);
    }

    @Override
    public FailedAttemptResult finalizeFailedTestAttempt(
        TestAttemptResult testAttemptResult, int attempt) throws IOException {
      return processFailedTestAttempt(
          attempt, actionExecutionContext, testAction, (StandaloneTestResult) testAttemptResult);
    }

    @Override
    public void finalizeTest(
        TestAttemptResult finalResult, List<FailedAttemptResult> failedAttempts)
        throws IOException {
      StandaloneTestStrategy.this.finalizeTest(
          testAction, actionExecutionContext, (StandaloneTestResult) finalResult, failedAttempts);
    }

    @Override
    public void finalizeCancelledTest(List<FailedAttemptResult> failedAttempts) throws IOException {
      TestResultData.Builder builder =
          TestResultData.newBuilder()
              .setCachable(false)
              .setTestPassed(false)
              .setStatus(BlazeTestStatus.INCOMPLETE);
      StandaloneTestResult standaloneTestResult =
          StandaloneTestResult.builder()
              .setSpawnResults(ImmutableList.of())
              .setTestResultDataBuilder(builder)
              .setExecutionInfo(ExecutionInfo.getDefaultInstance())
              .build();
      finalizeTest(standaloneTestResult, failedAttempts);
    }
  }

  private final class BazelTestAttemptContinuation extends TestAttemptContinuation {
    private final TestRunnerAction testAction;
    @Nullable private final TestMetadataHandler testMetadataHandler;
    private final ActionExecutionContext actionExecutionContext;
    private final Spawn spawn;
    private final ResolvedPaths resolvedPaths;
    private final FileOutErr fileOutErr;
    private final Closeable streamed;
    private final long startTimeMillis;
    private final SpawnContinuation spawnContinuation;
    private TestResultData.Builder testResultDataBuilder;
    private ImmutableList<SpawnResult> spawnResults;

    BazelTestAttemptContinuation(
        TestRunnerAction testAction,
        @Nullable TestMetadataHandler testMetadataHandler,
        ActionExecutionContext actionExecutionContext,
        Spawn spawn,
        ResolvedPaths resolvedPaths,
        FileOutErr fileOutErr,
        Closeable streamed,
        long startTimeMillis,
        SpawnContinuation spawnContinuation,
        TestResultData.Builder testResultDataBuilder,
        ImmutableList<SpawnResult> spawnResults) {
      this.testAction = testAction;
      this.testMetadataHandler = testMetadataHandler;
      this.actionExecutionContext = actionExecutionContext;
      this.spawn = spawn;
      this.resolvedPaths = resolvedPaths;
      this.fileOutErr = fileOutErr;
      this.streamed = streamed;
      this.startTimeMillis = startTimeMillis;
      this.spawnContinuation = spawnContinuation;
      this.testResultDataBuilder = testResultDataBuilder;
      this.spawnResults = spawnResults;
    }

    @Nullable
    @Override
    public ListenableFuture<?> getFuture() {
      return spawnContinuation.getFuture();
    }

    @Override
    public TestAttemptContinuation execute()
        throws InterruptedException, ExecException, IOException {

      if (testResultDataBuilder == null) {
        // We have two protos to represent test attempts:
        // 1. com.google.devtools.build.lib.view.test.TestStatus.TestResultData represents both
        //    failed attempts and finished tests. Bazel stores this to disk to persist cached test
        //    result information across server restarts.
        // 2. com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.TestResult
        //    represents only individual attempts (failed or not). Bazel reports this as an event to
        //    the Build Event Protocol, but never saves it to disk.
        //
        // The TestResult proto is always constructed from a TestResultData instance, either one
        // that is created right here, or one that is read back from disk.
        TestResultData.Builder builder = null;
        ImmutableList<SpawnResult> spawnResults;
        try {
          SpawnContinuation nextContinuation = spawnContinuation.execute();
          if (!nextContinuation.isDone()) {
            return new BazelTestAttemptContinuation(
                testAction,
                testMetadataHandler,
                actionExecutionContext,
                spawn,
                resolvedPaths,
                fileOutErr,
                streamed,
                startTimeMillis,
                nextContinuation,
                builder,
                /* spawnResults= */ null);
          }
          spawnResults = nextContinuation.get();
          builder = TestResultData.newBuilder();
          builder.setCachable(true).setTestPassed(true).setStatus(BlazeTestStatus.PASSED);
        } catch (SpawnExecException e) {
          if (e.isCatastrophic()) {
            closeSuppressed(e, streamed);
            closeSuppressed(e, fileOutErr);
            throw e;
          }
          if (!e.getSpawnResult().setupSuccess()) {
            closeSuppressed(e, streamed);
            closeSuppressed(e, fileOutErr);
            // Rethrow as the test could not be run and thus there's no point in retrying.
            throw e;
          }
          spawnResults = ImmutableList.of(e.getSpawnResult());
          builder = TestResultData.newBuilder();
          builder
              .setCachable(e.getSpawnResult().status().isConsideredUserError())
              .setTestPassed(false)
              .setStatus(e.hasTimedOut() ? BlazeTestStatus.TIMEOUT : BlazeTestStatus.FAILED);
        } catch (InterruptedException e) {
          closeSuppressed(e, streamed);
          closeSuppressed(e, fileOutErr);
          throw e;
        }
        long endTimeMillis = actionExecutionContext.getClock().currentTimeMillis();

        // SpawnActionContext guarantees the first entry to correspond to the spawn passed in (there
        // may be additional entries due to tree artifact handling).
        SpawnResult primaryResult = spawnResults.get(0);

        // The SpawnResult of a remotely cached or remotely executed action may not have walltime
        // set. We fall back to the time measured here for backwards compatibility.
        long durationMillis = endTimeMillis - startTimeMillis;
        durationMillis =
            primaryResult.getWallTime().orElse(Duration.ofMillis(durationMillis)).toMillis();

        builder
            .setStartTimeMillisEpoch(startTimeMillis)
            .addTestTimes(durationMillis)
            .addTestProcessTimes(durationMillis)
            .setRunDurationMillis(durationMillis)
            .setHasCoverage(testAction.isCoverageMode());

        if (testAction.isCoverageMode() && testAction.getSplitCoveragePostProcessing()) {
          actionExecutionContext
              .getMetadataHandler()
              .getMetadata(testAction.getCoverageDirectoryTreeArtifact());
          ImmutableSet<? extends ActionInput> expandedCoverageDir =
              actionExecutionContext
                  .getMetadataHandler()
                  .getTreeArtifactChildren(
                      (SpecialArtifact) testAction.getCoverageDirectoryTreeArtifact());
          Spawn coveragePostProcessingSpawn =
              createCoveragePostProcessingSpawn(
                  actionExecutionContext,
                  testAction,
                  ImmutableList.copyOf(expandedCoverageDir),
                  tmpDirRoot,
                  executionOptions.splitXmlGeneration);
          SpawnStrategyResolver spawnStrategyResolver =
              actionExecutionContext.getContext(SpawnStrategyResolver.class);

          Path testRoot =
              actionExecutionContext.getInputPath(testAction.getTestLog()).getParentDirectory();

          Path out = testRoot.getChild("coverage.log");
          Path err = testRoot.getChild("coverage.err");
          FileOutErr coverageOutErr = new FileOutErr(out, err);
          ActionExecutionContext actionExecutionContextWithCoverageFileOutErr =
              actionExecutionContext.withFileOutErr(coverageOutErr);

          SpawnContinuation coveragePostProcessingContinuation =
              spawnStrategyResolver.beginExecution(
                  coveragePostProcessingSpawn, actionExecutionContextWithCoverageFileOutErr);
          writeOutFile(coverageOutErr.getErrorPath(), coverageOutErr.getOutputPath());
          appendCoverageLog(coverageOutErr, fileOutErr);
          return new BazelCoveragePostProcessingContinuation(
              testAction,
              testMetadataHandler,
              actionExecutionContext,
              spawn,
              resolvedPaths,
              fileOutErr,
              streamed,
              builder,
              spawnResults,
              coveragePostProcessingContinuation);
        } else {
          this.spawnResults = spawnResults;
          this.testResultDataBuilder = builder;
        }
      }

      Verify.verify(
          !(testAction.isCoverageMode() && testAction.getSplitCoveragePostProcessing())
              || testAction.getCoverageData().getPath().exists());
      Verify.verifyNotNull(spawnResults);
      Verify.verifyNotNull(testResultDataBuilder);

      try {
        if (!fileOutErr.hasRecordedOutput()) {
          // Make sure that the test.log exists.
          FileSystemUtils.touchFile(fileOutErr.getOutputPath());
        }
        // Append any error output to the test.log. This is very rare.
        writeOutFile(fileOutErr.getErrorPath(), fileOutErr.getOutputPath());
        fileOutErr.close();
        if (streamed != null) {
          streamed.close();
        }
      } catch (IOException e) {
        throw new EnvironmentalExecException(e, Code.TEST_OUT_ERR_IO_EXCEPTION);
      }

      Path xmlOutputPath = resolvedPaths.getXmlOutputPath();
      boolean testXmlGenerated = xmlOutputPath.exists();
      if (!testXmlGenerated && testMetadataHandler != null) {
        testXmlGenerated =
            testMetadataHandler.fileInjected(
                createArtifactOutput(testAction, testAction.getXmlOutputPath()));
      }

      // If the test did not create a test.xml, and --experimental_split_xml_generation is enabled,
      // then we run a separate action to create a test.xml from test.log. We do this as a spawn
      // rather than doing it locally in-process, as the test.log file may only exist remotely (when
      // remote execution is enabled), and we do not want to have to download it.
      if (executionOptions.splitXmlGeneration
          && fileOutErr.getOutputPath().exists()
          && !testXmlGenerated) {
        Spawn xmlGeneratingSpawn =
            createXmlGeneratingSpawn(testAction, spawn.getEnvironment(), spawnResults.get(0));
        SpawnStrategyResolver spawnStrategyResolver =
            actionExecutionContext.getContext(SpawnStrategyResolver.class);
        // We treat all failures to generate the test.xml here as catastrophic, and won't rerun
        // the test if this fails. We redirect the output to a temporary file.
        FileOutErr xmlSpawnOutErr = actionExecutionContext.getFileOutErr().childOutErr();
        try {
          SpawnContinuation xmlContinuation =
              spawnStrategyResolver.beginExecution(
                  xmlGeneratingSpawn,
                  actionExecutionContext
                      .withFileOutErr(xmlSpawnOutErr)
                      .withMetadataHandler(testMetadataHandler));
          return new BazelXmlCreationContinuation(
              resolvedPaths, xmlSpawnOutErr, testResultDataBuilder, spawnResults, xmlContinuation);
        } catch (InterruptedException e) {
          closeSuppressed(e, xmlSpawnOutErr);
          throw e;
        }
      }

      TestCase details = parseTestResult(xmlOutputPath);
      if (details != null) {
        testResultDataBuilder.setTestCase(details);
      }

      BuildEventStreamProtos.TestResult.ExecutionInfo executionInfo =
          extractExecutionInfo(spawnResults.get(0), testResultDataBuilder);
      StandaloneTestResult standaloneTestResult =
          StandaloneTestResult.builder()
              .setSpawnResults(spawnResults)
              // We return the TestResultData.Builder rather than the finished TestResultData
              // instance, as we may have to rename the output files in case the test needs to be
              // rerun (if it failed here _and_ is marked flaky _and_ the number of flaky attempts
              // is larger than 1).
              .setTestResultDataBuilder(testResultDataBuilder)
              .setExecutionInfo(executionInfo)
              .build();
      return TestAttemptContinuation.of(standaloneTestResult);
    }
  }

  private final class BazelXmlCreationContinuation extends TestAttemptContinuation {
    private final ResolvedPaths resolvedPaths;
    private final FileOutErr fileOutErr;
    private final TestResultData.Builder builder;
    private final List<SpawnResult> primarySpawnResults;
    private final SpawnContinuation spawnContinuation;

    BazelXmlCreationContinuation(
        ResolvedPaths resolvedPaths,
        FileOutErr fileOutErr,
        TestResultData.Builder builder,
        List<SpawnResult> primarySpawnResults,
        SpawnContinuation spawnContinuation) {
      this.resolvedPaths = resolvedPaths;
      this.fileOutErr = fileOutErr;
      this.builder = builder;
      this.primarySpawnResults = primarySpawnResults;
      this.spawnContinuation = spawnContinuation;
    }

    @Nullable
    @Override
    public ListenableFuture<?> getFuture() {
      return spawnContinuation.getFuture();
    }

    @Override
    public TestAttemptContinuation execute() throws InterruptedException, ExecException {
      SpawnContinuation nextContinuation;
      try {
        nextContinuation = spawnContinuation.execute();
        if (!nextContinuation.isDone()) {
          return new BazelXmlCreationContinuation(
              resolvedPaths, fileOutErr, builder, primarySpawnResults, nextContinuation);
        }
      } catch (ExecException | InterruptedException e) {
        closeSuppressed(e, fileOutErr);
        throw e;
      }

      ImmutableList.Builder<SpawnResult> spawnResults = ImmutableList.builder();
      spawnResults.addAll(primarySpawnResults);
      spawnResults.addAll(nextContinuation.get());

      Path xmlOutputPath = resolvedPaths.getXmlOutputPath();
      TestCase details = parseTestResult(xmlOutputPath);
      if (details != null) {
        builder.setTestCase(details);
      }

      BuildEventStreamProtos.TestResult.ExecutionInfo executionInfo =
          extractExecutionInfo(primarySpawnResults.get(0), builder);
      StandaloneTestResult standaloneTestResult =
          StandaloneTestResult.builder()
              .setSpawnResults(spawnResults.build())
              // We return the TestResultData.Builder rather than the finished TestResultData
              // instance, as we may have to rename the output files in case the test needs to be
              // rerun (if it failed here _and_ is marked flaky _and_ the number of flaky attempts
              // is larger than 1).
              .setTestResultDataBuilder(builder)
              .setExecutionInfo(executionInfo)
              .build();
      return TestAttemptContinuation.of(standaloneTestResult);
    }
  }

  private final class BazelCoveragePostProcessingContinuation extends TestAttemptContinuation {
    private final ResolvedPaths resolvedPaths;
    @Nullable private final TestMetadataHandler testMetadataHandler;
    private final FileOutErr fileOutErr;
    private final Closeable streamed;
    private final TestResultData.Builder testResultDataBuilder;
    private final ImmutableList<SpawnResult> primarySpawnResults;
    private final SpawnContinuation spawnContinuation;
    private final TestRunnerAction testAction;
    private final ActionExecutionContext actionExecutionContext;
    private final Spawn spawn;

    BazelCoveragePostProcessingContinuation(
        TestRunnerAction testAction,
        @Nullable TestMetadataHandler testMetadataHandler,
        ActionExecutionContext actionExecutionContext,
        Spawn spawn,
        ResolvedPaths resolvedPaths,
        FileOutErr fileOutErr,
        Closeable streamed,
        TestResultData.Builder testResultDataBuilder,
        ImmutableList<SpawnResult> primarySpawnResults,
        SpawnContinuation spawnContinuation) {
      this.testAction = testAction;
      this.testMetadataHandler = testMetadataHandler;
      this.actionExecutionContext = actionExecutionContext;
      this.spawn = spawn;
      this.resolvedPaths = resolvedPaths;
      this.fileOutErr = fileOutErr;
      this.streamed = streamed;
      this.testResultDataBuilder = testResultDataBuilder;
      this.primarySpawnResults = primarySpawnResults;
      this.spawnContinuation = spawnContinuation;
    }

    @Nullable
    @Override
    public ListenableFuture<?> getFuture() {
      return spawnContinuation.getFuture();
    }

    @Override
    public TestAttemptContinuation execute() throws InterruptedException, ExecException {
      SpawnContinuation nextContinuation = null;
      try {
        nextContinuation = spawnContinuation.execute();
        if (!nextContinuation.isDone()) {
          return new BazelCoveragePostProcessingContinuation(
              testAction,
              testMetadataHandler,
              actionExecutionContext,
              spawn,
              resolvedPaths,
              fileOutErr,
              streamed,
              testResultDataBuilder,
              ImmutableList.<SpawnResult>builder()
                  .addAll(primarySpawnResults)
                  .addAll(nextContinuation.get())
                  .build(),
              nextContinuation);
        }
      } catch (SpawnExecException e) {
        if (e.isCatastrophic()) {
          closeSuppressed(e, streamed);
          closeSuppressed(e, fileOutErr);
          throw e;
        }
        if (!e.getSpawnResult().setupSuccess()) {
          closeSuppressed(e, streamed);
          closeSuppressed(e, fileOutErr);
          // Rethrow as the test could not be run and thus there's no point in retrying.
          throw e;
        }
        testResultDataBuilder
            .setCachable(e.getSpawnResult().status().isConsideredUserError())
            .setTestPassed(false)
            .setStatus(e.hasTimedOut() ? BlazeTestStatus.TIMEOUT : BlazeTestStatus.FAILED);
      } catch (ExecException | InterruptedException e) {
        closeSuppressed(e, fileOutErr);
        closeSuppressed(e, streamed);
        throw e;
      }

      return new BazelTestAttemptContinuation(
          testAction,
          testMetadataHandler,
          actionExecutionContext,
          spawn,
          resolvedPaths,
          fileOutErr,
          streamed,
          /* startTimeMillis= */ 0,
          nextContinuation,
          testResultDataBuilder,
          primarySpawnResults);
    }
  }
}