Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix output base under /tmp. #20603

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -171,9 +171,12 @@ protected boolean couldBeModifiedByMetadata(FileArtifactValue lastKnown) {
*
* <p>If present, this artifact is a copy of another artifact. It is still tracked as a
* non-symlink by Bazel, but materialized in the local filesystem as a symlink to the original
* artifact, whose contents live at this location. This is used by {@link
* artifact, whose contents live at this location.
*
* <p>This is used by {@link
* com.google.devtools.build.lib.remote.AbstractActionInputPrefetcher} to implement zero-cost
* copies of remotely stored artifacts.
* copies of remotely stored artifacts and to tack which real files (artifact or otherwise)
* resolved symlink artifacts point to.
*/
public Optional<PathFragment> getMaterializationExecPath() {
return Optional.empty();
Expand Down Expand Up @@ -214,6 +217,12 @@ public static FileArtifactValue createForSourceArtifact(
xattrProvider);
}

public static FileArtifactValue createForResolvedSymlink(
PathFragment realPath, FileArtifactValue metadata, @Nullable byte[] digest) {
return new ResolvedSymlinkFileArtifactValue(
realPath, digest, metadata.getContentsProxy(), metadata.getSize());
}

public static FileArtifactValue createFromInjectedDigest(
FileArtifactValue metadata, @Nullable byte[] digest) {
return createForNormalFile(digest, metadata.getContentsProxy(), metadata.getSize());
Expand Down Expand Up @@ -439,7 +448,22 @@ public String toString() {
}
}

private static final class RegularFileArtifactValue extends FileArtifactValue {
private static final class ResolvedSymlinkFileArtifactValue extends RegularFileArtifactValue {
private final PathFragment realPath;

private ResolvedSymlinkFileArtifactValue(PathFragment realPath, @Nullable byte[] digest,
@Nullable FileContentsProxy proxy, long size) {
super(digest, proxy, size);
this.realPath = realPath;
}

@Override
public Optional<PathFragment> getMaterializationExecPath() {
return Optional.of(realPath);
}
}

private static class RegularFileArtifactValue extends FileArtifactValue {
private final byte[] digest;
@Nullable private final FileContentsProxy proxy;
private final long size;
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/sandbox/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ java_library(
deps = [
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
"//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
"//src/main/java/com/google/devtools/build/lib/analysis:test/test_configuration",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/vfs",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
SandboxInputs inputs =
helpers.processInputFiles(
context.getInputMapping(PathFragment.EMPTY_FRAGMENT, /* willAccessRepeatedly= */ true),
context.getInputMetadataProvider(),
execRoot,
execRoot,
packageRoots,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
SandboxInputs inputs =
helpers.processInputFiles(
context.getInputMapping(PathFragment.EMPTY_FRAGMENT, /* willAccessRepeatedly= */ true),
context.getInputMetadataProvider(),
execRoot,
execRoot,
packageRoots,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,7 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
SandboxInputs inputs =
helpers.processInputFiles(
context.getInputMapping(PathFragment.EMPTY_FRAGMENT, /* willAccessRepeatedly= */ true),
context.getInputMetadataProvider(),
execRoot,
withinSandboxExecRoot,
packageRoots,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
SandboxInputs inputs =
helpers.processInputFiles(
context.getInputMapping(PathFragment.EMPTY_FRAGMENT, /* willAccessRepeatedly= */ true),
context.getInputMetadataProvider(),
execRoot,
execRoot,
packageRoots,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import static com.google.devtools.build.lib.vfs.Dirent.Type.SYMLINK;

import com.google.auto.value.AutoValue;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
Expand All @@ -29,6 +30,8 @@
import com.google.common.flogger.GoogleLogger;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.InputMetadataProvider;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
Expand Down Expand Up @@ -444,6 +447,29 @@ private static RootedPath processFilesetSymlink(
symlink.getPathString()));
}

private static RootedPath processResolvedSymlink(
Root absoluteRoot,
PathFragment execRootRelativeSymlinkTarget,
Root execRootWithinSandbox,
PathFragment execRootFragment,
ImmutableList<Root> packageRoots,
Function<Root, Root> sourceRooWithinSandbox) {
PathFragment symlinkTarget = execRootFragment.getRelative(execRootRelativeSymlinkTarget);
for (Root packageRoot : packageRoots) {
if (packageRoot.contains(symlinkTarget)) {
return RootedPath.toRootedPath(
sourceRooWithinSandbox.apply(packageRoot),
packageRoot.relativize(symlinkTarget));
}
}

if (symlinkTarget.startsWith(execRootFragment)) {
return RootedPath.toRootedPath(execRootWithinSandbox, symlinkTarget.relativeTo(execRootFragment));
}

return RootedPath.toRootedPath(absoluteRoot, symlinkTarget);
}

/**
* Returns the inputs of a Spawn as a map of PathFragments relative to an execRoot to paths in the
* host filesystem where the input files can be found.
Expand All @@ -458,6 +484,7 @@ private static RootedPath processFilesetSymlink(
*/
public SandboxInputs processInputFiles(
Map<PathFragment, ActionInput> inputMap,
InputMetadataProvider inputMetadataProvider,
Path execRootPath,
Path withinSandboxExecRootPath,
ImmutableList<Root> packageRoots,
Expand All @@ -468,12 +495,24 @@ public SandboxInputs processInputFiles(
withinSandboxExecRootPath.equals(execRootPath)
? withinSandboxExecRoot
: Root.fromPath(execRootPath);
Root absoluteRoot = Root.absoluteRoot(execRootPath.getFileSystem());

Map<PathFragment, RootedPath> inputFiles = new TreeMap<>();
Map<PathFragment, PathFragment> inputSymlinks = new TreeMap<>();
Map<VirtualActionInput, byte[]> virtualInputs = new HashMap<>();
Map<Root, Root> sourceRootToSandboxSourceRoot = new TreeMap<>();

Function<Root, Root> sourceRootWithinSandbox = r -> {
if (!sourceRootToSandboxSourceRoot.containsKey(r)) {
int next = sourceRootToSandboxSourceRoot.size();
sourceRootToSandboxSourceRoot.put(
r,
Root.fromPath(sandboxSourceRoots.getRelative(Integer.toString(next))));
}

return sourceRootToSandboxSourceRoot.get(r);
};

for (Map.Entry<PathFragment, ActionInput> e : inputMap.entrySet()) {
if (Thread.interrupted()) {
throw new InterruptedException();
Expand Down Expand Up @@ -503,23 +542,47 @@ public SandboxInputs processInputFiles(

if (actionInput instanceof EmptyActionInput) {
inputPath = null;
} else if (actionInput instanceof VirtualActionInput) {
inputPath = RootedPath.toRootedPath(withinSandboxExecRoot, actionInput.getExecPath());
} else if (actionInput instanceof Artifact) {
Artifact inputArtifact = (Artifact) actionInput;
if (inputArtifact.isSourceArtifact() && sandboxSourceRoots != null) {
Root sourceRoot = inputArtifact.getRoot().getRoot();
if (!sourceRootToSandboxSourceRoot.containsKey(sourceRoot)) {
int next = sourceRootToSandboxSourceRoot.size();
sourceRootToSandboxSourceRoot.put(
sourceRoot,
Root.fromPath(sandboxSourceRoots.getRelative(Integer.toString(next))));
}

inputPath =
RootedPath.toRootedPath(
sourceRootToSandboxSourceRoot.get(sourceRoot),
inputArtifact.getRootRelativePath());
} else {
if (sandboxSourceRoots == null) {
inputPath = RootedPath.toRootedPath(withinSandboxExecRoot, inputArtifact.getExecPath());
} else {
if (inputArtifact.isSourceArtifact()) {
Root sourceRoot = inputArtifact.getRoot().getRoot();
inputPath =
RootedPath.toRootedPath(
sourceRootWithinSandbox.apply(sourceRoot),
inputArtifact.getRootRelativePath());
} else {
PathFragment materializationExecPath = null;
if (inputArtifact.isChildOfDeclaredDirectory()) {
FileArtifactValue parentMetadata = inputMetadataProvider.getInputMetadata(inputArtifact.getParent());
if (parentMetadata.getMaterializationExecPath().isPresent()) {
materializationExecPath =
parentMetadata.getMaterializationExecPath().get()
.getRelative(inputArtifact.getParentRelativePath());
}
} else {
FileArtifactValue metadata = inputMetadataProvider.getInputMetadata(actionInput);
if (metadata.getMaterializationExecPath().isPresent()) {
materializationExecPath = metadata.getMaterializationExecPath().get();
}
}

if (materializationExecPath != null) {
inputPath = processResolvedSymlink(
absoluteRoot,
materializationExecPath,
withinSandboxExecRoot,
execRootPath.asFragment(),
packageRoots,
sourceRootWithinSandbox);
} else {
inputPath = RootedPath.toRootedPath(withinSandboxExecRoot, inputArtifact.getExecPath());
}
}
}
} else {
PathFragment execPath = actionInput.getExecPath();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
SandboxInputs readablePaths =
helpers.processInputFiles(
context.getInputMapping(PathFragment.EMPTY_FRAGMENT, /* willAccessRepeatedly= */ true),
context.getInputMetadataProvider(),
execRoot,
execRoot,
packageRoots,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -332,12 +332,19 @@ private TreeArtifactValue constructTreeArtifactValueFromFilesystem(SpecialArtifa
}

// Same rationale as for constructFileArtifactValue.
if (anyRemote.get() && treeDir.isSymbolicLink() && stat instanceof FileStatusWithMetadata) {
FileArtifactValue metadata = ((FileStatusWithMetadata) stat).getMetadata();
tree.setMaterializationExecPath(
metadata
.getMaterializationExecPath()
.orElse(treeDir.resolveSymbolicLinks().asFragment().relativeTo(execRoot)));
if (treeDir.isSymbolicLink()) {
PathFragment materializationExecPath = null;
if (stat instanceof FileStatusWithMetadata) {
materializationExecPath = ((FileStatusWithMetadata) stat).getMetadata()
.getMaterializationExecPath()
.orElse(null);
}

if (materializationExecPath == null) {
materializationExecPath = treeDir.resolveSymbolicLinks().asFragment().relativeTo(execRoot);
}

tree.setMaterializationExecPath(materializationExecPath);
}

return tree.build();
Expand Down Expand Up @@ -536,6 +543,16 @@ private FileArtifactValue constructFileArtifactValue(
// possible to hit the digest cache - we probably already computed the digest for the
// target during previous action execution.
path = statAndValue.realPath();
injectedDigest = DigestUtils.manuallyComputeDigest(path, value.getSize());

PathFragment execRootRelativeRealPath;
if (path.asFragment().startsWith(execRoot)) {
execRootRelativeRealPath = path.asFragment().relativeTo(execRoot);
} else {
execRootRelativeRealPath = path.asFragment();
}

return FileArtifactValue.createForResolvedSymlink(execRootRelativeRealPath, value, injectedDigest);
}

injectedDigest = DigestUtils.manuallyComputeDigest(path, value.getSize());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ public SpawnResult exec(Spawn spawn, SpawnExecutionContext context)
helpers.processInputFiles(
context.getInputMapping(
PathFragment.EMPTY_FRAGMENT, /* willAccessRepeatedly= */ true),
context.getInputMetadataProvider(),
execRoot,
execRoot,
packageRoots,
Expand Down
2 changes: 2 additions & 0 deletions src/test/java/com/google/devtools/build/lib/sandbox/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,14 @@ java_test(
deps = [
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
"//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
"//src/main/java/com/google/devtools/build/lib/exec:bin_tools",
"//src/main/java/com/google/devtools/build/lib/exec:tree_deleter",
"//src/main/java/com/google/devtools/build/lib/sandbox:sandbox_helpers",
"//src/main/java/com/google/devtools/build/lib/sandbox:sandbox_options",
"//src/main/java/com/google/devtools/build/lib/sandbox:sandboxed_spawns",
"//src/main/java/com/google/devtools/build/lib/sandbox:tree_deleter",
"//src/main/java/com/google/devtools/build/lib/skyframe:tree_artifact_value",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
Expand Down
Loading
Loading