Skip to content

Commit

Permalink
Inject metadata when creating a filesystem symlink for a non-symlink …
Browse files Browse the repository at this point in the history 
…artifact.

A ctx.actions.symlink whose output is a declare_file or declare_directory
(as opposed to a declare_symlink) has "copy" semantics, i.e., the output
artifact is indistinguishable from the referent except for its name; the
symlink is just a filesystem-level optimization to avoid an actual copy,
and is transparently resolved when collecting the action output metadata.

When the symlink points to an artifact that was built remotely and without the
bytes, we currently must download it before executing the symlink action, so
that the output metadata can be constructed from the local filesystem. This
change short-circuits the filesystem traversal by injecting output metadata,
which is identical to the input plus a pointer to the original path. This is
used by the prefetcher to avoid downloading the same files multiple times when
they're symlinked more than once.

(An alternative would have been to teach all of the RemoteActionFileSystem
methods to resolve symlinks by patching together the local and remote metadata,
but that would have resulted in an awful lot of complexity.)

Fixes bazelbuild#15678.
  • Loading branch information
@tjgq
tjgq committed Oct 18, 2022
1 parent 8d76ef1 commit f09c1dd
Show file tree
Hide file tree
Showing 17 changed files with 1,235 additions and 143 deletions.
91 changes: 83 additions & 8 deletions src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.Objects;
import java.util.Optional;
import javax.annotation.Nullable;

/**
Expand Down Expand Up @@ -172,6 +173,17 @@ protected boolean couldBeModifiedByMetadata(FileArtifactValue lastKnown) {
return true;
}

/**
* Optional symlink target path.
*
* <p>If present, this artifact is a copy of another artifact. It is materialized in the local
* filesystem as a symlink to the original artifact, whose contents live at this location. This is
* used to implement zero-cost copies of remotely stored artifacts.
*/
public Optional<PathFragment> getSymlinkTargetExecPath() {
return Optional.empty();
}

/**
* Marker interface for singleton implementations of this class.
*
Expand Down Expand Up @@ -514,9 +526,7 @@ public long getModifiedTime() {
@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add(
"digest",
digest == null ? "(null)" : BaseEncoding.base16().lowerCase().encode(digest))
.add("digest", BaseEncoding.base16().lowerCase().encode(digest))
.add("size", size)
.add("proxy", proxy)
.toString();
Expand All @@ -535,10 +545,10 @@ protected boolean couldBeModifiedByMetadata(FileArtifactValue o) {

/** Metadata for remotely stored files. */
public static class RemoteFileArtifactValue extends FileArtifactValue {
private final byte[] digest;
private final long size;
private final int locationIndex;
private final String actionId;
protected final byte[] digest;
protected final long size;
protected final int locationIndex;
protected final String actionId;

private RemoteFileArtifactValue(byte[] digest, long size, int locationIndex, String actionId) {
this.digest = Preconditions.checkNotNull(digest, actionId);
Expand All @@ -556,6 +566,19 @@ public static RemoteFileArtifactValue create(byte[] digest, long size, int locat
return new RemoteFileArtifactValue(digest, size, locationIndex, /* actionId= */ "");
}

public static RemoteFileArtifactValue create(
byte[] digest,
long size,
int locationIndex,
String actionId,
@Nullable PathFragment symlinkTargetExecPath) {
if (symlinkTargetExecPath != null) {
return new SymlinkToRemoteFileArtifactValue(
digest, size, locationIndex, actionId, symlinkTargetExecPath);
}
return new RemoteFileArtifactValue(digest, size, locationIndex, actionId);
}

@Override
public boolean equals(Object o) {
if (!(o instanceof RemoteFileArtifactValue)) {
Expand Down Expand Up @@ -602,7 +625,7 @@ public String getActionId() {
@Override
public long getModifiedTime() {
throw new UnsupportedOperationException(
"RemoteFileArifactValue doesn't support getModifiedTime");
"RemoteFileArtifactValue doesn't support getModifiedTime");
}

@Override
Expand All @@ -626,6 +649,58 @@ public String toString() {
.add("digest", bytesToString(digest))
.add("size", size)
.add("locationIndex", locationIndex)
.add("actionId", actionId)
.toString();
}
}

/** An artifact that should be materialized as a symlink to another remote artifact. */
public static final class SymlinkToRemoteFileArtifactValue extends RemoteFileArtifactValue {
private PathFragment symlinkTargetExecPath;

private SymlinkToRemoteFileArtifactValue(
byte[] digest,
long size,
int locationIndex,
String actionId,
PathFragment symlinkTargetExecPath) {
super(digest, size, locationIndex, actionId);
this.symlinkTargetExecPath = Preconditions.checkNotNull(symlinkTargetExecPath);
}

@Override
public Optional<PathFragment> getSymlinkTargetExecPath() {
return Optional.ofNullable(symlinkTargetExecPath);
}

@Override
public boolean equals(Object o) {
if (!(o instanceof SymlinkToRemoteFileArtifactValue)) {
return false;
}

SymlinkToRemoteFileArtifactValue that = (SymlinkToRemoteFileArtifactValue) o;
return Arrays.equals(digest, that.digest)
&& size == that.size
&& locationIndex == that.locationIndex
&& Objects.equals(actionId, that.actionId)
&& Objects.equals(symlinkTargetExecPath, that.symlinkTargetExecPath);
}

@Override
public int hashCode() {
return Objects.hash(
Arrays.hashCode(digest), size, locationIndex, actionId, symlinkTargetExecPath);
}

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("digest", bytesToString(digest))
.add("size", size)
.add("locationIndex", locationIndex)
.add("actionId", actionId)
.add("symlinkTargetExecPath", symlinkTargetExecPath)
.toString();
}
}
Expand Down
17 changes: 13 additions & 4 deletions src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,10 @@ final class Entry {
public abstract static class SerializableTreeArtifactValue {
public static SerializableTreeArtifactValue create(
ImmutableMap<String, RemoteFileArtifactValue> childValues,
Optional<RemoteFileArtifactValue> archivedFileValue) {
Optional<RemoteFileArtifactValue> archivedFileValue,
Optional<PathFragment> symlinkTargetExecPath) {
return new AutoValue_ActionCache_Entry_SerializableTreeArtifactValue(
childValues, archivedFileValue);
childValues, archivedFileValue, symlinkTargetExecPath);
}

/**
Expand All @@ -138,17 +139,25 @@ public static Optional<SerializableTreeArtifactValue> createSerializable(
.filter(ar -> ar.archivedFileValue().isRemote())
.map(ar -> (RemoteFileArtifactValue) ar.archivedFileValue());

if (childValues.isEmpty() && archivedFileValue.isEmpty()) {
Optional<PathFragment> symlinkTargetExecPath = treeMetadata.getSymlinkTargetExecPath();

if (childValues.isEmpty()
&& archivedFileValue.isEmpty()
&& symlinkTargetExecPath.isEmpty()) {
return Optional.empty();
}

return Optional.of(SerializableTreeArtifactValue.create(childValues, archivedFileValue));
return Optional.of(
SerializableTreeArtifactValue.create(
childValues, archivedFileValue, symlinkTargetExecPath));
}

// A map from parentRelativePath to the file metadata
public abstract ImmutableMap<String, RemoteFileArtifactValue> childValues();

public abstract Optional<RemoteFileArtifactValue> archivedFileValue();

public abstract Optional<PathFragment> symlinkTargetExecPath();
}

public Entry(String key, Map<String, String> usedClientEnv, boolean discoversInputs) {
Expand Down
55 changes: 50 additions & 5 deletions src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import com.google.devtools.build.lib.util.VarInt;
import com.google.devtools.build.lib.vfs.DigestUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.lib.vfs.UnixGlob;
import java.io.ByteArrayOutputStream;
Expand Down Expand Up @@ -75,7 +76,7 @@ public class CompactPersistentActionCache implements ActionCache {

private static final int NO_INPUT_DISCOVERY_COUNT = -1;

private static final int VERSION = 13;
private static final int VERSION = 14;

private static final class ActionMap extends PersistentMap<Integer, byte[]> {
private final Clock clock;
Expand Down Expand Up @@ -466,6 +467,14 @@ private static void encodeRemoteMetadata(
VarInt.putVarInt(value.getLocationIndex(), sink);

VarInt.putVarInt(indexer.getOrCreateIndex(value.getActionId()), sink);

Optional<PathFragment> symlinkTargetExecPath = value.getSymlinkTargetExecPath();
if (symlinkTargetExecPath.isPresent()) {
VarInt.putVarInt(1, sink);
VarInt.putVarInt(indexer.getOrCreateIndex(symlinkTargetExecPath.get().toString()), sink);
} else {
VarInt.putVarInt(0, sink);
}
}

private static final int MAX_REMOTE_METADATA_SIZE =
Expand All @@ -484,7 +493,18 @@ private static RemoteFileArtifactValue decodeRemoteMetadata(

String actionId = getStringForIndex(indexer, VarInt.getVarInt(source));

return RemoteFileArtifactValue.create(digest, size, locationIndex, actionId);
PathFragment symlinkTargetExecPath = null;
int numSymlinkTargetExecPath = VarInt.getVarInt(source);
if (numSymlinkTargetExecPath > 0) {
if (numSymlinkTargetExecPath != 1) {
throw new IOException("Invalid number of symlink target paths");
}
symlinkTargetExecPath =
PathFragment.create(getStringForIndex(indexer, VarInt.getVarInt(source)));
}

return RemoteFileArtifactValue.create(
digest, size, locationIndex, actionId, symlinkTargetExecPath);
}

/** @return action data encoded as a byte[] array. */
Expand All @@ -497,6 +517,7 @@ private static byte[] encode(StringIndexer indexer, ActionCache.Entry entry) thr
int maxOutputFilesSize =
VarInt.MAX_VARINT_SIZE // entry.getOutputFiles().size()
+ (VarInt.MAX_VARINT_SIZE // execPath
+ (1 + VarInt.MAX_VARINT_SIZE) // symlinkTargetExecPath optional
+ MAX_REMOTE_METADATA_SIZE)
* entry.getOutputFiles().size();

Expand All @@ -513,9 +534,12 @@ private static byte[] encode(StringIndexer indexer, ActionCache.Entry entry) thr
+ MAX_REMOTE_METADATA_SIZE)
* value.childValues().size();

maxOutputTreesSize += VarInt.MAX_VARINT_SIZE; // value.archivedFileValue() optional
maxOutputTreesSize +=
value.archivedFileValue().map(ignored -> MAX_REMOTE_METADATA_SIZE).orElse(0);
VarInt.MAX_VARINT_SIZE // value.archivedFileValue() optional
+ value.archivedFileValue().map(ignored -> MAX_REMOTE_METADATA_SIZE).orElse(0);
maxOutputTreesSize +=
VarInt.MAX_VARINT_SIZE // value.symlinkTargetExecPath() optional
+ value.symlinkTargetExecPath().map(ignored -> VarInt.MAX_VARINT_SIZE).orElse(0);
}

// Estimate the size of the buffer:
Expand Down Expand Up @@ -578,6 +602,15 @@ private static byte[] encode(StringIndexer indexer, ActionCache.Entry entry) thr
} else {
VarInt.putVarInt(0, sink);
}

Optional<PathFragment> symlinkTargetExecPath =
serializableTreeArtifactValue.symlinkTargetExecPath();
if (symlinkTargetExecPath.isPresent()) {
VarInt.putVarInt(1, sink);
VarInt.putVarInt(indexer.getOrCreateIndex(symlinkTargetExecPath.get().toString()), sink);
} else {
VarInt.putVarInt(0, sink);
}
}

return sink.toByteArray();
Expand Down Expand Up @@ -654,8 +687,20 @@ private static ActionCache.Entry decode(StringIndexer indexer, byte[] data) thro
archivedFileValue = Optional.of(decodeRemoteMetadata(indexer, source));
}

Optional<PathFragment> symlinkTargetPath = Optional.empty();
int numSymlinkTargetExecPath = VarInt.getVarInt(source);
if (numSymlinkTargetExecPath > 0) {
if (numSymlinkTargetExecPath != 1) {
throw new IOException("Invalid number of symlink target paths");
}
symlinkTargetPath =
Optional.of(
PathFragment.create(getStringForIndex(indexer, VarInt.getVarInt(source))));
}

SerializableTreeArtifactValue value =
SerializableTreeArtifactValue.create(childValues.buildOrThrow(), archivedFileValue);
SerializableTreeArtifactValue.create(
childValues.buildOrThrow(), archivedFileValue, symlinkTargetPath);
outputTrees.put(treeKey, value);
}

Expand Down
Loading

0 comments on commit f09c1dd

Please sign in to comment.