diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD index a560f86d8e56b3..6cbc67e01e63e5 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD @@ -37,6 +37,7 @@ java_library( "RemoteOutputChecker.java", "AbstractActionInputPrefetcher.java", "LeaseService.java", + "Scrubber.java", ], ), exports = [ @@ -51,6 +52,7 @@ java_library( ":abstract_action_input_prefetcher", ":lease_service", ":remote_output_checker", + ":scrubber", "//src/main/java/com/google/devtools/build/lib:build-request-options", "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib:runtime/command_line_path_factory", @@ -249,3 +251,15 @@ java_library( "//third_party:jsr305", ], ) + +java_library( + name = "scrubber", + srcs = ["Scrubber.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/actions", + "//src/main/java/com/google/devtools/build/lib/actions:artifacts", + "//src/main/java/com/google/devtools/build/lib/remote/options", + "//src/main/java/com/google/devtools/common/options:options_internal", + "//third_party:guava", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index 2f1d38faadf0e5..e743918f7b2eb7 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -87,6 +87,7 @@ import com.google.devtools.build.lib.remote.RemoteExecutionService.ActionResultMetadata.DirectoryMetadata; import com.google.devtools.build.lib.remote.RemoteExecutionService.ActionResultMetadata.FileMetadata; import com.google.devtools.build.lib.remote.RemoteExecutionService.ActionResultMetadata.SymlinkMetadata; +import com.google.devtools.build.lib.remote.Scrubber.SpawnScrubber; import com.google.devtools.build.lib.remote.common.BulkTransferException; import com.google.devtools.build.lib.remote.common.OperationObserver; import com.google.devtools.build.lib.remote.common.OutputDigestMismatchException; @@ -179,6 +180,8 @@ public class RemoteExecutionService { @Nullable private final RemoteOutputChecker remoteOutputChecker; + private final Scrubber scrubber; + public RemoteExecutionService( Executor executor, Reporter reporter, @@ -210,6 +213,7 @@ public RemoteExecutionService( if (remoteOptions.remoteMerkleTreeCacheSize != 0) { merkleTreeCacheBuilder.maximumSize(remoteOptions.remoteMerkleTreeCacheSize); } + this.scrubber = Scrubber.forOptions(remoteOptions); this.merkleTreeCache = merkleTreeCacheBuilder.build(); this.tempPathGenerator = tempPathGenerator; @@ -219,12 +223,13 @@ public RemoteExecutionService( this.remoteOutputChecker = remoteOutputChecker; } - static Command buildCommand( + private Command buildCommand( Collection outputs, List arguments, ImmutableMap env, @Nullable Platform platform, - RemotePathResolver remotePathResolver) { + RemotePathResolver remotePathResolver, + @Nullable SpawnScrubber spawnScrubber) { Command.Builder command = Command.newBuilder(); ArrayList outputFiles = new ArrayList<>(); ArrayList outputDirectories = new ArrayList<>(); @@ -249,6 +254,9 @@ static Command buildCommand( command.setPlatform(platform); } for (String arg : arguments) { + if (spawnScrubber != null) { + arg = spawnScrubber.transformArgument(arg); + } command.addArguments(decodeBytestringUtf8(arg)); } // Sorting the environment pairs by variable name. @@ -349,15 +357,16 @@ private SortedMap buildOutputDirMap(Spawn spawn) { } private MerkleTree buildInputMerkleTree( - Spawn spawn, SpawnExecutionContext context, ToolSignature toolSignature) + Spawn spawn, SpawnExecutionContext context, ToolSignature toolSignature, + @Nullable SpawnScrubber spawnScrubber) throws IOException, ForbiddenActionInputException { // Add output directories to inputs so that they are created as empty directories by the // executor. The spec only requires the executor to create the parent directory of an output // directory, which differs from the behavior of both local and sandboxed execution. SortedMap outputDirMap = buildOutputDirMap(spawn); boolean useMerkleTreeCache = remoteOptions.remoteMerkleTreeCache; - if (toolSignature != null) { - // Marking tool files is not yet supported in conjunction with the merkle tree cache. + if (toolSignature != null || spawnScrubber != null) { + // The Merkle tree cache is not yet compatible with scrubbing or marking tool files. useMerkleTreeCache = false; } if (useMerkleTreeCache) { @@ -369,7 +378,8 @@ private MerkleTree buildInputMerkleTree( (Object nodeKey, InputWalker walker) -> { subMerkleTrees.add( buildMerkleTreeVisitor( - nodeKey, walker, inputMetadataProvider, context.getPathResolver())); + nodeKey, walker, inputMetadataProvider, context.getPathResolver(), + spawnScrubber)); }); if (!outputDirMap.isEmpty()) { subMerkleTrees.add( @@ -378,6 +388,7 @@ private MerkleTree buildInputMerkleTree( inputMetadataProvider, execRoot, context.getPathResolver(), + /* scrubber= */ null, digestUtil)); } return MerkleTree.merge(subMerkleTrees, digestUtil); @@ -399,6 +410,7 @@ private MerkleTree buildInputMerkleTree( context.getInputMetadataProvider(), execRoot, context.getPathResolver(), + spawnScrubber, digestUtil); } } @@ -407,7 +419,8 @@ private MerkleTree buildMerkleTreeVisitor( Object nodeKey, InputWalker walker, InputMetadataProvider inputMetadataProvider, - ArtifactPathResolver artifactPathResolver) + ArtifactPathResolver artifactPathResolver, + @Nullable SpawnScrubber spawnScrubber) throws IOException, ForbiddenActionInputException { // Deduplicate concurrent computations for the same node. It's not possible to use // MerkleTreeCache#get(key, loader) because the loading computation may cause other nodes to be @@ -419,7 +432,8 @@ private MerkleTree buildMerkleTreeVisitor( // No preexisting cache entry, so we must do the computation ourselves. try { freshFuture.complete( - uncachedBuildMerkleTreeVisitor(walker, inputMetadataProvider, artifactPathResolver)); + uncachedBuildMerkleTreeVisitor(walker, inputMetadataProvider, artifactPathResolver, + spawnScrubber)); } catch (Exception e) { freshFuture.completeExceptionally(e); } @@ -443,7 +457,8 @@ private MerkleTree buildMerkleTreeVisitor( public MerkleTree uncachedBuildMerkleTreeVisitor( InputWalker walker, InputMetadataProvider inputMetadataProvider, - ArtifactPathResolver artifactPathResolver) + ArtifactPathResolver artifactPathResolver, + @Nullable SpawnScrubber scrubber) throws IOException, ForbiddenActionInputException { ConcurrentLinkedQueue subMerkleTrees = new ConcurrentLinkedQueue<>(); subMerkleTrees.add( @@ -452,18 +467,19 @@ public MerkleTree uncachedBuildMerkleTreeVisitor( inputMetadataProvider, execRoot, artifactPathResolver, + scrubber, digestUtil)); walker.visitNonLeaves( (Object subNodeKey, InputWalker subWalker) -> { subMerkleTrees.add( buildMerkleTreeVisitor( - subNodeKey, subWalker, inputMetadataProvider, artifactPathResolver)); + subNodeKey, subWalker, inputMetadataProvider, artifactPathResolver, scrubber)); }); return MerkleTree.merge(subMerkleTrees, digestUtil); } @Nullable - private static ByteString buildSalt(Spawn spawn) { + private static ByteString buildSalt(Spawn spawn, @Nullable SpawnScrubber spawnScrubber) { CacheSalt.Builder saltBuilder = CacheSalt.newBuilder().setMayBeExecutedRemotely(Spawns.mayBeExecutedRemotely(spawn)); @@ -473,6 +489,12 @@ private static ByteString buildSalt(Spawn spawn) { saltBuilder.setWorkspace(workspace); } + if (spawnScrubber != null) { + saltBuilder.setScrubSalt( + CacheSalt.ScrubSalt.newBuilder().setIsScrubbed(true).setSalt(spawnScrubber.getSalt()) + .build()); + } + return saltBuilder.build().toByteString(); } @@ -508,7 +530,9 @@ public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context remoteActionBuildingSemaphore.acquire(); try { ToolSignature toolSignature = getToolSignature(spawn, context); - final MerkleTree merkleTree = buildInputMerkleTree(spawn, context, toolSignature); + SpawnScrubber spawnScrubber = scrubber.forSpawn(spawn); + final MerkleTree merkleTree = buildInputMerkleTree(spawn, context, toolSignature, + spawnScrubber); // Get the remote platform properties. Platform platform = PlatformUtils.getPlatformProto(spawn, remoteOptions); @@ -526,7 +550,8 @@ public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context spawn.getArguments(), spawn.getEnvironment(), platform, - remotePathResolver); + remotePathResolver, + spawnScrubber); Digest commandHash = digestUtil.compute(command); Action action = Utils.buildAction( @@ -535,7 +560,7 @@ public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context platform, context.getTimeout(), Spawns.mayBeCachedRemotely(spawn), - buildSalt(spawn)); + buildSalt(spawn, spawnScrubber)); ActionKey actionKey = digestUtil.computeActionKey(action); @@ -1414,7 +1439,8 @@ public void uploadInputsIfNotPresent(RemoteAction action, boolean force) Spawn spawn = action.getSpawn(); SpawnExecutionContext context = action.getSpawnExecutionContext(); ToolSignature toolSignature = getToolSignature(spawn, context); - merkleTree = buildInputMerkleTree(spawn, context, toolSignature); + SpawnScrubber spawnScrubber = scrubber.forSpawn(spawn); + merkleTree = buildInputMerkleTree(spawn, context, toolSignature, spawnScrubber); } remoteExecutionCache.ensureInputsPresent( diff --git a/src/main/java/com/google/devtools/build/lib/remote/Scrubber.java b/src/main/java/com/google/devtools/build/lib/remote/Scrubber.java new file mode 100644 index 00000000000000..286c19c32b7a05 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/remote/Scrubber.java @@ -0,0 +1,156 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.remote; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.cache.VirtualActionInput; +import com.google.devtools.build.lib.remote.options.RemoteOptions; +import com.google.devtools.common.options.RegexPatternOption; +import java.util.Collection; +import java.util.Map; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; + +/** + * The {@link Scrubber} implements scrubbing of remote cache keys. + * + *

See the documentation for the {@code --experimental_remote_scrub_*} flags for more + * information. + */ +public class Scrubber { + + /** + * A {@link SpawnScrubber} determines how to scrub the cache key for a {@link Spawn}. + */ + public static class SpawnScrubber { + + private final Predicate inputMatcher; + private final ImmutableMap argReplacements; + private final String salt; + + private SpawnScrubber(RemoteOptions options) { + this.inputMatcher = buildInputMatcher(options); + this.argReplacements = ImmutableMap.copyOf(options.scrubArgReplacements); + this.salt = options.scrubSalt; + } + + private static Predicate buildInputMatcher(RemoteOptions options) { + Predicate execPathMatcher = buildStringMatcher(options.scrubInput); + return (input) -> !input.equals(VirtualActionInput.EMPTY_MARKER) && execPathMatcher.test(input.getExecPathString()); + } + + /** + * Whether the given input should be omitted from the cache key. + */ + public boolean shouldOmitInput(ActionInput input) { + return inputMatcher.test(input); + } + + /** + * Transforms an action command line argument. + */ + public String transformArgument(String arg) { + for (Map.Entry entry : argReplacements.entrySet()) { + Pattern pattern = entry.getKey().regexPattern(); + String replacement = entry.getValue(); + // Don't use Pattern#replaceFirst because it allows references to capture groups. + Matcher m = pattern.matcher(arg); + if (m.find()) { + arg = arg.substring(0, m.start()) + replacement + arg.substring(m.end()); + } + } + return arg; + } + + /** + * Returns the scrubbing salt. + */ + public String getSalt() { + return salt; + } + } + + private final Predicate spawnMatcher; + + private final SpawnScrubber spawnScrubber; + + private Scrubber(RemoteOptions options) { + this.spawnMatcher = buildSpawnMatcher(options); + this.spawnScrubber = new SpawnScrubber(options); + } + + /** + * Returns a {@link Scrubber} that performs scrubbing according to the {@link RemoteOptions}. + */ + @Nullable + public static Scrubber forOptions(RemoteOptions options) { + return new Scrubber(options); + } + + private static Predicate buildSpawnMatcher(RemoteOptions options) { + if (!options.scrubEnabled) { + return Predicates.alwaysFalse(); + } + + Predicate mnemonicMatcher = buildStringMatcher(options.scrubMnemonic); + Predicate repoMatcher = buildStringMatcher(options.scrubRepo); + boolean scrubExec = options.scrubExec; + + return (spawn) -> { + String mnemonic = spawn.getMnemonic(); + String repo = spawn.getResourceOwner().getOwner().getLabel().getRepository().getName(); + boolean isForTool = spawn.getResourceOwner().getOwner().isBuildConfigurationForTool(); + + return (!isForTool || scrubExec) && mnemonicMatcher.test(mnemonic) && repoMatcher.test(repo); + }; + } + + private static Predicate buildStringMatcher(Collection options) { + if (options.isEmpty()) { + // If no patterns are specified, match nothing. + return Predicates.alwaysFalse(); + } + // Combine multiple patterns into a single one for efficiency. + StringBuilder sb = new StringBuilder(); + for (RegexPatternOption opt : options) { + if (sb.length() > 0) { + sb.append("|"); + } + sb.append("(?:"); + sb.append(opt.regexPattern().pattern()); + sb.append(")"); + } + Pattern pattern = Pattern.compile(sb.toString()); + return (str) -> pattern.matcher(str).find(); + } + + /** + * Returns a {@link SpawnScrubber} suitable for a {@link Spawn}, or {@code null} if the spawn does + * not need to be scrubbed. + */ + @Nullable + public SpawnScrubber forSpawn(Spawn spawn) { + if (spawnMatcher.test(spawn)) { + return spawnScrubber; + } + return null; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/remote/merkletree/BUILD b/src/main/java/com/google/devtools/build/lib/remote/merkletree/BUILD index f19db0ddf06637..47f7e9e7325eb2 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/merkletree/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/merkletree/BUILD @@ -20,6 +20,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/actions:artifacts", "//src/main/java/com/google/devtools/build/lib/actions:file_metadata", "//src/main/java/com/google/devtools/build/lib/profiler", + "//src/main/java/com/google/devtools/build/lib/remote:scrubber", "//src/main/java/com/google/devtools/build/lib/remote/util", "//src/main/java/com/google/devtools/build/lib/util:string", "//src/main/java/com/google/devtools/build/lib/vfs", diff --git a/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java b/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java index 268bde9d2b33c9..390151e6162afd 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java @@ -24,6 +24,7 @@ import com.google.devtools.build.lib.actions.FileArtifactValue; import com.google.devtools.build.lib.actions.InputMetadataProvider; import com.google.devtools.build.lib.actions.cache.VirtualActionInput; +import com.google.devtools.build.lib.remote.Scrubber.SpawnScrubber; import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.DirectoryNode; import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.FileNode; import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.SymlinkNode; @@ -39,6 +40,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; +import javax.annotation.Nullable; /** Builder for directory trees. */ class DirectoryTreeBuilder { @@ -63,6 +65,7 @@ static DirectoryTree fromActionInputs( InputMetadataProvider inputMetadataProvider, Path execRoot, ArtifactPathResolver artifactPathResolver, + @Nullable SpawnScrubber spawnScrubber, DigestUtil digestUtil) throws IOException { return fromActionInputs( @@ -71,6 +74,7 @@ static DirectoryTree fromActionInputs( inputMetadataProvider, execRoot, artifactPathResolver, + spawnScrubber, digestUtil); } @@ -80,6 +84,7 @@ static DirectoryTree fromActionInputs( InputMetadataProvider inputMetadataProvider, Path execRoot, ArtifactPathResolver artifactPathResolver, + @Nullable SpawnScrubber spawnScrubber, DigestUtil digestUtil) throws IOException { Map tree = new HashMap<>(); @@ -90,6 +95,7 @@ static DirectoryTree fromActionInputs( inputMetadataProvider, execRoot, artifactPathResolver, + spawnScrubber, digestUtil, tree); return new DirectoryTree(tree, numFiles); @@ -114,6 +120,9 @@ static DirectoryTree fromPaths(SortedMap inputFiles, DigestU /** * Adds the files in {@code inputs} as nodes to {@code tree}. * + *

Prefer {@link #buildFromActionInputs} if this Merkle tree is for an action spawn (as + * opposed to repository fetching).

+ * *

This method mutates {@code tree}. * * @param inputs map of paths to files. The key determines the path at which the file should be @@ -128,6 +137,7 @@ private static int buildFromPaths( return build( inputs, tree, + /* spawnScrubber= */ null, (input, path, currDir) -> { if (!input.isFile(Symlinks.NOFOLLOW)) { throw new IOException(String.format("Input '%s' is not a file.", input)); @@ -152,12 +162,14 @@ private static int buildFromActionInputs( InputMetadataProvider inputMetadataProvider, Path execRoot, ArtifactPathResolver artifactPathResolver, + @Nullable SpawnScrubber spawnScrubber, DigestUtil digestUtil, Map tree) throws IOException { return build( inputs, tree, + spawnScrubber, (input, path, currDir) -> { if (input instanceof VirtualActionInput) { VirtualActionInput virtualActionInput = (VirtualActionInput) input; @@ -198,6 +210,7 @@ private static int buildFromActionInputs( inputMetadataProvider, execRoot, artifactPathResolver, + spawnScrubber, digestUtil, tree); @@ -234,6 +247,7 @@ private static int buildFromActionInputs( private static int build( SortedMap inputs, Map tree, + @Nullable SpawnScrubber scrubber, FileNodeVisitor fileNodeVisitor) throws IOException { if (inputs.isEmpty()) { @@ -248,6 +262,11 @@ private static int build( PathFragment path = e.getKey(); T input = e.getValue(); + if (scrubber != null && input instanceof ActionInput && scrubber.shouldOmitInput( + (ActionInput) input)) { + continue; + } + if (input instanceof DerivedArtifact && ((DerivedArtifact) input).isTreeArtifact()) { // SpawnInputExpander has already expanded non-empty tree artifacts into a collection of // TreeFileArtifacts. Thus, at this point, tree artifacts represent empty directories, which diff --git a/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java b/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java index 9089434bcf78f8..cb7c54966c55eb 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java +++ b/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java @@ -33,6 +33,7 @@ import com.google.devtools.build.lib.actions.InputMetadataProvider; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.SilentCloseable; +import com.google.devtools.build.lib.remote.Scrubber.SpawnScrubber; import com.google.devtools.build.lib.remote.util.DigestUtil; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; @@ -223,12 +224,14 @@ public static MerkleTree build( InputMetadataProvider inputMetadataProvider, Path execRoot, ArtifactPathResolver artifactPathResolver, + @Nullable SpawnScrubber spawnScrubber, DigestUtil digestUtil) throws IOException { try (SilentCloseable c = Profiler.instance().profile("MerkleTree.build(ActionInput)")) { DirectoryTree tree = DirectoryTreeBuilder.fromActionInputs( - inputs, inputMetadataProvider, execRoot, artifactPathResolver, digestUtil); + inputs, inputMetadataProvider, execRoot, artifactPathResolver, spawnScrubber, + digestUtil); return build(tree, digestUtil); } } @@ -250,6 +253,7 @@ public static MerkleTree build( InputMetadataProvider inputMetadataProvider, Path execRoot, ArtifactPathResolver artifactPathResolver, + @Nullable SpawnScrubber spawnScrubber, DigestUtil digestUtil) throws IOException { try (SilentCloseable c = Profiler.instance().profile("MerkleTree.build(ActionInput)")) { @@ -260,6 +264,7 @@ public static MerkleTree build( inputMetadataProvider, execRoot, artifactPathResolver, + spawnScrubber, digestUtil); return build(tree, digestUtil); } diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java index b74563d3c08c80..1d5d4358444b42 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java +++ b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java @@ -18,19 +18,24 @@ import build.bazel.remote.execution.v2.Platform.Property; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSortedMap; +import com.google.common.collect.Maps; import com.google.devtools.build.lib.actions.UserExecException; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.server.FailureDetails.RemoteExecution; import com.google.devtools.build.lib.server.FailureDetails.RemoteExecution.Code; import com.google.devtools.build.lib.util.OptionsUtils; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Converter; import com.google.devtools.common.options.Converters; import com.google.devtools.common.options.Converters.AssignmentConverter; +import com.google.devtools.common.options.Converters.RegexPatternConverter; import com.google.devtools.common.options.EnumConverter; import com.google.devtools.common.options.Option; import com.google.devtools.common.options.OptionDocumentationCategory; import com.google.devtools.common.options.OptionEffectTag; import com.google.devtools.common.options.OptionMetadataTag; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.RegexPatternOption; import com.google.protobuf.TextFormat; import com.google.protobuf.TextFormat.ParseException; import java.time.Duration; @@ -38,6 +43,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.SortedMap; +import java.util.regex.Pattern; /** Options for remote execution and distributed caching for Bazel only. */ public final class RemoteOptions extends CommonRemoteOptions { @@ -709,6 +715,128 @@ public RemoteOutputsStrategyConverter() { + " frequency is based on the value of `--experimental_remote_cache_ttl`.") public boolean remoteCacheLeaseExtension; + @Option( + name = "experimental_remote_scrub_enabled", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "Whether remote cache key scrubbing is enabled.\n\n" + + "This feature is intended to facilitate sharing a remote cache between actions" + + " executing on different platforms but targeting the same platform. It should be" + + " used with extreme care, as improper settings may cause accidental sharing of" + + " cache entries and result in incorrect builds.\n\n" + + "The --experimental_remote_scrub_mnemonic, --experimental_remote_scrub_repo and" + + " --experimental_remote_scrub_exec options determine the scope for scrubbing;" + + " an action must obey all three criteria to be eligible.\n\n" + + "The --experimental_remote_scrub_input, --experimental_remote_arg_replace and" + + " --experimental_remote_scrub_salt options determine how the scrubbing is performed" + + " for an eligible action.\n\n" + + "These options do not affect how an action is executed, only how its remote cache" + + " key is computed for the purpose of retrieving or storing an action result." + + " They cannot be successfully used in conjunction with remote execution. Modifying" + + " the options does not invalidate outputs present in the local filesystem or" + + " internal caches; a clean build is required to reexecute affected actions.\n\n" + + "In order to successfully use this feature, you likely want to set a custom" + + " --host_platform together with --experimental_platform_in_output_dir (to normalize" + + " output prefixes) and --incompatible_strict_action_env (to normalize environment" + + " variables)." + ) + public boolean scrubEnabled; + + @Option( + name = "experimental_remote_scrub_mnemonic", + converter = Converters.RegexPatternConverter.class, + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "Matches mnemonics for which scrubbing is enabled. May be specified multiple times. An" + + " action is eligible for scrubbing if its mnemonic partially matches one of the" + + " patterns.", + allowMultiple = true) + public List scrubMnemonic; + + @Option( + name = "experimental_remote_scrub_repo", + converter = Converters.RegexPatternConverter.class, + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "Matches repositories for which scrubbing is enabled. May be specified multiple times. An" + + " action is eligible for scrubbing if the repository name of its owner partially" + + " matches one of the patterns.", + allowMultiple = true) + public List scrubRepo; + + @Option( + name = "experimental_remote_scrub_exec", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Whether actions targeting the execution platform should be scrubbed.") + public boolean scrubExec; + + @Option( + name = "experimental_remote_scrub_input", + converter = Converters.RegexPatternConverter.class, + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "Matches input paths that should be omitted from the remote cache key when scrubbing is" + + " enabled. May be specified multiple times. An input is omitted if its path" + + " relative to the execution root partially matches one of the patterns.", + allowMultiple = true) + public List scrubInput; + + @Option( + name = "experimental_remote_scrub_arg_replacement", + converter = ScrubArgReplacementConverter.class, + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "A regex and replacement string pair, separated by `=`, describing a transformation to be" + + " made to each action command line argument appearing in the remote cache key when" + + " scrubbing is enabled. If a partial match for the regex is found, the first" + + " matching region is replaced with the replacement string; otherwise, no" + + " replacement is made. May be specified multiple times. Each transformation is" + + " successively applied to the command line argument, operating on the result of the" + + " previous transformation.", + allowMultiple = true) + public List> scrubArgReplacements; + + @Option( + name = "experimental_remote_scrub_salt", + defaultValue = "", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "A unique token to be included in the remote cache key when scrubbing is enabled." + + " This may be used to bust the cache after external factors have changed.") + public String scrubSalt; + + private static class ScrubArgReplacementConverter + extends Converter.Contextless> { + + private static final AssignmentConverter assignmentConverter = new AssignmentConverter(); + private static final RegexPatternConverter patternConverter = new RegexPatternConverter(); + + @Override + public Map.Entry convert(String input) + throws OptionsParsingException, NumberFormatException { + Map.Entry assignmentEntry = assignmentConverter.convert(input); + RegexPatternOption pattern = patternConverter.convert(assignmentEntry.getKey()); + return Maps.immutableEntry(pattern, assignmentEntry.getValue()); + } + + @Override + public String getTypeDescription() { + return "a named float, 'name=value'"; + } + } + // The below options are not configurable by users, only tests. // This is part of the effort to reduce the overall number of flags. diff --git a/src/main/java/com/google/devtools/common/options/RegexPatternOption.java b/src/main/java/com/google/devtools/common/options/RegexPatternOption.java index bdf6313a0fea01..cdc2bd4c3a6689 100644 --- a/src/main/java/com/google/devtools/common/options/RegexPatternOption.java +++ b/src/main/java/com/google/devtools/common/options/RegexPatternOption.java @@ -28,7 +28,7 @@ */ @AutoValue public abstract class RegexPatternOption { - static RegexPatternOption create(Pattern regexPattern) { + public static RegexPatternOption create(Pattern regexPattern) { return new AutoValue_RegexPatternOption(Preconditions.checkNotNull(regexPattern)); } diff --git a/src/main/protobuf/spawn.proto b/src/main/protobuf/spawn.proto index 5863f973424431..d2dd839c09b07a 100644 --- a/src/main/protobuf/spawn.proto +++ b/src/main/protobuf/spawn.proto @@ -202,4 +202,16 @@ message CacheSalt { // Requires the execution service do NOT share caches across different // workspace. string workspace = 2; + + message ScrubSalt { + // Whether the cache key was scrubbed. + // Ensures that a scrubbed action can never collide with a non-scrubbed one. + bool is_scrubbed = 1; + + // A unique value used to bust the cache for scrubbed actions. + // See --experimental_remote_scrub_salt. + string salt = 2; + } + + ScrubSalt scrub_salt = 3; } diff --git a/src/test/java/com/google/devtools/build/lib/remote/BUILD b/src/test/java/com/google/devtools/build/lib/remote/BUILD index e49e1669d5efe7..8f0c789f5f1baf 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/test/java/com/google/devtools/build/lib/remote/BUILD @@ -76,6 +76,7 @@ java_test( "//src/main/java/com/google/devtools/build/lib/remote", "//src/main/java/com/google/devtools/build/lib/remote:abstract_action_input_prefetcher", "//src/main/java/com/google/devtools/build/lib/remote:remote_output_checker", + "//src/main/java/com/google/devtools/build/lib/remote:scrubber", "//src/main/java/com/google/devtools/build/lib/remote/circuitbreaker", "//src/main/java/com/google/devtools/build/lib/remote/common", "//src/main/java/com/google/devtools/build/lib/remote/common:bulk_transfer_exception", diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java index d0377e21ad3584..c70bce83e4f377 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java @@ -109,6 +109,7 @@ public void testVirtualActionInputSupport() throws Exception { fakeFileCache, execRoot, ArtifactPathResolver.forExecRoot(execRoot), + /* spawnScrubber= */ null, DIGEST_UTIL); Digest digest = DIGEST_UTIL.compute(virtualActionInput.getBytes().toByteArray()); diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java index 173033e241e744..6424ecb811badc 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java @@ -72,6 +72,7 @@ import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnResult; import com.google.devtools.build.lib.actions.SpawnResult.Status; +import com.google.devtools.build.lib.actions.Spawns; import com.google.devtools.build.lib.actions.StaticInputMetadataProvider; import com.google.devtools.build.lib.actions.util.ActionsTestUtil; import com.google.devtools.build.lib.clock.JavaClock; @@ -93,6 +94,7 @@ import com.google.devtools.build.lib.remote.common.RemotePathResolver; import com.google.devtools.build.lib.remote.common.RemotePathResolver.DefaultRemotePathResolver; import com.google.devtools.build.lib.remote.common.RemotePathResolver.SiblingRepositoryLayoutResolver; +import com.google.devtools.build.lib.remote.merkletree.MerkleTree; import com.google.devtools.build.lib.remote.options.RemoteOptions; import com.google.devtools.build.lib.remote.util.DigestUtil; import com.google.devtools.build.lib.remote.util.FakeSpawnExecutionContext; @@ -111,14 +113,17 @@ import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; +import com.google.devtools.common.options.RegexPatternOption; import com.google.protobuf.ByteString; import java.io.IOException; +import java.util.AbstractMap.SimpleEntry; import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -132,6 +137,8 @@ /** Tests for {@link RemoteExecutionService}. */ @RunWith(JUnit4.class) public class RemoteExecutionServiceTest { + private static final Pattern DOT_STAR = Pattern.compile(".*"); + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); @Rule public final RxNoGlobalErrorsRule rxNoGlobalErrorsRule = new RxNoGlobalErrorsRule(); @@ -2077,7 +2084,7 @@ public void buildMerkleTree_withMemoization_works() throws Exception { service.buildRemoteAction(spawn1, context1); // assert first time - verify(service, times(6)).uncachedBuildMerkleTreeVisitor(any(), any(), any()); + verify(service, times(6)).uncachedBuildMerkleTreeVisitor(any(), any(), any(), any()); assertThat(service.getMerkleTreeCache().asMap().keySet()) .containsExactly( ImmutableList.of(ImmutableMap.of(), PathFragment.EMPTY_FRAGMENT), // fileset mapping @@ -2091,7 +2098,7 @@ public void buildMerkleTree_withMemoization_works() throws Exception { service.buildRemoteAction(spawn2, context2); // assert second time - verify(service, times(6 + 2)).uncachedBuildMerkleTreeVisitor(any(), any(), any()); + verify(service, times(6 + 2)).uncachedBuildMerkleTreeVisitor(any(), any(), any(), any()); assertThat(service.getMerkleTreeCache().asMap().keySet()) .containsExactly( ImmutableList.of(ImmutableMap.of(), PathFragment.EMPTY_FRAGMENT), // fileset mapping @@ -2181,6 +2188,47 @@ public void buildRemoteActionForRemotePersistentWorkers() throws Exception { .build()); } + @Test + public void buildRemoteActionWithScrubbing() throws Exception { + var keptInput = ActionsTestUtil.createArtifact(artifactRoot, "kept_input"); + fakeFileCache.createScratchInput(keptInput, "kept"); + var scrubbedInput = ActionsTestUtil.createArtifact(artifactRoot, "scrubbed_input"); + fakeFileCache.createScratchInput(scrubbedInput, "scrubbed"); + + Spawn spawn = + new SpawnBuilder("some/path/cmd") + .withInputs(keptInput, scrubbedInput) + .withExecutionInfo(ExecutionRequirements.NO_REMOTE_EXEC, "") + .build(); + + FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn); + remoteOptions.scrubEnabled = true; + remoteOptions.scrubMnemonic = ImmutableList.of(RegexPatternOption.create(DOT_STAR)); + remoteOptions.scrubRepo = ImmutableList.of(RegexPatternOption.create(DOT_STAR)); + remoteOptions.scrubInput = ImmutableList.of( + RegexPatternOption.create(Pattern.compile(".*scrubbed.*"))); + remoteOptions.scrubArgReplacements = ImmutableList.of( + new SimpleEntry<>(RegexPatternOption.create(Pattern.compile("some/path")), "another/dir")); + remoteOptions.scrubSalt = "NaCl"; + RemoteExecutionService service = newRemoteExecutionService(remoteOptions); + + RemoteAction remoteAction = service.buildRemoteAction(spawn, context); + + MerkleTree merkleTree = remoteAction.getMerkleTree(); + Directory rootDir = merkleTree.getDirectoryByDigest( + merkleTree.getRootProto().getDirectories(0).getDigest()); + assertThat(rootDir).isEqualTo(Directory.newBuilder() + .addFiles(FileNode.newBuilder().setName("kept_input").setDigest(Digest.newBuilder() + .setHash("79f076abdd19a752db7267bfff2f9022161d120dea919fdaca2ffdfc24ca8c96") + .setSizeBytes(4).build()).setIsExecutable(true).build()).build()); + + assertThat(remoteAction.getCommand().getArgumentsList()).containsExactly("another/dir/cmd"); + + assertThat(remoteAction.getAction().getSalt()).isEqualTo(CacheSalt.newBuilder() + .setScrubSalt(CacheSalt.ScrubSalt.newBuilder().setIsScrubbed(true).setSalt("NaCl")).build() + .toByteString()); + } + private Spawn newSpawnFromResult(RemoteActionResult result) { return newSpawnFromResult(ImmutableMap.of(), result); } diff --git a/src/test/java/com/google/devtools/build/lib/remote/ScrubberTest.java b/src/test/java/com/google/devtools/build/lib/remote/ScrubberTest.java new file mode 100644 index 00000000000000..22b2a3ba97361a --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/remote/ScrubberTest.java @@ -0,0 +1,414 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.remote; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionInputHelper; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.cache.VirtualActionInput; +import com.google.devtools.build.lib.exec.util.SpawnBuilder; +import com.google.devtools.build.lib.remote.Scrubber.SpawnScrubber; +import com.google.devtools.build.lib.remote.options.RemoteOptions; +import com.google.devtools.common.options.Options; +import com.google.devtools.common.options.RegexPatternOption; +import java.util.AbstractMap.SimpleEntry; +import java.util.Map; +import java.util.regex.Pattern; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link Scrubber}. + */ +@RunWith(JUnit4.class) +public class ScrubberTest { + + @Test + public void noScrubbing() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + assertThat(Scrubber.forOptions(options).forSpawn(createSpawn())).isNull(); + } + + @Test + public void noRepoPattern() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList("Foo"); + assertThat(Scrubber.forOptions(options).forSpawn(createSpawn("Foo"))).isNull(); + } + + @Test + public void noMnemonicPattern() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubRepo = patternList("spam"); + assertThat(Scrubber.forOptions(options).forSpawn(createSpawn("spam", "Foo"))).isNull(); + } + + @Test + public void simpleMnemonicPattern() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList("Foo"); + options.scrubRepo = patternList(".*"); + + Scrubber scrubber = Scrubber.forOptions(options); + + assertThat(scrubber.forSpawn(createSpawn("Foo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("Foobar"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("Bar"))).isNull(); + } + + @Test + public void anchoredMnemonicPattern() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList("^Foo$"); + options.scrubRepo = patternList(".*"); + + Scrubber scrubber = Scrubber.forOptions(options); + + assertThat(scrubber.forSpawn(createSpawn("Foo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("Foobar"))).isNull(); + assertThat(scrubber.forSpawn(createSpawn("Bar"))).isNull(); + } + + @Test + public void wildcardMnemonicPattern() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList("Foo[12]"); + options.scrubRepo = patternList(".*"); + + Scrubber scrubber = Scrubber.forOptions(options); + + assertThat(scrubber.forSpawn(createSpawn("Foo1"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("Foo2"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("Foo3"))).isNull(); + } + + @Test + public void multipleMnemonicPatterns() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList("Foo", "Bar"); + options.scrubRepo = patternList(".*"); + + Scrubber scrubber = Scrubber.forOptions(options); + + assertThat(scrubber.forSpawn(createSpawn("Foo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("Foobar"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("Bar"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("BarFoo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("Baz"))).isNull(); + } + + @Test + public void simpleRepoPattern() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList("spam"); + + Scrubber scrubber = Scrubber.forOptions(options); + + assertThat(scrubber.forSpawn(createSpawn("spam", "Foo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("spameggs", "Foo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("bacon", "Foo"))).isNull(); + } + + @Test + public void anchoredRepoPattern() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList("^spam$"); + + Scrubber scrubber = Scrubber.forOptions(options); + + assertThat(scrubber.forSpawn(createSpawn("spam", "Foo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("spameggs", "Foo"))).isNull(); + assertThat(scrubber.forSpawn(createSpawn("bacon", "Foo"))).isNull(); + } + + @Test + public void wildcardRepoPattern() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList("spam[12]"); + + Scrubber scrubber = Scrubber.forOptions(options); + + assertThat(scrubber.forSpawn(createSpawn("spam1", "Foo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("spam2", "Foo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("spam3", "Foo"))).isNull(); + } + + @Test + public void multipleRepoPatterns() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList("spam", "eggs"); + + Scrubber scrubber = Scrubber.forOptions(options); + + assertThat(scrubber.forSpawn(createSpawn("spam", "Foo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("spambacon", "Foo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("eggs", "Foo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("eggsbacon", "Foo"))).isNotNull(); + assertThat(scrubber.forSpawn(createSpawn("ham", "Foo"))).isNull(); + } + + @Test + public void rejectToolAction() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + options.scrubExec = false; + + assertThat(Scrubber.forOptions(options) + .forSpawn(createSpawn("spam", "Foo", /* forTool= */ true))).isNull(); + } + + @Test + public void acceptToolAction() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + options.scrubExec = true; + + assertThat(Scrubber.forOptions(options) + .forSpawn(createSpawn("spam", "Foo", /* forTool= */ true))).isNotNull(); + } + + @Test + public void noInputPattern() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + + SpawnScrubber spawnScrubber = Scrubber.forOptions(options).forSpawn(createSpawn()); + + assertThat(spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("foo/bar"))).isFalse(); + } + + @Test + public void simpleInputPattern() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + options.scrubInput = patternList("foo/bar"); + + SpawnScrubber spawnScrubber = Scrubber.forOptions(options).forSpawn(createSpawn()); + + assertThat(spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("foo/bar"))).isTrue(); + assertThat(spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("foo/bar/baz"))).isTrue(); + assertThat(spawnScrubber.shouldOmitInput( + ActionInputHelper.fromPath("bazel-out/host/bin/foo/bar"))).isTrue(); + assertThat(spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("foo/baz"))).isFalse(); + } + + @Test + public void anchoredInputPattern() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + options.scrubInput = patternList("^foo/bar$"); + + SpawnScrubber spawnScrubber = Scrubber.forOptions(options).forSpawn(createSpawn()); + + assertThat(spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("foo/bar"))).isTrue(); + assertThat(spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("foo/bar/baz"))).isFalse(); + assertThat(spawnScrubber.shouldOmitInput( + ActionInputHelper.fromPath("bazel-out/host/bin/foo/bar"))).isFalse(); + } + + @Test + public void wildcardInputPattern() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + options.scrubInput = patternList("foo/bar/[12]"); + + SpawnScrubber spawnScrubber = Scrubber.forOptions(options).forSpawn(createSpawn()); + + assertThat(spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("foo/bar/1"))).isTrue(); + assertThat(spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("foo/bar/2"))).isTrue(); + assertThat(spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("foo/bar/3"))).isFalse(); + } + + @Test + public void multipleInputPatterns() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + options.scrubInput = patternList("foo/bar", "spam/eggs"); + + SpawnScrubber spawnScrubber = Scrubber.forOptions(options).forSpawn(createSpawn()); + + assertThat(spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("foo/bar/baz"))).isTrue(); + assertThat( + spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("spam/eggs/bacon"))).isTrue(); + assertThat(spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("foo/baz"))).isFalse(); + assertThat(spawnScrubber.shouldOmitInput(ActionInputHelper.fromPath("spam/bacon"))).isFalse(); + } + + @Test + public void doNotScrubEmptyMarker() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + options.scrubInput = patternList(".*"); + + SpawnScrubber spawnScrubber = Scrubber.forOptions(options).forSpawn(createSpawn()); + + assertThat(spawnScrubber.shouldOmitInput(VirtualActionInput.EMPTY_MARKER)).isFalse(); + } + + @Test + public void simpleArgReplacement() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + options.scrubArgReplacements = replacementList("foo", "bar"); + + SpawnScrubber spawnScrubber = Scrubber.forOptions(options).forSpawn(createSpawn()); + + assertThat(spawnScrubber.transformArgument("foo")).isEqualTo("bar"); + assertThat(spawnScrubber.transformArgument("abcfooxyz")).isEqualTo("abcbarxyz"); + assertThat(spawnScrubber.transformArgument("bar")).isEqualTo("bar"); + assertThat(spawnScrubber.transformArgument("foofoo")).isEqualTo("barfoo"); + } + + @Test + public void anchoredArgReplacement() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + options.scrubArgReplacements = replacementList("^foo$", "bar"); + + SpawnScrubber spawnScrubber = Scrubber.forOptions(options).forSpawn(createSpawn()); + + assertThat(spawnScrubber.transformArgument("foo")).isEqualTo("bar"); + assertThat(spawnScrubber.transformArgument("abcfooxyz")).isEqualTo("abcfooxyz"); + } + + @Test + public void wildcardArgReplacement() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + options.scrubArgReplacements = replacementList("foo[12]", "bar"); + + SpawnScrubber spawnScrubber = Scrubber.forOptions(options).forSpawn(createSpawn()); + + assertThat(spawnScrubber.transformArgument("foo1")).isEqualTo("bar"); + assertThat(spawnScrubber.transformArgument("foo2")).isEqualTo("bar"); + assertThat(spawnScrubber.transformArgument("foo3")).isEqualTo("foo3"); + } + + @Test + public void multipleArgReplacements() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + options.scrubArgReplacements = replacementList("foo", "bar", "spam", "eggs"); + + SpawnScrubber spawnScrubber = Scrubber.forOptions(options).forSpawn(createSpawn()); + + assertThat(spawnScrubber.transformArgument("abcfoo123spamxyz")).isEqualTo("abcbar123eggsxyz"); + assertThat(spawnScrubber.transformArgument("abcfoo")).isEqualTo("abcbar"); + assertThat(spawnScrubber.transformArgument("abcspam")).isEqualTo("abceggs"); + assertThat(spawnScrubber.transformArgument("bareggs")).isEqualTo("bareggs"); + } + + @Test + public void withoutSalt() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + + SpawnScrubber spawnScrubber = Scrubber.forOptions(options).forSpawn(createSpawn()); + + assertThat(spawnScrubber.getSalt()).isEmpty(); + } + + @Test + public void withSalt() { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.scrubEnabled = true; + options.scrubMnemonic = patternList(".*"); + options.scrubRepo = patternList(".*"); + options.scrubSalt = "NaCl"; + + SpawnScrubber spawnScrubber = Scrubber.forOptions(options).forSpawn(createSpawn()); + + assertThat(spawnScrubber.getSalt()).isEqualTo("NaCl"); + } + + private static ImmutableList patternList(String... patterns) { + ImmutableList.Builder builder = ImmutableList.builder(); + for (String pattern : patterns) { + builder.add(RegexPatternOption.create(Pattern.compile(pattern))); + } + return builder.build(); + } + + private static ImmutableList> replacementList( + String... args) { + checkState(args.length % 2 == 0); + ImmutableList.Builder> builder = ImmutableList.builder(); + for (int i = 0; i < args.length; i += 2) { + builder.add( + new SimpleEntry<>(RegexPatternOption.create(Pattern.compile(args[i])), args[i + 1])); + } + return builder.build(); + } + + private static Spawn createSpawn() { + return createSpawn(/* mnemonic= */ ""); + } + + private static Spawn createSpawn(String mnemonic) { + return createSpawn(/* repo= */ "", mnemonic); + } + + private static Spawn createSpawn(String repo, String mnemonic) { + return createSpawn(repo, mnemonic, /* forTool= */ false); + } + + private static Spawn createSpawn(String repo, String mnemonic, boolean forTool) { + return new SpawnBuilder("cmd").withOwnerLabel(String.format("@%s//some:target", repo)).withMnemonic(mnemonic).setBuiltForToolConfiguration(forTool).build(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/remote/merkletree/ActionInputDirectoryTreeTest.java b/src/test/java/com/google/devtools/build/lib/remote/merkletree/ActionInputDirectoryTreeTest.java index 87eef72bf77077..0b6ef3c1200f4b 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/merkletree/ActionInputDirectoryTreeTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/merkletree/ActionInputDirectoryTreeTest.java @@ -55,6 +55,7 @@ protected DirectoryTree build(Path... paths) throws IOException { new StaticInputMetadataProvider(metadata), execRoot, ArtifactPathResolver.forExecRoot(execRoot), + /* spawnScrubber= */ null, digestUtil); } @@ -72,6 +73,7 @@ public void virtualActionInputShouldWork() throws Exception { new StaticInputMetadataProvider(metadata), execRoot, ArtifactPathResolver.forExecRoot(execRoot), + /* spawnScrubber= */ null, digestUtil); assertLexicographicalOrder(tree); @@ -121,6 +123,7 @@ public void directoryInputShouldBeExpanded() throws Exception { new StaticInputMetadataProvider(metadata), execRoot, ArtifactPathResolver.forExecRoot(execRoot), + /* spawnScrubber= */ null, digestUtil); assertLexicographicalOrder(tree); @@ -169,6 +172,7 @@ public void filesShouldBeDeduplicated() throws Exception { new StaticInputMetadataProvider(metadata), execRoot, ArtifactPathResolver.forExecRoot(execRoot), + /* spawnScrubber= */ null, digestUtil); assertLexicographicalOrder(tree); diff --git a/src/test/java/com/google/devtools/build/lib/remote/merkletree/MerkleTreeTest.java b/src/test/java/com/google/devtools/build/lib/remote/merkletree/MerkleTreeTest.java index 86be3e066731a8..6c91c85bde8517 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/merkletree/MerkleTreeTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/merkletree/MerkleTreeTest.java @@ -74,6 +74,7 @@ public void emptyMerkleTree() throws IOException { new StaticInputMetadataProvider(Collections.emptyMap()), execRoot, ArtifactPathResolver.forExecRoot(execRoot), + /* spawnScrubber= */ null, digestUtil); Digest emptyDigest = digestUtil.compute(new byte[0]); assertThat(tree.getRootDigest()).isEqualTo(emptyDigest); @@ -115,6 +116,7 @@ public void buildMerkleTree() throws IOException { new StaticInputMetadataProvider(metadata), execRoot, ArtifactPathResolver.forExecRoot(execRoot), + /* spawnScrubber= */ null, digestUtil); // assert @@ -168,6 +170,7 @@ public void mergeMerkleTrees() throws IOException { new StaticInputMetadataProvider(metadata), execRoot, ArtifactPathResolver.forExecRoot(execRoot), + /* spawnScrubber= */ null, digestUtil); MerkleTree tree1 = MerkleTree.build( @@ -175,6 +178,7 @@ public void mergeMerkleTrees() throws IOException { new StaticInputMetadataProvider(metadata), execRoot, ArtifactPathResolver.forExecRoot(execRoot), + /* spawnScrubber= */ null, digestUtil); MerkleTree tree2 = MerkleTree.build( @@ -182,6 +186,7 @@ public void mergeMerkleTrees() throws IOException { new StaticInputMetadataProvider(metadata), execRoot, ArtifactPathResolver.forExecRoot(execRoot), + /* spawnScrubber= */ null, digestUtil); MerkleTree treeAll = MerkleTree.build( @@ -189,6 +194,7 @@ public void mergeMerkleTrees() throws IOException { new StaticInputMetadataProvider(metadata), execRoot, ArtifactPathResolver.forExecRoot(execRoot), + /* spawnScrubber= */ null, digestUtil); // act