Skip to content

Commit

Permalink
Implement a RemoteAnalysisCachingDependenciesProvider interface.
Browse files Browse the repository at this point in the history 
This interface groups together the functionalities needed for serialization, such as `ObjectCodecs`, `FingerprintValueStore`, and the `PathFragmentPrefixTrie` (for active_directories boundary matching).

The implementation is instantiated immediately after evaluating `PROJECT.scl`.
The implementation can then be passed as an argument into the analysis and
execution phases for SkyFunction in-line deserialization operations.

For this CL, we are just updating `FrontierSerializer` to use this provider
instead of individual dependency objects.

PiperOrigin-RevId: 673722701
Change-Id: I7c0426185d2dba9659d204019f022e5bb5a77725
  • Loading branch information
@jin @copybara-github
jin authored and copybara-github committed Sep 12, 2024
1 parent 1c4d145 commit be9668d
Show file tree
Hide file tree
Showing 8 changed files with 154 additions and 97 deletions.
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,6 +453,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/skyframe/config",
"//src/main/java/com/google/devtools/build/lib/skyframe/rewinding:action_rewound_event",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:dependencies_provider",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:event_listener",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:frontier_serializer",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:options",
Expand Down
92 changes: 80 additions & 12 deletions src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Stopwatch;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableClassToInstanceMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
Expand All @@ -32,6 +35,7 @@
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Multimaps;
import com.google.common.flogger.GoogleLogger;
import com.google.devtools.build.lib.actions.Artifact.ArtifactSerializationContext;
import com.google.devtools.build.lib.actions.BuildFailedException;
import com.google.devtools.build.lib.actions.CommandLineExpansionException;
import com.google.devtools.build.lib.actions.TestExecException;
Expand All @@ -43,6 +47,8 @@
import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
import com.google.devtools.build.lib.analysis.actions.TemplateExpansionException;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.BuildOptions.MapBackedChecksumCache;
import com.google.devtools.build.lib.analysis.config.BuildOptions.OptionsChecksumCache;
import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
import com.google.devtools.build.lib.buildeventstream.BuildEvent.LocalFile.LocalFileType;
import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader.UploadContext;
Expand All @@ -56,13 +62,15 @@
import com.google.devtools.build.lib.buildtool.buildevent.NoExecutionEvent;
import com.google.devtools.build.lib.buildtool.buildevent.StartingAqueryDumpAfterBuildEvent;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.cmdline.TargetParsingException;
import com.google.devtools.build.lib.collect.PathFragmentPrefixTrie;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.events.OutputFilter;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.exec.ExecutionOptions;
import com.google.devtools.build.lib.packages.RuleClassProvider;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.pkgcache.LoadingFailedException;
import com.google.devtools.build.lib.profiler.ProfilePhase;
Expand All @@ -75,6 +83,7 @@
import com.google.devtools.build.lib.server.FailureDetails.BuildConfiguration.Code;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.skyframe.BuildResultListener;
import com.google.devtools.build.lib.skyframe.PrerequisitePackageFunction;
import com.google.devtools.build.lib.skyframe.ProjectValue;
import com.google.devtools.build.lib.skyframe.RepositoryMappingValue.RepositoryMappingResolutionException;
import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutor;
Expand All @@ -86,7 +95,11 @@
import com.google.devtools.build.lib.skyframe.actiongraph.v2.AqueryOutputHandler.OutputType;
import com.google.devtools.build.lib.skyframe.actiongraph.v2.InvalidAqueryOutputFormatException;
import com.google.devtools.build.lib.skyframe.config.FlagSetValue;
import com.google.devtools.build.lib.skyframe.serialization.FingerprintValueService;
import com.google.devtools.build.lib.skyframe.serialization.ObjectCodecRegistry;
import com.google.devtools.build.lib.skyframe.serialization.ObjectCodecs;
import com.google.devtools.build.lib.skyframe.serialization.analysis.FrontierSerializer;
import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingDependenciesProvider;
import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingOptions;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.CrashFailureDetails;
Expand All @@ -95,6 +108,7 @@
import com.google.devtools.build.lib.util.InterruptedFailureDetails;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.skyframe.EvaluationResult;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.common.options.RegexPatternOption;
Expand Down Expand Up @@ -224,6 +238,10 @@ public void buildTargets(BuildRequest request, BuildResult result, TargetValidat
ProjectEvaluationResult projectEvaluationResult =
evaluateProjectFile(request, buildOptions, targetPatternPhaseValue, env);

var remoteAnalysisCachingDependenciesProvider =
new RemoteAnalysisCachingDependenciesProviderImpl(
env, projectEvaluationResult.activeDirectoriesMatcher());

// ------------------------------------
// Analysis/Execution phases (Skymeld).
// ------------------------------------
Expand All @@ -235,7 +253,7 @@ public void buildTargets(BuildRequest request, BuildResult result, TargetValidat
// Skymeld is useful only for commands that perform execution.
buildTargetsWithMergedAnalysisExecution(
request, result, targetPatternPhaseValue, projectEvaluationResult.buildOptions());
serializeFrontier(request, projectEvaluationResult.activeDirectoriesMatcher());
serializeFrontier(request, remoteAnalysisCachingDependenciesProvider);
return;
}

Expand Down Expand Up @@ -303,7 +321,7 @@ public void buildTargets(BuildRequest request, BuildResult result, TargetValidat
delayedFailureDetail.getMessage(), DetailedExitCode.of(delayedFailureDetail));
}

serializeFrontier(request, projectEvaluationResult.activeDirectoriesMatcher());
serializeFrontier(request, remoteAnalysisCachingDependenciesProvider);

// Only run this post-build step for builds with SequencedSkyframeExecutor.
if (env.getSkyframeExecutor() instanceof SequencedSkyframeExecutor) {
Expand Down Expand Up @@ -807,7 +825,8 @@ public BuildResult processRequest(

/** Frontier serialization is common to both Skymeld and non-Skymeld builds. */
private void serializeFrontier(
BuildRequest request, Optional<PathFragmentPrefixTrie> activeDirectoriesMatcher)
BuildRequest request,
RemoteAnalysisCachingDependenciesProvider remoteAnalysisCachingDependenciesProvider)
throws InterruptedException, AbruptExitException {
if (!(env.getSkyframeExecutor() instanceof SequencedSkyframeExecutor)) {
return;
Expand All @@ -819,18 +838,10 @@ private void serializeFrontier(
}

try (SilentCloseable closeable = Profiler.instance().profile("serializeAndUploadFrontier")) {
// TODO: b/353233779 - consider falling back on full serialization when there is
// no project matcher.
Preconditions.checkState(
activeDirectoriesMatcher.isPresent(),
"the PROJECT.scl active_directories matcher is missing, was it initialized correctly?");

Optional<FailureDetail> maybeFailureDetail =
FrontierSerializer.serializeAndUploadFrontier(
requireNonNull(env.getAnalysisObjectCodecsSupplier()),
remoteAnalysisCachingDependenciesProvider,
env.getSkyframeExecutor(),
activeDirectoriesMatcher.get(),
requireNonNull(env.getFingerprintValueService()),
env.getReporter(),
env.getEventBus(),
options.serializedFrontierProfile);
Expand Down Expand Up @@ -1062,4 +1073,61 @@ DetailedExitCode getDetailedExitCode() {
return detailedExitCode;
}
}

static class RemoteAnalysisCachingDependenciesProviderImpl
implements RemoteAnalysisCachingDependenciesProvider {
private final Supplier<ObjectCodecs> analysisObjectCodecsSupplier;
private final FingerprintValueService fingerprintValueService;
private final Optional<PathFragmentPrefixTrie> activeDirectoriesMatcher;

public RemoteAnalysisCachingDependenciesProviderImpl(
CommandEnvironment env, Optional<PathFragmentPrefixTrie> activeDirectoriesMatcher) {
this.analysisObjectCodecsSupplier =
Suppliers.memoize(
() ->
initAnalysisObjectCodecs(
requireNonNull(
env.getBlazeWorkspace().getAnalysisObjectCodecRegistrySupplier())
.get(),
env.getRuntime().getRuleClassProvider(),
env.getBlazeWorkspace().getSkyframeExecutor()));
this.fingerprintValueService =
env.getBlazeWorkspace().getFingerprintValueServiceFactory().create(env.getOptions());
this.activeDirectoriesMatcher = activeDirectoriesMatcher;
}

private static ObjectCodecs initAnalysisObjectCodecs(
ObjectCodecRegistry registry,
RuleClassProvider ruleClassProvider,
SkyframeExecutor skyframeExecutor) {
ImmutableClassToInstanceMap.Builder<Object> serializationDeps =
ImmutableClassToInstanceMap.builder()
.put(
ArtifactSerializationContext.class,
skyframeExecutor.getSkyframeBuildView().getArtifactFactory()::getSourceArtifact)
.put(OptionsChecksumCache.class, new MapBackedChecksumCache())
.put(RuleClassProvider.class, ruleClassProvider)
// We need a RootCodecDependencies but don't care about the likely roots.
.put(Root.RootCodecDependencies.class, new Root.RootCodecDependencies())
// This is needed to determine TargetData for a ConfiguredTarget during serialization.
.put(PrerequisitePackageFunction.class, skyframeExecutor::getExistingPackage);

return new ObjectCodecs(registry, serializationDeps.build());
}

@Override
public boolean withinActiveDirectories(PackageIdentifier pkg) {
return activeDirectoriesMatcher.get().includes(pkg.getPackageFragment());
}

@Override
public ObjectCodecs getObjectCodecs() {
return analysisObjectCodecsSupplier.get();
}

@Override
public FingerprintValueService getFingerprintValueService() {
return fingerprintValueService;
}
}
}
11 changes: 9 additions & 2 deletions src/main/java/com/google/devtools/build/lib/runtime/BlazeWorkspace.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,11 @@ public final class BlazeWorkspace {
private final SyscallCache syscallCache;
private final QuiescingExecutorsImpl quiescingExecutors;
@Nullable private final Supplier<ObjectCodecRegistry> analysisCodecRegistrySupplier;

/**
* Null only during tests; should be created by a BlazeModule#workspaceInit hook for regular
* operations.
*/
@Nullable private final FingerprintValueService.Factory fingerprintValueServiceFactory;

/**
Expand All @@ -101,7 +106,7 @@ public BlazeWorkspace(
@Nullable AllocationTracker allocationTracker,
SyscallCache syscallCache,
Supplier<ObjectCodecRegistry> analysisCodecRegistrySupplier,
FingerprintValueService.Factory fingerprintValueServiceFactory,
@Nullable FingerprintValueService.Factory fingerprintValueServiceFactory,
boolean allowExternalRepositories) {
this.runtime = runtime;
this.eventBusExceptionHandler = Preconditions.checkNotNull(eventBusExceptionHandler);
Expand Down Expand Up @@ -369,8 +374,10 @@ public Supplier<ObjectCodecRegistry> getAnalysisObjectCodecRegistrySupplier() {
return analysisCodecRegistrySupplier;
}

@Nullable
public FingerprintValueService.Factory getFingerprintValueServiceFactory() {
if (fingerprintValueServiceFactory == null) {
return (unused) -> FingerprintValueService.createForTesting();
}
return fingerprintValueServiceFactory;
}

Expand Down
66 changes: 0 additions & 66 deletions src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,20 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingOptions.RemoteAnalysisCacheMode.OFF;
import static java.util.Objects.requireNonNull;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableClassToInstanceMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import com.google.devtools.build.lib.actions.ActionOutputDirectoryHelper;
import com.google.devtools.build.lib.actions.Artifact.ArtifactSerializationContext;
import com.google.devtools.build.lib.actions.InputMetadataProvider;
import com.google.devtools.build.lib.actions.ResourceManager;
import com.google.devtools.build.lib.analysis.AnalysisOptions;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.BuildInfoEvent;
import com.google.devtools.build.lib.analysis.config.AdditionalConfigurationChangeEvent;
import com.google.devtools.build.lib.analysis.config.BuildOptions.MapBackedChecksumCache;
import com.google.devtools.build.lib.analysis.config.BuildOptions.OptionsChecksumCache;
import com.google.devtools.build.lib.analysis.config.CoreOptions;
import com.google.devtools.build.lib.bazel.repository.downloader.DelegatingDownloader;
import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
Expand All @@ -47,7 +41,6 @@
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.exec.SingleBuildFileCache;
import com.google.devtools.build.lib.packages.RuleClassProvider;
import com.google.devtools.build.lib.pkgcache.PackageManager;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
import com.google.devtools.build.lib.profiler.Profiler;
Expand All @@ -60,16 +53,11 @@
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.Skyfocus;
import com.google.devtools.build.lib.skyframe.BuildResultListener;
import com.google.devtools.build.lib.skyframe.PrerequisitePackageFunction;
import com.google.devtools.build.lib.skyframe.SkyfocusOptions;
import com.google.devtools.build.lib.skyframe.SkyframeBuildView;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
import com.google.devtools.build.lib.skyframe.WorkspaceInfoFromDiff;
import com.google.devtools.build.lib.skyframe.serialization.FingerprintValueService;
import com.google.devtools.build.lib.skyframe.serialization.ObjectCodecRegistry;
import com.google.devtools.build.lib.skyframe.serialization.ObjectCodecs;
import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingEventListener;
import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingOptions;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.DetailedExitCode;
import com.google.devtools.build.lib.util.io.CommandExtensionReporter;
Expand All @@ -80,7 +68,6 @@
import com.google.devtools.build.lib.vfs.OutputService;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.lib.vfs.XattrProvider;
import com.google.devtools.common.options.OptionAndRawValue;
Expand All @@ -99,7 +86,6 @@
import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Supplier;
import javax.annotation.Nullable;
import javax.annotation.concurrent.GuardedBy;

Expand Down Expand Up @@ -145,8 +131,6 @@ public class CommandEnvironment {
private final int attemptNumber;
private final HttpDownloader httpDownloader;
private final DelegatingDownloader delegatingDownloader;
@Nullable private final Supplier<ObjectCodecs> analysisObjectCodecsSupplier;
@Nullable private final FingerprintValueService fingerprintValueService;
private final RemoteAnalysisCachingEventListener remoteAnalysisCachingEventListener;

private boolean mergedAnalysisAndExecution;
Expand Down Expand Up @@ -367,44 +351,10 @@ public void exit(AbruptExitException exception) {
this.commandLinePathFactory =
CommandLinePathFactory.create(runtime.getFileSystem(), directories);

var remoteAnalysisCachingOptions = options.getOptions(RemoteAnalysisCachingOptions.class);
if (remoteAnalysisCachingOptions != null && remoteAnalysisCachingOptions.mode != OFF) {
this.analysisObjectCodecsSupplier =
() ->
initAnalysisObjectCodecs(
requireNonNull(workspace.getAnalysisObjectCodecRegistrySupplier()).get(),
runtime.getRuleClassProvider(),
workspace.getSkyframeExecutor());
this.fingerprintValueService =
requireNonNull(workspace.getFingerprintValueServiceFactory()).create(options);
} else {
this.analysisObjectCodecsSupplier = null;
this.fingerprintValueService = null;
}
this.remoteAnalysisCachingEventListener = new RemoteAnalysisCachingEventListener();
this.eventBus.register(remoteAnalysisCachingEventListener);
}

private static ObjectCodecs initAnalysisObjectCodecs(
ObjectCodecRegistry registry,
RuleClassProvider ruleClassProvider,
SkyframeExecutor skyframeExecutor) {

ImmutableClassToInstanceMap.Builder<Object> serializationDeps =
ImmutableClassToInstanceMap.builder()
.put(
ArtifactSerializationContext.class,
skyframeExecutor.getSkyframeBuildView().getArtifactFactory()::getSourceArtifact)
.put(OptionsChecksumCache.class, new MapBackedChecksumCache())
.put(RuleClassProvider.class, ruleClassProvider)
// We need a RootCodecDependencies but don't care about the likely roots.
.put(Root.RootCodecDependencies.class, new Root.RootCodecDependencies())
// This is needed to determine TargetData for a ConfiguredTarget during serialization.
.put(PrerequisitePackageFunction.class, skyframeExecutor::getExistingPackage);

return new ObjectCodecs(registry, serializationDeps.build());
}

private Path computeWorkingDirectory(CommonCommandOptions commandOptions)
throws AbruptExitException {
Path workspace = getWorkspace();
Expand Down Expand Up @@ -1077,20 +1027,4 @@ public HttpDownloader getHttpDownloader() {
public DelegatingDownloader getDownloaderDelegate() {
return delegatingDownloader;
}

/**
* Returns the {@link ObjectCodecs} supplier for remote analysis caching.
*
* <p>Calling #get on this can be an expensive process as the codec registry will be constructed.
* Callers are advised to get the object asynchronously.
*/
@Nullable
public Supplier<ObjectCodecs> getAnalysisObjectCodecsSupplier() {
return analysisObjectCodecsSupplier;
}

@Nullable
public FingerprintValueService getFingerprintValueService() {
return fingerprintValueService;
}
}
Loading

0 comments on commit be9668d

Please sign in to comment.