diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index d9c57260139340..b369931fd13397 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -453,6 +453,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/skyframe/config", "//src/main/java/com/google/devtools/build/lib/skyframe/rewinding:action_rewound_event", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization", + "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:dependencies_provider", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:event_listener", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:frontier_serializer", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:options", diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java index 385705d3e6b4db..a52c3e9b181073 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java @@ -23,7 +23,10 @@ import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.common.base.Stopwatch; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableClassToInstanceMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; @@ -32,6 +35,7 @@ import com.google.common.collect.ListMultimap; import com.google.common.collect.Multimaps; import com.google.common.flogger.GoogleLogger; +import com.google.devtools.build.lib.actions.Artifact.ArtifactSerializationContext; import com.google.devtools.build.lib.actions.BuildFailedException; import com.google.devtools.build.lib.actions.CommandLineExpansionException; import com.google.devtools.build.lib.actions.TestExecException; @@ -43,6 +47,8 @@ import com.google.devtools.build.lib.analysis.ViewCreationFailedException; import com.google.devtools.build.lib.analysis.actions.TemplateExpansionException; import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.BuildOptions.MapBackedChecksumCache; +import com.google.devtools.build.lib.analysis.config.BuildOptions.OptionsChecksumCache; import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; import com.google.devtools.build.lib.buildeventstream.BuildEvent.LocalFile.LocalFileType; import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader.UploadContext; @@ -56,6 +62,7 @@ import com.google.devtools.build.lib.buildtool.buildevent.NoExecutionEvent; import com.google.devtools.build.lib.buildtool.buildevent.StartingAqueryDumpAfterBuildEvent; import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.cmdline.TargetParsingException; import com.google.devtools.build.lib.collect.PathFragmentPrefixTrie; import com.google.devtools.build.lib.events.Event; @@ -63,6 +70,7 @@ import com.google.devtools.build.lib.events.OutputFilter; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.packages.RuleClassProvider; import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.pkgcache.LoadingFailedException; import com.google.devtools.build.lib.profiler.ProfilePhase; @@ -75,6 +83,7 @@ import com.google.devtools.build.lib.server.FailureDetails.BuildConfiguration.Code; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.skyframe.BuildResultListener; +import com.google.devtools.build.lib.skyframe.PrerequisitePackageFunction; import com.google.devtools.build.lib.skyframe.ProjectValue; import com.google.devtools.build.lib.skyframe.RepositoryMappingValue.RepositoryMappingResolutionException; import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutor; @@ -86,7 +95,11 @@ import com.google.devtools.build.lib.skyframe.actiongraph.v2.AqueryOutputHandler.OutputType; import com.google.devtools.build.lib.skyframe.actiongraph.v2.InvalidAqueryOutputFormatException; import com.google.devtools.build.lib.skyframe.config.FlagSetValue; +import com.google.devtools.build.lib.skyframe.serialization.FingerprintValueService; +import com.google.devtools.build.lib.skyframe.serialization.ObjectCodecRegistry; +import com.google.devtools.build.lib.skyframe.serialization.ObjectCodecs; import com.google.devtools.build.lib.skyframe.serialization.analysis.FrontierSerializer; +import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingDependenciesProvider; import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingOptions; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.CrashFailureDetails; @@ -95,6 +108,7 @@ import com.google.devtools.build.lib.util.InterruptedFailureDetails; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Root; import com.google.devtools.build.skyframe.EvaluationResult; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.RegexPatternOption; @@ -224,6 +238,10 @@ public void buildTargets(BuildRequest request, BuildResult result, TargetValidat ProjectEvaluationResult projectEvaluationResult = evaluateProjectFile(request, buildOptions, targetPatternPhaseValue, env); + var remoteAnalysisCachingDependenciesProvider = + new RemoteAnalysisCachingDependenciesProviderImpl( + env, projectEvaluationResult.activeDirectoriesMatcher()); + // ------------------------------------ // Analysis/Execution phases (Skymeld). // ------------------------------------ @@ -235,7 +253,7 @@ public void buildTargets(BuildRequest request, BuildResult result, TargetValidat // Skymeld is useful only for commands that perform execution. buildTargetsWithMergedAnalysisExecution( request, result, targetPatternPhaseValue, projectEvaluationResult.buildOptions()); - serializeFrontier(request, projectEvaluationResult.activeDirectoriesMatcher()); + serializeFrontier(request, remoteAnalysisCachingDependenciesProvider); return; } @@ -303,7 +321,7 @@ public void buildTargets(BuildRequest request, BuildResult result, TargetValidat delayedFailureDetail.getMessage(), DetailedExitCode.of(delayedFailureDetail)); } - serializeFrontier(request, projectEvaluationResult.activeDirectoriesMatcher()); + serializeFrontier(request, remoteAnalysisCachingDependenciesProvider); // Only run this post-build step for builds with SequencedSkyframeExecutor. if (env.getSkyframeExecutor() instanceof SequencedSkyframeExecutor) { @@ -807,7 +825,8 @@ public BuildResult processRequest( /** Frontier serialization is common to both Skymeld and non-Skymeld builds. */ private void serializeFrontier( - BuildRequest request, Optional activeDirectoriesMatcher) + BuildRequest request, + RemoteAnalysisCachingDependenciesProvider remoteAnalysisCachingDependenciesProvider) throws InterruptedException, AbruptExitException { if (!(env.getSkyframeExecutor() instanceof SequencedSkyframeExecutor)) { return; @@ -819,18 +838,10 @@ private void serializeFrontier( } try (SilentCloseable closeable = Profiler.instance().profile("serializeAndUploadFrontier")) { - // TODO: b/353233779 - consider falling back on full serialization when there is - // no project matcher. - Preconditions.checkState( - activeDirectoriesMatcher.isPresent(), - "the PROJECT.scl active_directories matcher is missing, was it initialized correctly?"); - Optional maybeFailureDetail = FrontierSerializer.serializeAndUploadFrontier( - requireNonNull(env.getAnalysisObjectCodecsSupplier()), + remoteAnalysisCachingDependenciesProvider, env.getSkyframeExecutor(), - activeDirectoriesMatcher.get(), - requireNonNull(env.getFingerprintValueService()), env.getReporter(), env.getEventBus(), options.serializedFrontierProfile); @@ -1062,4 +1073,61 @@ DetailedExitCode getDetailedExitCode() { return detailedExitCode; } } + + static class RemoteAnalysisCachingDependenciesProviderImpl + implements RemoteAnalysisCachingDependenciesProvider { + private final Supplier analysisObjectCodecsSupplier; + private final FingerprintValueService fingerprintValueService; + private final Optional activeDirectoriesMatcher; + + public RemoteAnalysisCachingDependenciesProviderImpl( + CommandEnvironment env, Optional activeDirectoriesMatcher) { + this.analysisObjectCodecsSupplier = + Suppliers.memoize( + () -> + initAnalysisObjectCodecs( + requireNonNull( + env.getBlazeWorkspace().getAnalysisObjectCodecRegistrySupplier()) + .get(), + env.getRuntime().getRuleClassProvider(), + env.getBlazeWorkspace().getSkyframeExecutor())); + this.fingerprintValueService = + env.getBlazeWorkspace().getFingerprintValueServiceFactory().create(env.getOptions()); + this.activeDirectoriesMatcher = activeDirectoriesMatcher; + } + + private static ObjectCodecs initAnalysisObjectCodecs( + ObjectCodecRegistry registry, + RuleClassProvider ruleClassProvider, + SkyframeExecutor skyframeExecutor) { + ImmutableClassToInstanceMap.Builder serializationDeps = + ImmutableClassToInstanceMap.builder() + .put( + ArtifactSerializationContext.class, + skyframeExecutor.getSkyframeBuildView().getArtifactFactory()::getSourceArtifact) + .put(OptionsChecksumCache.class, new MapBackedChecksumCache()) + .put(RuleClassProvider.class, ruleClassProvider) + // We need a RootCodecDependencies but don't care about the likely roots. + .put(Root.RootCodecDependencies.class, new Root.RootCodecDependencies()) + // This is needed to determine TargetData for a ConfiguredTarget during serialization. + .put(PrerequisitePackageFunction.class, skyframeExecutor::getExistingPackage); + + return new ObjectCodecs(registry, serializationDeps.build()); + } + + @Override + public boolean withinActiveDirectories(PackageIdentifier pkg) { + return activeDirectoriesMatcher.get().includes(pkg.getPackageFragment()); + } + + @Override + public ObjectCodecs getObjectCodecs() { + return analysisObjectCodecsSupplier.get(); + } + + @Override + public FingerprintValueService getFingerprintValueService() { + return fingerprintValueService; + } + } } diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeWorkspace.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeWorkspace.java index 7c7e766ed69c50..aeca4123d833b8 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeWorkspace.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeWorkspace.java @@ -76,6 +76,11 @@ public final class BlazeWorkspace { private final SyscallCache syscallCache; private final QuiescingExecutorsImpl quiescingExecutors; @Nullable private final Supplier analysisCodecRegistrySupplier; + + /** + * Null only during tests; should be created by a BlazeModule#workspaceInit hook for regular + * operations. + */ @Nullable private final FingerprintValueService.Factory fingerprintValueServiceFactory; /** @@ -101,7 +106,7 @@ public BlazeWorkspace( @Nullable AllocationTracker allocationTracker, SyscallCache syscallCache, Supplier analysisCodecRegistrySupplier, - FingerprintValueService.Factory fingerprintValueServiceFactory, + @Nullable FingerprintValueService.Factory fingerprintValueServiceFactory, boolean allowExternalRepositories) { this.runtime = runtime; this.eventBusExceptionHandler = Preconditions.checkNotNull(eventBusExceptionHandler); @@ -369,8 +374,10 @@ public Supplier getAnalysisObjectCodecRegistrySupplier() { return analysisCodecRegistrySupplier; } - @Nullable public FingerprintValueService.Factory getFingerprintValueServiceFactory() { + if (fingerprintValueServiceFactory == null) { + return (unused) -> FingerprintValueService.createForTesting(); + } return fingerprintValueServiceFactory; } diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java index a27efa11767037..096a2965f25225 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java @@ -17,26 +17,20 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; -import static com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingOptions.RemoteAnalysisCacheMode.OFF; -import static java.util.Objects.requireNonNull; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableClassToInstanceMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedMap; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; import com.google.devtools.build.lib.actions.ActionOutputDirectoryHelper; -import com.google.devtools.build.lib.actions.Artifact.ArtifactSerializationContext; import com.google.devtools.build.lib.actions.InputMetadataProvider; import com.google.devtools.build.lib.actions.ResourceManager; import com.google.devtools.build.lib.analysis.AnalysisOptions; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.BuildInfoEvent; import com.google.devtools.build.lib.analysis.config.AdditionalConfigurationChangeEvent; -import com.google.devtools.build.lib.analysis.config.BuildOptions.MapBackedChecksumCache; -import com.google.devtools.build.lib.analysis.config.BuildOptions.OptionsChecksumCache; import com.google.devtools.build.lib.analysis.config.CoreOptions; import com.google.devtools.build.lib.bazel.repository.downloader.DelegatingDownloader; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; @@ -47,7 +41,6 @@ import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.exec.SingleBuildFileCache; -import com.google.devtools.build.lib.packages.RuleClassProvider; import com.google.devtools.build.lib.pkgcache.PackageManager; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; import com.google.devtools.build.lib.profiler.Profiler; @@ -60,16 +53,11 @@ import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.server.FailureDetails.Skyfocus; import com.google.devtools.build.lib.skyframe.BuildResultListener; -import com.google.devtools.build.lib.skyframe.PrerequisitePackageFunction; import com.google.devtools.build.lib.skyframe.SkyfocusOptions; import com.google.devtools.build.lib.skyframe.SkyframeBuildView; import com.google.devtools.build.lib.skyframe.SkyframeExecutor; import com.google.devtools.build.lib.skyframe.WorkspaceInfoFromDiff; -import com.google.devtools.build.lib.skyframe.serialization.FingerprintValueService; -import com.google.devtools.build.lib.skyframe.serialization.ObjectCodecRegistry; -import com.google.devtools.build.lib.skyframe.serialization.ObjectCodecs; import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingEventListener; -import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingOptions; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.io.CommandExtensionReporter; @@ -80,7 +68,6 @@ import com.google.devtools.build.lib.vfs.OutputService; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; -import com.google.devtools.build.lib.vfs.Root; import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.XattrProvider; import com.google.devtools.common.options.OptionAndRawValue; @@ -99,7 +86,6 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; -import java.util.function.Supplier; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; @@ -145,8 +131,6 @@ public class CommandEnvironment { private final int attemptNumber; private final HttpDownloader httpDownloader; private final DelegatingDownloader delegatingDownloader; - @Nullable private final Supplier analysisObjectCodecsSupplier; - @Nullable private final FingerprintValueService fingerprintValueService; private final RemoteAnalysisCachingEventListener remoteAnalysisCachingEventListener; private boolean mergedAnalysisAndExecution; @@ -367,44 +351,10 @@ public void exit(AbruptExitException exception) { this.commandLinePathFactory = CommandLinePathFactory.create(runtime.getFileSystem(), directories); - var remoteAnalysisCachingOptions = options.getOptions(RemoteAnalysisCachingOptions.class); - if (remoteAnalysisCachingOptions != null && remoteAnalysisCachingOptions.mode != OFF) { - this.analysisObjectCodecsSupplier = - () -> - initAnalysisObjectCodecs( - requireNonNull(workspace.getAnalysisObjectCodecRegistrySupplier()).get(), - runtime.getRuleClassProvider(), - workspace.getSkyframeExecutor()); - this.fingerprintValueService = - requireNonNull(workspace.getFingerprintValueServiceFactory()).create(options); - } else { - this.analysisObjectCodecsSupplier = null; - this.fingerprintValueService = null; - } this.remoteAnalysisCachingEventListener = new RemoteAnalysisCachingEventListener(); this.eventBus.register(remoteAnalysisCachingEventListener); } - private static ObjectCodecs initAnalysisObjectCodecs( - ObjectCodecRegistry registry, - RuleClassProvider ruleClassProvider, - SkyframeExecutor skyframeExecutor) { - - ImmutableClassToInstanceMap.Builder serializationDeps = - ImmutableClassToInstanceMap.builder() - .put( - ArtifactSerializationContext.class, - skyframeExecutor.getSkyframeBuildView().getArtifactFactory()::getSourceArtifact) - .put(OptionsChecksumCache.class, new MapBackedChecksumCache()) - .put(RuleClassProvider.class, ruleClassProvider) - // We need a RootCodecDependencies but don't care about the likely roots. - .put(Root.RootCodecDependencies.class, new Root.RootCodecDependencies()) - // This is needed to determine TargetData for a ConfiguredTarget during serialization. - .put(PrerequisitePackageFunction.class, skyframeExecutor::getExistingPackage); - - return new ObjectCodecs(registry, serializationDeps.build()); - } - private Path computeWorkingDirectory(CommonCommandOptions commandOptions) throws AbruptExitException { Path workspace = getWorkspace(); @@ -1077,20 +1027,4 @@ public HttpDownloader getHttpDownloader() { public DelegatingDownloader getDownloaderDelegate() { return delegatingDownloader; } - - /** - * Returns the {@link ObjectCodecs} supplier for remote analysis caching. - * - *

Calling #get on this can be an expensive process as the codec registry will be constructed. - * Callers are advised to get the object asynchronously. - */ - @Nullable - public Supplier getAnalysisObjectCodecsSupplier() { - return analysisObjectCodecsSupplier; - } - - @Nullable - public FingerprintValueService getFingerprintValueService() { - return fingerprintValueService; - } } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/BUILD index 280f8995fbdaa0..a6f21b06b050a0 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/BUILD @@ -21,14 +21,23 @@ java_library( ], ) +java_library( + name = "dependencies_provider", + srcs = ["RemoteAnalysisCachingDependenciesProvider.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/skyframe/serialization", + ], +) + java_library( name = "frontier_serializer", srcs = ["FrontierSerializer.java"], deps = [ + ":dependencies_provider", ":event_listener", "//src/main/java/com/google/devtools/build/lib/actions:action_lookup_key", "//src/main/java/com/google/devtools/build/lib/cmdline", - "//src/main/java/com/google/devtools/build/lib/collect", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/skyframe:aspect_key_creator", "//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster", diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/FrontierSerializer.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/FrontierSerializer.java index 8aa37d41754a13..626fd5b0e04acb 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/FrontierSerializer.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/FrontierSerializer.java @@ -27,7 +27,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.devtools.build.lib.actions.ActionLookupKey; import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.collect.PathFragmentPrefixTrie; +import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.server.FailureDetails; @@ -35,7 +35,6 @@ import com.google.devtools.build.lib.server.FailureDetails.RemoteAnalysisCaching.Code; import com.google.devtools.build.lib.skyframe.AspectKeyCreator.AspectBaseKey; import com.google.devtools.build.lib.skyframe.SkyframeExecutor; -import com.google.devtools.build.lib.skyframe.serialization.FingerprintValueService; import com.google.devtools.build.lib.skyframe.serialization.ObjectCodecs; import com.google.devtools.build.lib.skyframe.serialization.ProfileCollector; import com.google.devtools.build.lib.skyframe.serialization.SerializationException; @@ -55,7 +54,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.FutureTask; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; +import java.util.function.Predicate; /** * Implements frontier serialization with pprof dumping using {@code @@ -72,23 +71,21 @@ private FrontierSerializer() {} * @return empty if successful, otherwise a result containing the appropriate error */ public static Optional serializeAndUploadFrontier( - Supplier codecsSupplier, + RemoteAnalysisCachingDependenciesProvider dependenciesProvider, SkyframeExecutor skyframeExecutor, - PathFragmentPrefixTrie matcher, - FingerprintValueService fingerprintValueService, Reporter reporter, EventBus eventBus, String profilePath) throws InterruptedException { // Starts initializing ObjectCodecs in a background thread as it can take some time. - var futureCodecs = new FutureTask<>(codecsSupplier::get); + var futureCodecs = new FutureTask<>(dependenciesProvider::getObjectCodecs); commonPool().execute(futureCodecs); var stopwatch = new ResettingStopwatch(Stopwatch.createStarted()); InMemoryGraph graph = skyframeExecutor.getEvaluator().getInMemoryGraph(); ConcurrentHashMap selection = - computeSelection(graph, matcher); + computeSelection(graph, dependenciesProvider::withinActiveDirectories); reporter.handle( Event.info( String.format("Found %d active or frontier keys in %s", selection.size(), stopwatch))); @@ -111,6 +108,7 @@ public static Optional serializeAndUploadFrontier( var writeStatuses = Collections.synchronizedList(new ArrayList>()); AtomicInteger frontierValueCount = new AtomicInteger(); + var fingerprintValueService = dependenciesProvider.getFingerprintValueService(); selection.forEach( /* parallelismThreshold= */ 0, (key, marking) -> { @@ -197,7 +195,7 @@ enum SelectionMarking { @VisibleForTesting static ConcurrentHashMap computeSelection( - InMemoryGraph graph, PathFragmentPrefixTrie matcher) { + InMemoryGraph graph, Predicate matcher) { var selection = new ConcurrentHashMap(); graph.parallelForEach( node -> { @@ -208,7 +206,7 @@ static ConcurrentHashMap computeSelection( if (label == null) { return; } - if (!matcher.includes(label.getPackageFragment())) { + if (!matcher.test(label.getPackageIdentifier())) { return; } markActiveAndTraverseEdges(graph, actionLookupKey, selection); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/RemoteAnalysisCachingDependenciesProvider.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/RemoteAnalysisCachingDependenciesProvider.java new file mode 100644 index 00000000000000..98da415b34b627 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/RemoteAnalysisCachingDependenciesProvider.java @@ -0,0 +1,38 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe.serialization.analysis; + +import com.google.devtools.build.lib.cmdline.PackageIdentifier; +import com.google.devtools.build.lib.skyframe.serialization.FingerprintValueService; +import com.google.devtools.build.lib.skyframe.serialization.ObjectCodecs; + +/** + * An interface providing the functionalities used for analysis caching serialization and + * deserialization. + */ +public interface RemoteAnalysisCachingDependenciesProvider { + + /** Returns true if the {@link PackageIdentifier} is in the set of active directories. */ + boolean withinActiveDirectories(PackageIdentifier pkg); + + /** + * Returns the {@link ObjectCodecs} supplier for remote analysis caching. + * + *

Calling this can be an expensive process as the codec registry will be initialized. + */ + ObjectCodecs getObjectCodecs(); + + /** Returns the {@link FingerprintValueService} implementation. */ + FingerprintValueService getFingerprintValueService(); +} diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/serialization/analysis/FrontierSerializerTestBase.java b/src/test/java/com/google/devtools/build/lib/skyframe/serialization/analysis/FrontierSerializerTestBase.java index c77b5839317be1..b0636ba4f223d8 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/serialization/analysis/FrontierSerializerTestBase.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/serialization/analysis/FrontierSerializerTestBase.java @@ -26,6 +26,7 @@ import com.google.devtools.build.lib.buildtool.BuildTool; import com.google.devtools.build.lib.buildtool.util.BuildIntegrationTestCase; import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.pkgcache.LoadingFailedException; import com.google.devtools.build.lib.skyframe.AspectKeyCreator.AspectBaseKey; import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; @@ -168,14 +169,15 @@ public void activeAspect_activatesBaseConfiguredTarget() throws Exception { assertThat(key).isInstanceOf(AspectBaseKey.class); } + var matcher = + BuildTool.getWorkingSetMatcherForSkyfocus( + // We know exactly which PROJECT file is used, so inject it here. + Label.parseCanonicalUnchecked("//bar:PROJECT.scl"), + getSkyframeExecutor(), + getCommandEnvironment().getReporter()); Map selection = FrontierSerializer.computeSelection( - graph, - BuildTool.getWorkingSetMatcherForSkyfocus( - // We know exactly which PROJECT file is used, so inject it here. - Label.parseCanonicalUnchecked("//bar:PROJECT.scl"), - getSkyframeExecutor(), - getCommandEnvironment().getReporter())); + graph, (PackageIdentifier pkgId) -> matcher.includes(pkgId.getPackageFragment())); ImmutableSet activeKeys = selection.entrySet().stream()