From 08cc45b5812afd1c0b13bf6c36905b42d1469140 Mon Sep 17 00:00:00 2001 From: Salma Samy Date: Mon, 19 Feb 2024 14:55:00 +0200 Subject: [PATCH] [7.1.0] Fetch refactor and mod command fix (#21385) - Refactor fetch command ([commit](https://github.com/bazelbuild/bazel/commit/5cb37c1b2114e4feaac16073d596074a373d1b5c)) - Mod command fails when bzlmod is disabled ([commit](https://github.com/bazelbuild/bazel/commit/aa60e35eb9d2695b2d8ca8d8c9873bf1c2ba29d7)) --- site/en/run/build.md | 7 + .../lib/bazel/commands/FetchCommand.java | 248 ++++++------------ .../lib/bazel/commands/FetchOptions.java | 9 +- .../build/lib/bazel/commands/ModCommand.java | 21 +- .../lib/bazel/commands/TargetFetcher.java | 129 +++++++++ .../devtools/build/lib/bazel/commands/mod.txt | 1 + .../RepositoryDelegatorFunction.java | 3 - src/test/py/bazel/bzlmod/bazel_fetch_test.py | 12 +- src/test/py/bazel/bzlmod/mod_command_test.py | 10 + .../bazel/bazel_repository_cache_test.sh | 10 +- 10 files changed, 244 insertions(+), 206 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/commands/TargetFetcher.java diff --git a/site/en/run/build.md b/site/en/run/build.md index 2ecbb440564db9..6a371ce1fd5b5b 100644 --- a/site/en/run/build.md +++ b/site/en/run/build.md @@ -314,6 +314,13 @@ To fetch all external dependencies for a workspace, run: bazel fetch //... ``` +With Bazel 7.1 or later, if you have Bzlmod enabled, you can also fetch all +external dependencies by running + +```posix-terminal +bazel fetch +``` + You do not need to run bazel fetch at all if you have all of the tools you are using (from library jars to the JDK itself) under your workspace root. However, if you're using anything outside of the workspace directory then Bazel diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java index d745e80fb3ded8..f1cd8511a2adb2 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java @@ -11,12 +11,12 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package com.google.devtools.build.lib.bazel.commands; import static com.google.common.primitives.Booleans.countTrue; import static java.util.stream.Collectors.joining; -import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -24,24 +24,12 @@ import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent; import com.google.devtools.build.lib.bazel.bzlmod.BazelFetchAllValue; import com.google.devtools.build.lib.bazel.commands.RepositoryFetcher.RepositoryFetcherException; -import com.google.devtools.build.lib.cmdline.RepositoryMapping; +import com.google.devtools.build.lib.bazel.commands.TargetFetcher.TargetFetcherException; import com.google.devtools.build.lib.cmdline.RepositoryName; -import com.google.devtools.build.lib.cmdline.TargetPattern; -import com.google.devtools.build.lib.cmdline.TargetPattern.Parser; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.Reporter; -import com.google.devtools.build.lib.packages.LabelPrinter; -import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.pkgcache.PackageOptions; -import com.google.devtools.build.lib.query2.common.AbstractBlazeQueryEnvironment; -import com.google.devtools.build.lib.query2.common.UniverseScope; -import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting; -import com.google.devtools.build.lib.query2.engine.QueryEvalResult; -import com.google.devtools.build.lib.query2.engine.QueryException; -import com.google.devtools.build.lib.query2.engine.QueryExpression; -import com.google.devtools.build.lib.query2.engine.QuerySyntaxException; -import com.google.devtools.build.lib.query2.engine.ThreadSafeOutputFormatterCallback; import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue; import com.google.devtools.build.lib.runtime.BlazeCommand; @@ -50,24 +38,20 @@ import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.runtime.KeepGoingOption; import com.google.devtools.build.lib.runtime.LoadingPhaseThreadsOption; -import com.google.devtools.build.lib.runtime.commands.QueryCommand; import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.server.FailureDetails.FetchCommand.Code; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.RepositoryMappingValue.RepositoryMappingResolutionException; -import com.google.devtools.build.lib.skyframe.SkyframeExecutor; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.DetailedExitCode; -import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsParsingResult; -import java.io.IOException; -import java.util.EnumSet; import java.util.List; +import javax.annotation.Nullable; /** Fetches external repositories. Which is so fetch. */ @Command( @@ -83,32 +67,15 @@ allowResidue = true, completion = "label") public final class FetchCommand implements BlazeCommand { - // TODO(kchodorow): this would be a great time to check for difference and invalidate the upward - // transitive closure for local repositories. public static final String NAME = "fetch"; @Override public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult options) { - PackageOptions pkgOptions = options.getOptions(PackageOptions.class); - if (!pkgOptions.fetch) { - return createFailedBlazeCommandResult( - env.getReporter(), Code.OPTIONS_INVALID, "You cannot run fetch with --nofetch"); - } - FetchOptions fetchOptions = options.getOptions(FetchOptions.class); - int optionsCount = - countTrue( - fetchOptions.all, - fetchOptions.configure, - !fetchOptions.repos.isEmpty(), - !options.getResidue().isEmpty()); - if (optionsCount > 1) { - return createFailedBlazeCommandResult( - env.getReporter(), - Code.OPTIONS_INVALID, - "Only one fetch option should be provided for fetch command."); + BlazeCommandResult invalidResult = validateOptions(env, options); + if (invalidResult != null) { + return invalidResult; } - LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); env.getEventBus() .post( @@ -117,8 +84,9 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti env.getCommandStartTime(), /* separateFinishedEvent= */ true, /* showProgress= */ true, - /* id= */ null)); - BlazeCommandResult result; + env.getCommandId().toString())); + + FetchOptions fetchOptions = options.getOptions(FetchOptions.class); if (fetchOptions.force) { // Using commandId as the value -instead of true/false- to make sure to invalidate skyframe // and to actually force fetch each time @@ -128,14 +96,17 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti PrecomputedValue.injected( RepositoryDelegatorFunction.FORCE_FETCH, env.getCommandId().toString()))); } + + BlazeCommandResult result; + LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); try { env.syncPackageLoading(options); - if (fetchOptions.all || fetchOptions.configure) { - result = fetchAll(env, options, threadsOption, fetchOptions.configure); + if (!options.getResidue().isEmpty()) { + result = fetchTarget(env, options, options.getResidue()); } else if (!fetchOptions.repos.isEmpty()) { result = fetchRepos(env, threadsOption, fetchOptions.repos); - } else { - result = fetchTarget(env, options, threadsOption); + } else { // --all, --configure, or just 'fetch' + result = fetchAll(env, threadsOption, fetchOptions.configure); } } catch (AbruptExitException e) { return createFailedBlazeCommandResult( @@ -144,6 +115,7 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti return createFailedBlazeCommandResult( env.getReporter(), "Fetch interrupted: " + e.getMessage()); } + env.getEventBus() .post( new NoBuildRequestFinishedEvent( @@ -151,19 +123,42 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti return result; } + @Nullable + private BlazeCommandResult validateOptions(CommandEnvironment env, OptionsParsingResult options) { + PackageOptions pkgOptions = options.getOptions(PackageOptions.class); + if (!pkgOptions.fetch) { + return createFailedBlazeCommandResult( + env.getReporter(), Code.OPTIONS_INVALID, "You cannot run fetch with --nofetch"); + } + FetchOptions fetchOptions = options.getOptions(FetchOptions.class); + // Only fetch targets works without bzlmod, other than that, fail. + if (options.getResidue().isEmpty() + && !options.getOptions(BuildLanguageOptions.class).enableBzlmod) { + return createFailedBlazeCommandResult( + env.getReporter(), + "Bzlmod has to be enabled for the following options to work: --all, " + + "--configure, --repo or --force. Run with --enable_bzlmod"); + } + int optionsCount = + countTrue( + fetchOptions.all, + fetchOptions.configure, + !fetchOptions.repos.isEmpty(), + !options.getResidue().isEmpty()); + if (optionsCount > 1) { + return createFailedBlazeCommandResult( + env.getReporter(), + Code.OPTIONS_INVALID, + "Only one fetch option can be provided for fetch command"); + } + return null; + } + private BlazeCommandResult fetchAll( CommandEnvironment env, - OptionsParsingResult options, LoadingPhaseThreadsOption threadsOption, boolean configureEnabled) throws InterruptedException { - if (!options.getOptions(BuildLanguageOptions.class).enableBzlmod) { - return createFailedBlazeCommandResult( - env.getReporter(), - "Bzlmod has to be enabled for fetch --all to work, run with --enable_bzlmod"); - } - - SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor(); EvaluationContext evaluationContext = EvaluationContext.newBuilder() .setParallelism(threadsOption.threads) @@ -171,157 +166,62 @@ private BlazeCommandResult fetchAll( .build(); EvaluationResult evaluationResult = - skyframeExecutor.prepareAndGet( - ImmutableSet.of(BazelFetchAllValue.key(configureEnabled)), evaluationContext); + env.getSkyframeExecutor() + .prepareAndGet( + ImmutableSet.of(BazelFetchAllValue.key(configureEnabled)), evaluationContext); if (evaluationResult.hasError()) { Exception e = evaluationResult.getError().getException(); return createFailedBlazeCommandResult( env.getReporter(), e != null ? e.getMessage() : "Unexpected error during fetching all external deps."); } + + env.getReporter().handle(Event.info("All external dependencies fetched successfully.")); return BlazeCommandResult.success(); } private BlazeCommandResult fetchRepos( CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, List repos) throws InterruptedException { + ImmutableMap repositoryNamesAndValues; try { - ImmutableMap repositoryNamesAndValues = - RepositoryFetcher.fetchRepos(repos, env, threadsOption); - String notFoundRepos = - repositoryNamesAndValues.values().stream() - .filter(value -> !value.repositoryExists()) - .map(value -> value.getErrorMsg()) - .collect(joining("; ")); - if (!notFoundRepos.isEmpty()) { - return createFailedBlazeCommandResult( - env.getReporter(), "Fetching repos failed with errors: " + notFoundRepos); - } - return BlazeCommandResult.success(); + repositoryNamesAndValues = RepositoryFetcher.fetchRepos(repos, env, threadsOption); } catch (RepositoryMappingResolutionException e) { return createFailedBlazeCommandResult( env.getReporter(), "Invalid repo name: " + e.getMessage(), e.getDetailedExitCode()); } catch (RepositoryFetcherException e) { return createFailedBlazeCommandResult(env.getReporter(), e.getMessage()); } - } - private BlazeCommandResult fetchTarget( - CommandEnvironment env, OptionsParsingResult options, LoadingPhaseThreadsOption threadsOption) - throws InterruptedException { - if (options.getResidue().isEmpty()) { + String notFoundRepos = + repositoryNamesAndValues.values().stream() + .filter(value -> !value.repositoryExists()) + .map(value -> value.getErrorMsg()) + .collect(joining("; ")); + if (!notFoundRepos.isEmpty()) { return createFailedBlazeCommandResult( - env.getReporter(), - Code.EXPRESSION_MISSING, - String.format( - "missing fetch expression. Type '%s help fetch' for syntax and help", - env.getRuntime().getProductName())); + env.getReporter(), "Fetching some repos failed with errors: " + notFoundRepos); } + env.getReporter().handle(Event.info("All requested repos fetched successfully.")); + return BlazeCommandResult.success(); + } - boolean keepGoing = options.getOptions(KeepGoingOption.class).keepGoing; - TargetPattern.Parser mainRepoTargetParser; + private BlazeCommandResult fetchTarget( + CommandEnvironment env, OptionsParsingResult options, List targets) + throws InterruptedException { try { - RepositoryMapping repoMapping = - env.getSkyframeExecutor() - .getMainRepoMapping(keepGoing, threadsOption.threads, env.getReporter()); - - mainRepoTargetParser = - new Parser(env.getRelativeWorkingDirectory(), RepositoryName.MAIN, repoMapping); + TargetFetcher.fetchTargets(env, options, targets); + } catch (TargetFetcherException e) { + return createFailedBlazeCommandResult( + env.getReporter(), Code.QUERY_EVALUATION_ERROR, e.getMessage()); } catch (RepositoryMappingResolutionException e) { return createFailedBlazeCommandResult( env.getReporter(), e.getMessage(), e.getDetailedExitCode()); } - // Querying for all of the dependencies of the targets has the side-effect of populating the - // Skyframe graph for external targets, which requires downloading them. The JDK is required to - // build everything but isn't counted as a dep in the build graph so we add it manually. - ImmutableList.Builder labelsToLoad = - new ImmutableList.Builder().addAll(options.getResidue()); - - String query = Joiner.on(" union ").join(labelsToLoad.build()); - query = "deps(" + query + ")"; - - AbstractBlazeQueryEnvironment queryEnv = - QueryCommand.newQueryEnvironment( - env, - keepGoing, - false, - UniverseScope.EMPTY, - threadsOption.threads, - EnumSet.noneOf(Setting.class), - /* useGraphlessQuery= */ true, - mainRepoTargetParser, - LabelPrinter.legacy()); - - // 1. Parse query: - QueryExpression expr; - try { - expr = QueryExpression.parse(query, queryEnv); - } catch (QuerySyntaxException e) { - return createFailedBlazeCommandResult( - env.getReporter(), - Code.QUERY_PARSE_ERROR, - String.format( - "Error while parsing '%s': %s", QueryExpression.truncate(query), e.getMessage())); - } - - env.getReporter() - .post( - new NoBuildEvent( - env.getCommandName(), - env.getCommandStartTime(), - true, - true, - env.getCommandId().toString())); - - // 2. Evaluate expression: - QueryEvalResult queryEvalResult = null; - try { - queryEvalResult = - queryEnv.evaluateQuery( - expr, - new ThreadSafeOutputFormatterCallback() { - @Override - public void processOutput(Iterable partialResult) { - // Throw away the result. - } - }); - } catch (InterruptedException e) { - env.getReporter() - .post( - new NoBuildRequestFinishedEvent( - ExitCode.COMMAND_LINE_ERROR, env.getRuntime().getClock().currentTimeMillis())); - return BlazeCommandResult.detailedExitCode( - InterruptedFailureDetails.detailedExitCode(e.getMessage())); - } catch (QueryException e) { - // Keep consistent with reportBuildFileError() - env.getReporter() - .post( - new NoBuildRequestFinishedEvent( - ExitCode.COMMAND_LINE_ERROR, env.getRuntime().getClock().currentTimeMillis())); - return createFailedBlazeCommandResult( - env.getReporter(), Code.QUERY_PARSE_ERROR, e.getMessage()); - } catch (IOException e) { - // Should be impossible since our OutputFormatterCallback doesn't throw IOException. - throw new IllegalStateException(e); - } - - if (queryEvalResult.getSuccess()) { - env.getReporter().handle(Event.info("All external dependencies fetched successfully.")); - } env.getReporter() - .post( - new NoBuildRequestFinishedEvent( - queryEvalResult.getSuccess() ? ExitCode.SUCCESS : ExitCode.COMMAND_LINE_ERROR, - env.getRuntime().getClock().currentTimeMillis())); - return queryEvalResult.getSuccess() - ? BlazeCommandResult.success() - : createFailedBlazeCommandResult( - env.getReporter(), - Code.QUERY_EVALUATION_ERROR, - String.format( - "Evaluation of query \"%s\" failed but --keep_going specified, ignoring errors", - expr)); + .handle(Event.info("All external dependencies for these targets fetched successfully.")); + return BlazeCommandResult.success(); } private static BlazeCommandResult createFailedBlazeCommandResult( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchOptions.java index e5801a1581afa2..529bde9f84b786 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchOptions.java @@ -19,8 +19,9 @@ import com.google.devtools.common.options.OptionsBase; import java.util.List; -/** Defines the options specific to Bazel's sync command */ +/** Defines the options specific to Bazel's fetch command */ public class FetchOptions extends OptionsBase { + @Option( name = "all", defaultValue = "false", @@ -34,7 +35,7 @@ public class FetchOptions extends OptionsBase { @Option( name = "configure", defaultValue = "false", - documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY, + documentationCategory = OptionDocumentationCategory.BZLMOD, effectTags = {OptionEffectTag.CHANGES_INPUTS}, help = "Only fetches repositories marked as 'configure' for system-configuration purpose. Only" @@ -45,7 +46,7 @@ public class FetchOptions extends OptionsBase { name = "repo", defaultValue = "null", allowMultiple = true, - documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY, + documentationCategory = OptionDocumentationCategory.BZLMOD, effectTags = {OptionEffectTag.CHANGES_INPUTS}, help = "Only fetches the specified repository, which can be either {@apparent_repo_name} or" @@ -55,7 +56,7 @@ public class FetchOptions extends OptionsBase { @Option( name = "force", defaultValue = "false", - documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY, + documentationCategory = OptionDocumentationCategory.BZLMOD, effectTags = {OptionEffectTag.CHANGES_INPUTS}, help = "Ignore existing repository if any and force fetch the repository again. Only works when " diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java index 611764b553e274..ec08bfafd50d55 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java @@ -56,6 +56,7 @@ import com.google.devtools.build.lib.cmdline.RepositoryMapping; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.pkgcache.PackageOptions; import com.google.devtools.build.lib.runtime.BlazeCommand; import com.google.devtools.build.lib.runtime.BlazeCommandResult; @@ -82,8 +83,6 @@ import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; -import com.google.devtools.common.options.OptionPriority.PriorityCategory; -import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.OptionsParsingException; import com.google.devtools.common.options.OptionsParsingResult; import com.google.gson.Gson; @@ -120,20 +119,14 @@ public final class ModCommand implements BlazeCommand { public static final String NAME = "mod"; @Override - public void editOptions(OptionsParser optionsParser) { - try { - optionsParser.parse( - PriorityCategory.SOFTWARE_REQUIREMENT, - "Option required by the mod command", - ImmutableList.of("--enable_bzlmod")); - } catch (OptionsParsingException e) { - // Should never happen. - throw new IllegalStateException("Unexpected exception", e); + public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult options) { + if (!options.getOptions(BuildLanguageOptions.class).enableBzlmod) { + return reportAndCreateFailureResult( + env, + "Bzlmod has to be enabled for mod command to work, run with --enable_bzlmod", + Code.MISSING_ARGUMENTS); } - } - @Override - public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult options) { env.getEventBus() .post( new NoBuildEvent( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/TargetFetcher.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/TargetFetcher.java new file mode 100644 index 00000000000000..94de6ab029f779 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/TargetFetcher.java @@ -0,0 +1,129 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.commands; + +import com.google.common.base.Joiner; +import com.google.devtools.build.lib.cmdline.RepositoryMapping; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.cmdline.TargetPattern; +import com.google.devtools.build.lib.cmdline.TargetPattern.Parser; +import com.google.devtools.build.lib.packages.LabelPrinter; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.query2.common.AbstractBlazeQueryEnvironment; +import com.google.devtools.build.lib.query2.common.UniverseScope; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting; +import com.google.devtools.build.lib.query2.engine.QueryEvalResult; +import com.google.devtools.build.lib.query2.engine.QueryException; +import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.engine.QuerySyntaxException; +import com.google.devtools.build.lib.query2.engine.ThreadSafeOutputFormatterCallback; +import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.KeepGoingOption; +import com.google.devtools.build.lib.runtime.LoadingPhaseThreadsOption; +import com.google.devtools.build.lib.runtime.commands.QueryCommand; +import com.google.devtools.build.lib.skyframe.RepositoryMappingValue.RepositoryMappingResolutionException; +import com.google.devtools.common.options.OptionsParsingResult; +import java.io.IOException; +import java.util.EnumSet; +import java.util.List; + +/** Fetches all repos needed for building a given set of targets. */ +public class TargetFetcher { + private final CommandEnvironment env; + + private TargetFetcher(CommandEnvironment env) { + this.env = env; + } + + /** Uses `deps` query to find and fetch all repos needed for these targets */ + public static void fetchTargets( + CommandEnvironment env, OptionsParsingResult options, List targets) + throws RepositoryMappingResolutionException, InterruptedException, TargetFetcherException { + new TargetFetcher(env).fetchTargets(options, targets); + } + + private void fetchTargets(OptionsParsingResult options, List targets) + throws InterruptedException, TargetFetcherException, RepositoryMappingResolutionException { + AbstractBlazeQueryEnvironment queryEnv = getQueryEnv(options); + QueryExpression expr = createQueryExpression(targets, queryEnv); + QueryEvalResult queryEvalResult; + try { + queryEvalResult = + queryEnv.evaluateQuery( + expr, + new ThreadSafeOutputFormatterCallback<>() { + @Override + public void processOutput(Iterable partialResult) {} + }); + } catch (IOException e) { + // Should be impossible since our OutputFormatterCallback doesn't throw IOException. + throw new IllegalStateException(e); + } catch (QueryException e) { + throw new TargetFetcherException( + String.format( + "Fetching target dependencies for %s encountered an error: %s", + expr, e.getMessage())); + } + + if (!queryEvalResult.getSuccess()) { + throw new TargetFetcherException( + String.format( + "Fetching some target dependencies for %s failed, but --keep_going specified. " + + " Ignoring errors", + expr)); + } + } + + AbstractBlazeQueryEnvironment getQueryEnv(OptionsParsingResult options) + throws RepositoryMappingResolutionException, InterruptedException { + boolean keepGoing = options.getOptions(KeepGoingOption.class).keepGoing; + LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); + RepositoryMapping repoMapping = + env.getSkyframeExecutor() + .getMainRepoMapping(keepGoing, threadsOption.threads, env.getReporter()); + TargetPattern.Parser targetParser = + new Parser(env.getRelativeWorkingDirectory(), RepositoryName.MAIN, repoMapping); + return QueryCommand.newQueryEnvironment( + env, + keepGoing, + false, + UniverseScope.EMPTY, + threadsOption.threads, + EnumSet.noneOf(Setting.class), + /* useGraphlessQuery= */ true, + targetParser, + LabelPrinter.legacy()); + } + + private QueryExpression createQueryExpression( + List targets, AbstractBlazeQueryEnvironment queryEnv) + throws TargetFetcherException { + String query = "deps(" + Joiner.on(" union ").join(targets) + ")"; + try { + return QueryExpression.parse(query, queryEnv); + } catch (QuerySyntaxException e) { + throw new TargetFetcherException( + String.format( + "Fetching target dependencies for %s encountered an error: %s", + QueryExpression.truncate(query), e.getMessage())); + } + } + + static class TargetFetcherException extends Exception { + public TargetFetcherException(String message) { + super(message); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt b/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt index efd63df44df19b..f81995a286db83 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt @@ -2,6 +2,7 @@ Usage: %{product} %{command} [