diff --git a/site/en/_book.yaml b/site/en/_book.yaml index 3a3b00cd26d953..157298427b7ae5 100644 --- a/site/en/_book.yaml +++ b/site/en/_book.yaml @@ -132,6 +132,8 @@ upper_tabs: path: /external/extension - title: Module lockfile path: /external/lockfile + - title: Vendor mode + path: /external/vendor - title: '`mod` command' path: /external/mod-command - title: Bzlmod migration guide diff --git a/site/en/external/vendor.md b/site/en/external/vendor.md new file mode 100644 index 00000000000000..4b1a93542c0c5c --- /dev/null +++ b/site/en/external/vendor.md @@ -0,0 +1,219 @@ +Project: /_project.yaml +Book: /_book.yaml +keywords: product:Bazel,Bzlmod,vendor + +{# disableFinding("vendoring") #} +{# disableFinding("Vendoring") #} +{# disableFinding("vendored") #} +{# disableFinding("repo") #} + +# Vendor Mode + +{% include "_buttons.html" %} + +Vendor mode is a feature of Bzlmod that lets you create a local copy of +external dependencies. This is useful for offline builds, or when you want to +control the source of an external dependency. + +## Enable vendor mode {:#enable-vendor-mode} + +You can enable vendor mode by specifying `--vendor_dir` flag. + +For example, by adding it to your `.bazelrc` file: + +```none +# Enable vendor mode with vendor directory under /vendor_src +common --vendor_dir=vendor_src +``` + +The vendor directory can be either a relative path to your workspace root or an +absolute path. + +## Vendor a specific external repository {:#vendor-specific-repository} + +You can use the `vendor` command with the `--repo` flag to specify which repo +to vendor, it accepts both [canonical repo +name](/external/overview#canonical-repo-name) and [apparent repo +name](/external/overview#apparent-repo-name). + +For example, running: + +```none +bazel vendor --vendor_dir=vendor_src --repo=@rules_cc +``` + +or + +```none +bazel vendor --vendor_dir=vendor_src --repo=@@rules_cc~ +``` + +will both get rules_cc to be vendored under +`/vendor_src/rules_cc~`. + +## Vendor external dependencies for given targets {:#vendor-target-dependencies} + +To vendor all external dependencies required for building given target patterns, +you can run `bazel vendor `. + +For example + +```none +bazel vendor --vendor_dir=vendor_src //src/main:hello-world //src/test/... +``` + +will vendor all repos required for building the `//src/main:hello-world` target +and all targets under `//src/test/...` with the current configuration. + +Under the hood, it's doing a `bazel build --nobuild` command to analyze the +target patterns, therefore build flags could be applied to this command and +affect the result. + +### Build the target offline {:#build-the-target-offline} + +With the external dependencies vendored, you can build the target offline by + +```none +bazel build --vendor_dir=vendor_src //src/main:hello-world //src/test/... +``` + +The build should work in a clean build environment without network access and +repository cache. + +Therefore, you should be able to check in the vendored source and build the same +targets offline on another machine. + +Note: If you make changes to the targets to build, the external dependencies, +the build configuration, or the Bazel version, you may need to re-vendor to make +sure offline build still works. + +## Vendor all external dependencies {:#vendor-all-dependencies} + +To vendor all repos in your transitive external dependencies graph, you can +run: + +```none +bazel vendor --vendor_dir=vendor_src +``` + +Note that vendoring all dependencies has a few **disadvantages**: + +- Fetching all repos, including those introduced transitively, can be time-consuming. +- The vendor directory can become very large. +- Some repos may fail to fetch if they are not compatible with the current platform or environment. + +Therefore, consider vendoring for specific targets first. + +## Configure vendor mode with VENDOR.bazel {:#configure-vendor-mode} + +You can control how given repos are handled with the VENDOR.bazel file located +under the vendor directory. + +There are two directives available, both accepting a list of +[canonical repo names](/external/overview#canonical-repo-name) as arguments: + +- `ignore()`: to completely ignore a repository from vendor mode. +- `pin()`: to pin a repository to its current vendored source as if there is a + `--override_repository` flag for this repo. Bazel will NOT update the vendored + source for this repo while running the vendor command unless it's unpinned. + The user can modify and maintain the vendored source for this repo manually. + +For example + +```python +ignore("@@rules_cc~") +pin("@@bazel_skylib~") +``` + +With this configuration + +- Both repos will be excluded from subsequent vendor commands. +- Repo `bazel_skylib` will be overridden to the source located under the + vendor directory. +- The user can safely modify the vendored source of `bazel_skylib`. +- To re-vendor `bazel_skylib`, the user has to disable the pin statement + first. + +Note: Repository rules with +[`local`](/rules/lib/globals/bzl#repository_rule.local) or +[`configure`](/rules/lib/globals/bzl#repository_rule.configure) set to true are +always excluded from vendoring. + +## Understand how vendor mode works {:#how-vendor-mode-works} + +Bazel fetches external dependencies of a project under `$(bazel info +output_base)/external`. Vendoring external dependencies means moving out +relevant files and directories to the given vendor directory and use the +vendored source for later builds. + +The content being vendored includes: + +- The repo directory +- The repo marker file + +During a build, if the vendored marker file is up-to-date or the repo is +pinned in the VENDOR.bazel file, then Bazel uses the vendored source by creating +a symlink to it under `$(bazel info output_base)/external` instead of actually +running the repository rule. Otherwise, a warning is printed and Bazel will +fallback to fetching the latest version of the repo. + +Note: Bazel assumes the vendored source is not changed by users unless the repo +is pinned in the VENDOR.bazel file. If a user does change the vendored source +without pinning the repo, the changed vendored source will be used, but it will +be overwritten if its existing marker file is +outdated and the repo is vendored again. + +### Vendor registry files {:#vendor-registry-files} + +Bazel has to perform the Bazel module resolution in order to fetch external +dependencies, which may require accessing registry files through internet. To +achieve offline build, Bazel vendors all registry files fetched from +network under the `/_registries` directory. + +### Vendor symlinks {:#vendor-symlinks} + +External repositories may contain symlinks pointing to other files or +directories. To make sure symlinks work correctly, Bazel uses the following +strategy to rewrite symlinks in the vendored source: + +- Create a symlink `/bazel-external` that points to `$(bazel info + output_base)/external`. It is refreshed by every Bazel command + automatically. +- For the vendored source, rewrite all symlinks that originally point to a + path under `$(bazel info output_base)/external` to a relative path under + `/bazel-external`. + +For example, if the original symlink is + +```none +/repo_foo~/link => $(bazel info output_base)/external/repo_bar~/file +``` + +It will be rewritten to + +```none +/repo_foo~/link => ../../bazel-external/repo_bar~/file +``` + +where + +```none +/bazel-external => $(bazel info output_base)/external # This might be new if output base is changed +``` + +Since `/bazel-external` is generated by Bazel automatically, it's +recommended to add it to `.gitignore` or equivalent to avoid checking it in. + +With this strategy, symlinks in the vendored source should work correctly even +after the vendored source is moved to another location or the bazel output base +is changed. + +Note: symlinks that point to an absolute path outside of $(bazel info +output_base)/external are not rewritten. Therefore, it could still break +cross-machine compatibility. + +Note: On Windows, vendoring symlinks only works with +[`--windows_enable_symlinks`][windows_enable_symlinks] +flag enabled. + +[windows_enable_symlinks]: /reference/command-line-reference#flag--windows_enable_symlinks diff --git a/src/main/java/com/google/devtools/build/docgen/annot/GlobalMethods.java b/src/main/java/com/google/devtools/build/docgen/annot/GlobalMethods.java index fdfadba95c7eab..0fdeb668031d15 100644 --- a/src/main/java/com/google/devtools/build/docgen/annot/GlobalMethods.java +++ b/src/main/java/com/google/devtools/build/docgen/annot/GlobalMethods.java @@ -32,7 +32,7 @@ enum Environment { ALL( "All Bazel files", "Methods available in all Bazel files, including .bzl files, BUILD, MODULE.bazel," - + " and WORKSPACE."), + + " VENDOR.bazel, and WORKSPACE."), BZL(".bzl files", "Global methods available in all .bzl files."), BUILD( "BUILD files", @@ -40,6 +40,7 @@ enum Environment { + " Encyclopedia for extra functions and build rules," + " which can also be used in BUILD files."), MODULE("MODULE.bazel files", "Methods available in MODULE.bazel files."), + VENDOR("VENDOR.bazel files", "Methods available in VENDOR.bazel files."), WORKSPACE("WORKSPACE files", "Methods available in WORKSPACE files."); private final String title; diff --git a/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkGlobalsImpl.java b/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkGlobalsImpl.java index 1450ab51f1fc44..bae72e298463c5 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkGlobalsImpl.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkGlobalsImpl.java @@ -28,6 +28,7 @@ import com.google.devtools.build.lib.packages.StarlarkGlobals; import com.google.devtools.build.lib.packages.StarlarkNativeModule; import com.google.devtools.build.lib.packages.StructProvider; +import com.google.devtools.build.lib.packages.VendorFileGlobals; import net.starlark.java.eval.Starlark; import net.starlark.java.lib.json.Json; @@ -135,4 +136,11 @@ public ImmutableMap getRepoToplevels() { Starlark.addMethods(env, RepoCallable.INSTANCE); return env.buildOrThrow(); } + + @Override + public ImmutableMap getVendorToplevels() { + ImmutableMap.Builder env = ImmutableMap.builder(); + Starlark.addMethods(env, VendorFileGlobals.INSTANCE); + return env.buildOrThrow(); + } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD index 6d8abb60844550..815dee82de214c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD @@ -34,6 +34,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:tidy_impl", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:vendor", "//src/main/java/com/google/devtools/build/lib/bazel/commands", "//src/main/java/com/google/devtools/build/lib/bazel/repository", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", @@ -57,6 +58,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository", "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception", "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code", + "//src/main/java/com/google/devtools/build/lib/util:os", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/common/options", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 866c3e240eb4e4..b04d25024fd25c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -54,6 +54,8 @@ import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionEvalFunction; import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionFunction; import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionUsagesFunction; +import com.google.devtools.build.lib.bazel.bzlmod.VendorFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.VendorManager; import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsFunction; import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.commands.FetchCommand; @@ -81,6 +83,7 @@ import com.google.devtools.build.lib.bazel.rules.android.AndroidSdkRepositoryFunction; import com.google.devtools.build.lib.bazel.rules.android.AndroidSdkRepositoryRule; import com.google.devtools.build.lib.clock.Clock; +import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.pkgcache.PackageOptions; @@ -113,7 +116,9 @@ import com.google.devtools.build.lib.starlarkbuildapi.repository.RepositoryBootstrap; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.DetailedExitCode; +import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.Root; @@ -164,7 +169,7 @@ public class BazelRepositoryModule extends BlazeModule { private Clock clock; private Instant lastRegistryInvalidation = Instant.EPOCH; - private Optional vendorDirectory; + private Optional vendorDirectory = Optional.empty(); private List allowedYankedVersions = ImmutableList.of(); private boolean disableNativeRepoRules; private SingleExtensionEvalFunction singleExtensionEvalFunction; @@ -223,7 +228,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde builder.addCommands(new FetchCommand()); builder.addCommands(new ModCommand()); builder.addCommands(new SyncCommand()); - builder.addCommands(new VendorCommand()); + builder.addCommands(new VendorCommand(downloadManager, clientEnvironmentSupplier)); builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache)); } @@ -278,6 +283,9 @@ SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace directories.getWorkspace())) .addSkyFunction(SkyFunctions.REPO_SPEC, new RepoSpecFunction()) .addSkyFunction(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction()) + .addSkyFunction( + SkyFunctions.VENDOR_FILE, + new VendorFileFunction(runtime.getRuleClassProvider().getBazelStarlarkEnvironment())) .addSkyFunction( SkyFunctions.MODULE_EXTENSION_REPO_MAPPING_ENTRIES, new ModuleExtensionRepoMappingEntriesFunction()); @@ -499,15 +507,37 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { bazelCompatibilityMode = repoOptions.bazelCompatibilityMode; bazelLockfileMode = repoOptions.lockfileMode; allowedYankedVersions = repoOptions.allowedYankedVersions; - - if (repoOptions.vendorDirectory != null) { + if (env.getWorkspace() != null) { vendorDirectory = - Optional.of( - repoOptions.vendorDirectory.isAbsolute() - ? filesystem.getPath(repoOptions.vendorDirectory) - : env.getWorkspace().getRelative(repoOptions.vendorDirectory)); - } else { - vendorDirectory = Optional.empty(); + Optional.ofNullable(repoOptions.vendorDirectory) + .map(vendorDirectory -> env.getWorkspace().getRelative(vendorDirectory)); + + if (vendorDirectory.isPresent()) { + try { + Path externalRoot = + env.getOutputBase().getRelative(LabelConstants.EXTERNAL_PATH_PREFIX); + FileSystemUtils.ensureSymbolicLink( + vendorDirectory.get().getChild(VendorManager.EXTERNAL_ROOT_SYMLINK_NAME), + externalRoot); + if (OS.getCurrent() == OS.WINDOWS) { + // On Windows, symlinks are resolved differently. + // Given /repo_foo/link, + // where /repo_foo points to /repo_foo in vendor mode + // and repo_foo/link points to a relative path ../bazel-external/repo_bar/data. + // Windows won't resolve `repo_foo` before resolving `link`, which causes + // /repo_foo/link to be resolved to /bazel-external/repo_bar/data + // To work around this, we create a symlink /bazel-external -> . + FileSystemUtils.ensureSymbolicLink( + externalRoot.getChild(VendorManager.EXTERNAL_ROOT_SYMLINK_NAME), externalRoot); + } + } catch (IOException e) { + env.getReporter() + .handle( + Event.error( + "Failed to create symlink to external repo root under vendor directory: " + + e.getMessage())); + } + } } if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 6c1d900a94fc36..1c4ed82a6c99e0 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -46,6 +46,20 @@ java_library( ], ) +java_library( + name = "vendor", + srcs = ["VendorManager.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/profiler", + "//src/main/java/com/google/devtools/build/lib/util:os", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//third_party:guava", + ], +) + java_library( name = "module_extension", srcs = [ @@ -80,6 +94,7 @@ java_library( ], deps = [ ":common", + ":vendor", ":yanked_versions_value", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", @@ -87,10 +102,12 @@ java_library( "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/util:os", + "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//third_party:gson", "//third_party:guava", + "//third_party:jsr305", ], ) @@ -146,6 +163,7 @@ java_library( "SingleExtensionUsagesValue.java", "SingleExtensionValue.java", "SingleVersionOverride.java", + "VendorFileValue.java", "YankedVersionsValue.java", ], deps = [ @@ -204,6 +222,7 @@ java_library( "SingleExtensionUsagesFunction.java", "StarlarkBazelModule.java", "TypeCheckedTag.java", + "VendorFileFunction.java", "YankedVersionsFunction.java", "YankedVersionsUtil.java", ], diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java index 3dc6849db48f4b..e776d19d317811 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java @@ -48,7 +48,7 @@ public abstract class BazelModuleResolutionValue implements SkyValue { /** * Hashes of files obtained (or known to be missing) from registries while performing resolution. */ - abstract ImmutableMap> getRegistryFileHashes(); + public abstract ImmutableMap> getRegistryFileHashes(); /** * Selected module versions that are known to be yanked (and hence must have been explicitly diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index d0cef909bc9834..79adfebcfda048 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -32,6 +32,7 @@ import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.gson.FieldNamingPolicy; import com.google.gson.Gson; @@ -45,6 +46,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; +import javax.annotation.Nullable; /** * Represents a Bazel module registry that serves a list of module metadata from a static HTTP @@ -88,6 +90,7 @@ public enum KnownFileHashesMode { private final Gson gson; private final ImmutableMap> knownFileHashes; private final ImmutableMap previouslySelectedYankedVersions; + @Nullable private final VendorManager vendorManager; private final KnownFileHashesMode knownFileHashesMode; private volatile Optional bazelRegistryJson; private volatile StoredEventHandler bazelRegistryJsonEvents; @@ -100,7 +103,8 @@ public IndexRegistry( Map clientEnv, ImmutableMap> knownFileHashes, KnownFileHashesMode knownFileHashesMode, - ImmutableMap previouslySelectedYankedVersions) { + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) { this.uri = uri; this.downloadManager = downloadManager; this.clientEnv = clientEnv; @@ -111,6 +115,7 @@ public IndexRegistry( this.knownFileHashes = knownFileHashes; this.knownFileHashesMode = knownFileHashesMode; this.previouslySelectedYankedVersions = previouslySelectedYankedVersions; + this.vendorManager = vendorDir.map(VendorManager::new).orElse(null); } @Override @@ -143,11 +148,11 @@ private Optional grabFile( } private Optional doGrabFile( - String url, ExtendedEventHandler eventHandler, boolean useChecksum) + String rawUrl, ExtendedEventHandler eventHandler, boolean useChecksum) throws IOException, InterruptedException { Optional checksum; if (knownFileHashesMode != KnownFileHashesMode.IGNORE && useChecksum) { - Optional knownChecksum = knownFileHashes.get(url); + Optional knownChecksum = knownFileHashes.get(rawUrl); if (knownChecksum == null) { if (knownFileHashesMode == KnownFileHashesMode.ENFORCE) { throw new MissingChecksumException( @@ -155,7 +160,7 @@ private Optional doGrabFile( "Missing checksum for registry file %s not permitted with --lockfile_mode=error." + " Please run `bazel mod deps --lockfile_mode=update` to update your" + " lockfile.", - url)); + rawUrl)); } // This is a new file, download without providing a checksum. checksum = Optional.empty(); @@ -182,17 +187,40 @@ private Optional doGrabFile( "Cannot fetch a file without a checksum in ENFORCE mode. This is a bug in Bazel, please " + "report at https://github.com/bazelbuild/bazel/issues/new/choose."); } + + URL url = URI.create(rawUrl).toURL(); + // Don't read the registry URL from the vendor directory in the following cases: + // 1. vendorUtil is null, which means vendor mode is disabled. + // 2. The checksum is not present, which means the URL is not vendored or the vendored content + // is out-dated. + // 3. The URL starts with "file:", which means it's a local file and isn't vendored. + // 4. The vendor path doesn't exist, which means the URL is not vendored. + if (vendorManager != null + && checksum.isPresent() + && !url.getProtocol().equals("file") + && vendorManager.isUrlVendored(url)) { + try { + return Optional.of(vendorManager.readRegistryUrl(url, checksum.get())); + } catch (IOException e) { + throw new IOException( + String.format( + "Failed to read vendored registry file %s at %s: %s. Please rerun the bazel" + + " vendor command.", + rawUrl, vendorManager.getVendorPathForUrl(url), e.getMessage()), + e); + } + } + try (SilentCloseable c = - Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + url)) { + Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + rawUrl)) { return Optional.of( - downloadManager.downloadAndReadOneUrlForBzlmod( - new URL(url), eventHandler, clientEnv, checksum)); + downloadManager.downloadAndReadOneUrlForBzlmod(url, eventHandler, clientEnv, checksum)); } catch (FileNotFoundException e) { return Optional.empty(); } catch (IOException e) { // Include the URL in the exception message for easier debugging. throw new IOException( - "Failed to fetch registry file %s: %s".formatted(url, e.getMessage()), e); + "Failed to fetch registry file %s: %s".formatted(rawUrl, e.getMessage()), e); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java index d910c3d2d75c33..47d0bb77e9579b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.vfs.Path; import java.net.URISyntaxException; import java.util.Optional; @@ -33,6 +34,7 @@ Registry createRegistry( String url, RepositoryOptions.LockfileMode lockfileMode, ImmutableMap> fileHashes, - ImmutableMap previouslySelectedYankedVersions) + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) throws URISyntaxException; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java index 32c34daa0c4d4a..e26468f4890528 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java @@ -20,6 +20,7 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; +import com.google.devtools.build.lib.vfs.Path; import java.net.URI; import java.net.URISyntaxException; import java.util.Map; @@ -42,7 +43,8 @@ public Registry createRegistry( String url, LockfileMode lockfileMode, ImmutableMap> knownFileHashes, - ImmutableMap previouslySelectedYankedVersions) + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) throws URISyntaxException { URI uri = new URI(url); if (uri.getScheme() == null) { @@ -75,6 +77,7 @@ public Registry createRegistry( clientEnvironmentSupplier.get(), knownFileHashes, knownFileHashesMode, - previouslySelectedYankedVersions); + previouslySelectedYankedVersions, + vendorDir); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java index 4e72d565bca8fe..836c19e90bdb84 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; import com.google.devtools.build.lib.vfs.Path; @@ -26,6 +27,7 @@ import java.net.URISyntaxException; import java.time.Duration; import java.time.Instant; +import java.util.Optional; import javax.annotation.Nullable; /** A simple SkyFunction that creates a {@link Registry} with a given URL. */ @@ -56,6 +58,7 @@ public RegistryFunction(RegistryFactory registryFactory, Path workspaceRoot) { public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException, RegistryException { LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env); + Optional vendorDir = RepositoryDelegatorFunction.VENDOR_DIRECTORY.get(env); if (lockfileMode == LockfileMode.REFRESH) { RegistryFunction.LAST_INVALIDATION.get(env); @@ -72,7 +75,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) key.getUrl().replace("%workspace%", workspaceRoot.getPathString()), lockfileMode, lockfile.getRegistryFileHashes(), - lockfile.getSelectedYankedVersions()); + lockfile.getSelectedYankedVersions(), + vendorDir); } catch (URISyntaxException e) { throw new RegistryException( ExternalDepsException.withCauseAndMessage( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorFileFunction.java new file mode 100644 index 00000000000000..e10d7ab934d58c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorFileFunction.java @@ -0,0 +1,178 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.FileValue; +import com.google.devtools.build.lib.cmdline.LabelConstants; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.BazelStarlarkEnvironment; +import com.google.devtools.build.lib.packages.DotBazelFileSyntaxChecker; +import com.google.devtools.build.lib.packages.VendorThreadContext; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import java.io.IOException; +import javax.annotation.Nullable; +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.Mutability; +import net.starlark.java.eval.Starlark; +import net.starlark.java.eval.StarlarkSemantics; +import net.starlark.java.eval.StarlarkThread; +import net.starlark.java.syntax.ParserInput; +import net.starlark.java.syntax.Program; +import net.starlark.java.syntax.StarlarkFile; +import net.starlark.java.syntax.SyntaxError; + +/** + * The function to evaluate the VENDOR.bazel file under the vendor directory specified by the flag: + * --vendor_dir. + */ +public class VendorFileFunction implements SkyFunction { + + private static final String VENDOR_FILE_HEADER = + """ +############################################################################### +# This file is used to configure how external repositories are handled in vendor mode. +# ONLY the two following functions can be used: +# +# ignore('@@', ...) is used to completely ignore this repo from vendoring. +# Bazel will use the normal external cache and fetch process for this repo. +# +# pin('@@', ...) is used to pin the contents of this repo under the vendor +# directory as if there is a --override_repository flag for this repo. +# Note that Bazel will NOT update the vendored source for this repo while running vendor command +# unless it's unpinned. The user can modify and maintain the vendored source for this repo manually. +############################################################################### +"""; + + private final BazelStarlarkEnvironment starlarkEnv; + + public VendorFileFunction(BazelStarlarkEnvironment starlarkEnv) { + this.starlarkEnv = starlarkEnv; + } + + @Nullable + @Override + public SkyValue compute(SkyKey skyKey, Environment env) + throws SkyFunctionException, InterruptedException { + if (RepositoryDelegatorFunction.VENDOR_DIRECTORY.get(env).isEmpty()) { + throw new VendorFileFunctionException( + new IllegalStateException( + "VENDOR.bazel file is not accessible with vendor mode off (without --vendor_dir" + + " flag)"), + Transience.PERSISTENT); + } + + Path vendorPath = RepositoryDelegatorFunction.VENDOR_DIRECTORY.get(env).get(); + RootedPath vendorFilePath = + RootedPath.toRootedPath(Root.fromPath(vendorPath), LabelConstants.VENDOR_FILE_NAME); + + FileValue vendorFileValue = (FileValue) env.getValue(FileValue.key(vendorFilePath)); + if (vendorFileValue == null) { + return null; + } + if (!vendorFileValue.exists()) { + createVendorFile(vendorPath, vendorFilePath.asPath()); + return VendorFileValue.create(ImmutableList.of(), ImmutableList.of()); + } + + StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env); + if (starlarkSemantics == null) { + return null; + } + VendorThreadContext context = + getVendorFileContext(env, skyKey, vendorFilePath.asPath(), starlarkSemantics); + return VendorFileValue.create(context.getIgnoredRepos(), context.getPinnedRepos()); + } + + private VendorThreadContext getVendorFileContext( + Environment env, SkyKey skyKey, Path vendorFilePath, StarlarkSemantics starlarkSemantics) + throws VendorFileFunctionException, InterruptedException { + try (Mutability mu = Mutability.create("vendor file")) { + StarlarkFile vendorFile = readAndParseVendorFile(vendorFilePath, env); + new DotBazelFileSyntaxChecker("VENDOR.bazel files", /* canLoadBzl= */ false) + .check(vendorFile); + net.starlark.java.eval.Module predeclaredEnv = + net.starlark.java.eval.Module.withPredeclared( + starlarkSemantics, starlarkEnv.getStarlarkGlobals().getVendorToplevels()); + Program program = Program.compileFile(vendorFile, predeclaredEnv); + StarlarkThread thread = + new StarlarkThread(mu, starlarkSemantics, /* contextDescription= */ ""); + VendorThreadContext context = new VendorThreadContext(); + context.storeInThread(thread); + Starlark.execFileProgram(program, predeclaredEnv, thread); + return context; + } catch (SyntaxError.Exception | EvalException e) { + throw new VendorFileFunctionException( + new BadVendorFileException("error parsing VENDOR.bazel file: " + e.getMessage()), + Transience.PERSISTENT); + } + } + + private void createVendorFile(Path vendorPath, Path vendorFilePath) + throws VendorFileFunctionException { + try { + vendorPath.createDirectoryAndParents(); + byte[] vendorFileContents = VENDOR_FILE_HEADER.getBytes(UTF_8); + FileSystemUtils.writeContent(vendorFilePath, vendorFileContents); + } catch (IOException e) { + throw new VendorFileFunctionException( + new IOException("error creating VENDOR.bazel file", e), Transience.TRANSIENT); + } + } + + private static StarlarkFile readAndParseVendorFile(Path path, Environment env) + throws VendorFileFunctionException { + byte[] contents; + try { + contents = FileSystemUtils.readWithKnownFileSize(path, path.getFileSize()); + } catch (IOException e) { + throw new VendorFileFunctionException( + new IOException("error reading VENDOR.bazel file", e), Transience.TRANSIENT); + } + StarlarkFile starlarkFile = + StarlarkFile.parse(ParserInput.fromUTF8(contents, path.getPathString())); + if (!starlarkFile.ok()) { + Event.replayEventsOn(env.getListener(), starlarkFile.errors()); + throw new VendorFileFunctionException( + new BadVendorFileException("error parsing VENDOR.bazel file"), Transience.PERSISTENT); + } + return starlarkFile; + } + + /** Thrown when something is wrong with the contents of the VENDOR.bazel file. */ + public static class BadVendorFileException extends Exception { + public BadVendorFileException(String message) { + super(message); + } + } + + static class VendorFileFunctionException extends SkyFunctionException { + private VendorFileFunctionException(Exception e, Transience transience) { + super(e, transience); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorFileValue.java new file mode 100644 index 00000000000000..ed7df24d1b274c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorFileValue.java @@ -0,0 +1,39 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** Represent the parsed VENDOR.bazel file */ +@AutoValue +public abstract class VendorFileValue implements SkyValue { + + @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.VENDOR_FILE; + + public abstract ImmutableList getIgnoredRepos(); + + public abstract ImmutableList getPinnedRepos(); + + public static VendorFileValue create( + ImmutableList ignoredRepos, ImmutableList pinnedRepos) { + return new AutoValue_VendorFileValue(ignoredRepos, pinnedRepos); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorManager.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorManager.java new file mode 100644 index 00000000000000..88bdc61b33fdc8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorManager.java @@ -0,0 +1,244 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.bzlmod; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hasher; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.profiler.SilentCloseable; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.file.Files; +import java.util.Collection; +import java.util.Locale; +import java.util.Objects; + +/** Class to manage the vendor directory. */ +public class VendorManager { + + private static final String REGISTRIES_DIR = "_registries"; + + public static final String EXTERNAL_ROOT_SYMLINK_NAME = "bazel-external"; + + private final Path vendorDirectory; + + public VendorManager(Path vendorDirectory) { + this.vendorDirectory = vendorDirectory; + } + + /** + * Vendors the specified repositories under the vendor directory. + * + *

TODO(pcloudy): Parallelize vendoring repos + * + * @param externalRepoRoot The root directory of the external repositories. + * @param reposToVendor The list of repositories to vendor. + * @throws IOException if an I/O error occurs. + */ + public void vendorRepos(Path externalRepoRoot, ImmutableList reposToVendor) + throws IOException { + if (!vendorDirectory.exists()) { + vendorDirectory.createDirectoryAndParents(); + } + + for (RepositoryName repo : reposToVendor) { + try (SilentCloseable c = + Profiler.instance().profile(ProfilerTask.REPOSITORY_VENDOR, repo.toString())) { + Path repoUnderExternal = externalRepoRoot.getChild(repo.getName()); + Path repoUnderVendor = vendorDirectory.getChild(repo.getName()); + // This could happen when running the vendor command twice without changing anything. + if (repoUnderExternal.isSymbolicLink() + && repoUnderExternal.resolveSymbolicLinks().equals(repoUnderVendor)) { + continue; + } + + // At this point, the repo should exist under external dir, but check if the vendor src is + // already up-to-date. + Path markerUnderExternal = externalRepoRoot.getChild(repo.getMarkerFileName()); + Path markerUnderVendor = vendorDirectory.getChild(repo.getMarkerFileName()); + if (isRepoUpToDate(markerUnderVendor, markerUnderExternal)) { + continue; + } + + // Actually vendor the repo: + // 1. Clean up existing marker file and vendor dir. + markerUnderVendor.delete(); + repoUnderVendor.deleteTree(); + repoUnderVendor.createDirectory(); + // 2. Move the marker file to a temporary one under vendor dir. + Path tMarker = vendorDirectory.getChild(repo.getMarkerFileName() + ".tmp"); + FileSystemUtils.moveFile(markerUnderExternal, tMarker); + // 3. Move the external repo to vendor dir. It's fine if this step fails or is interrupted, + // because the marker file under external is gone anyway. + FileSystemUtils.moveTreesBelow(repoUnderExternal, repoUnderVendor); + // 4. Re-plant symlinks pointing a path under the external root to a relative path + // to make sure the vendor src keep working after being moved or output base changed + replantSymlinks(repoUnderVendor, externalRepoRoot); + // 5. Rename the temporary marker file after the move is done. + tMarker.renameTo(markerUnderVendor); + // 6. Leave a symlink in external dir to keep things working. + repoUnderExternal.deleteTree(); + FileSystemUtils.ensureSymbolicLink(repoUnderExternal, repoUnderVendor); + } + } + } + + /** + * Replants the symlinks under the specified repository directory. + * + *

Re-write symlinks that originally pointing to a path under the external root to a relative + * path pointing to an external root symlink under the vendor directory. + * + * @param repoUnderVendor The path to the repository directory under the vendor directory. + * @param externalRepoRoot The path to the root of external repositories. + * @throws IOException If an I/O error occurs while replanting the symlinks. + */ + private void replantSymlinks(Path repoUnderVendor, Path externalRepoRoot) throws IOException { + try { + Collection symlinks = + FileSystemUtils.traverseTree(repoUnderVendor, Path::isSymbolicLink); + Path externalSymlinkUnderVendor = vendorDirectory.getChild(EXTERNAL_ROOT_SYMLINK_NAME); + FileSystemUtils.ensureSymbolicLink(externalSymlinkUnderVendor, externalRepoRoot); + for (Path symlink : symlinks) { + PathFragment target = symlink.readSymbolicLink(); + if (!target.startsWith(externalRepoRoot.asFragment())) { + // TODO: print a warning for absolute symlinks? + continue; + } + PathFragment newTarget = + PathFragment.create( + "../".repeat(symlink.relativeTo(vendorDirectory).segmentCount() - 1)) + .getRelative(EXTERNAL_ROOT_SYMLINK_NAME) + .getRelative(target.relativeTo(externalRepoRoot.asFragment())); + if (OS.getCurrent() == OS.WINDOWS) { + // On Windows, FileSystemUtils.ensureSymbolicLink always resolves paths to absolute path. + // Use Files.createSymbolicLink here instead to preserve relative target path. + symlink.delete(); + Files.createSymbolicLink( + java.nio.file.Path.of(symlink.getPathString()), + java.nio.file.Path.of(newTarget.getPathString())); + } else { + FileSystemUtils.ensureSymbolicLink(symlink, newTarget); + } + } + } catch (IOException e) { + throw new IOException( + String.format("Failed to rewrite symlinks under %s: ", repoUnderVendor), e); + } + } + + /** + * Checks if the given URL is vendored. + * + * @param url The URL to check. + * @return true if the URL is vendored, false otherwise. + * @throws UnsupportedEncodingException if the URL decoding fails. + */ + public boolean isUrlVendored(URL url) throws UnsupportedEncodingException { + return getVendorPathForUrl(url).isFile(); + } + + /** + * Vendors the registry URL with the specified content. + * + * @param url The registry URL to vendor. + * @param content The content to write. + * @throws IOException if an I/O error occurs. + */ + public void vendorRegistryUrl(URL url, byte[] content) throws IOException { + Path outputPath = getVendorPathForUrl(url); + Objects.requireNonNull(outputPath.getParentDirectory()).createDirectoryAndParents(); + FileSystemUtils.writeContent(outputPath, content); + } + + /** + * Reads the content of the registry URL and verifies its checksum. + * + * @param url The registry URL to read. + * @param checksum The checksum to verify. + * @return The content of the registry URL. + * @throws IOException if an I/O error occurs or the checksum verification fails. + */ + public byte[] readRegistryUrl(URL url, Checksum checksum) throws IOException { + byte[] content = FileSystemUtils.readContent(getVendorPathForUrl(url)); + Hasher hasher = checksum.getKeyType().newHasher(); + hasher.putBytes(content); + HashCode actual = hasher.hash(); + if (!checksum.getHashCode().equals(actual)) { + throw new IOException( + String.format( + "Checksum was %s but wanted %s", + checksum.emitOtherHashInSameFormat(actual), + checksum.emitOtherHashInSameFormat(checksum.getHashCode()))); + } + return content; + } + + /** + * Checks if the repository under vendor dir is up-to-date by comparing its marker file with the + * one under /external. This function assumes the marker file under + * /external exists and is up-to-date. + * + * @param markerUnderVendor The marker file path under vendor dir + * @param markerUnderExternal The marker file path under external dir + * @return true if the repository is up-to-date, false otherwise. + * @throws IOException if an I/O error occurs. + */ + private boolean isRepoUpToDate(Path markerUnderVendor, Path markerUnderExternal) + throws IOException { + if (!markerUnderVendor.exists()) { + return false; + } + String vendorMarkerContent = FileSystemUtils.readContent(markerUnderVendor, UTF_8); + String externalMarkerContent = FileSystemUtils.readContent(markerUnderExternal, UTF_8); + return Objects.equals(vendorMarkerContent, externalMarkerContent); + } + + /** + * Returns the vendor path for the given URL. + * + *

The vendor path is constructed as follows: /registry_cache// + * + *

The host name is case-insensitive, so it is converted to lowercase. The path is + * case-sensitive, so it is left as is. The port number is not included in the vendor path. + * + *

Note that the vendor path may conflict if two URLs only differ by the case or port number. + * But this is unlikely to happen in practice, and conflicts are checked in VendorCommand.java. + * + * @param url The URL to get the vendor path for. + * @return The vendor path. + * @throws UnsupportedEncodingException if the URL decoding fails. + */ + public Path getVendorPathForUrl(URL url) throws UnsupportedEncodingException { + String host = url.getHost().toLowerCase(Locale.ROOT); // Host names are case-insensitive + String path = url.getPath(); + path = URLDecoder.decode(path, "UTF-8"); + if (path.startsWith("/")) { + path = path.substring(1); + } + return vendorDirectory.getRelative(REGISTRIES_DIR).getRelative(host).getRelative(path); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD index bd2243525de4d4..37fcb87144414c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD @@ -31,17 +31,17 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:no_build_request_finished_event", "//src/main/java/com/google/devtools/build/lib/bazel:resolved_event", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", - "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:exception", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_value", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", - "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:root_module_file_fixup", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:tidy", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:vendor", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand", "//src/main/java/com/google/devtools/build/lib/bazel/repository", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", @@ -53,6 +53,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/rules:repository/resolved_file_value", "//src/main/java/com/google/devtools/build/lib/runtime/commands", "//src/main/java/com/google/devtools/build/lib/shell", + "//src/main/java/com/google/devtools/build/lib/skyframe:configured_target_key", "//src/main/java/com/google/devtools/build/lib/skyframe:package_lookup_value", "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value", "//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_value", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java index 72bcbd76155c58..1dc3be132eb9da 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java @@ -217,13 +217,15 @@ private BlazeCommandResult fetchRepos( private BlazeCommandResult fetchTarget( CommandEnvironment env, OptionsParsingResult options, List targets) { try { - TargetFetcher.fetchTargets(env, options, targets); + var unused = TargetFetcher.fetchTargets(env, options, targets); } catch (TargetFetcherException e) { return createFailedBlazeCommandResult( env.getReporter(), Code.QUERY_EVALUATION_ERROR, e.getMessage()); } env.getReporter() - .handle(Event.info("All external dependencies for these targets fetched successfully.")); + .handle( + Event.info( + "All external dependencies for the requested targets fetched successfully.")); return BlazeCommandResult.success(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/TargetFetcher.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/TargetFetcher.java index dbfd5fc4f39507..d6de01572fe18f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/TargetFetcher.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/TargetFetcher.java @@ -34,13 +34,13 @@ private TargetFetcher(CommandEnvironment env) { } /** Creates a no-build build request to fetch all repos needed to build these targets */ - public static void fetchTargets( + public static BuildResult fetchTargets( CommandEnvironment env, OptionsParsingResult options, List targets) throws TargetFetcherException { - new TargetFetcher(env).fetchTargets(options, targets); + return new TargetFetcher(env).fetchTargets(options, targets); } - private void fetchTargets(OptionsParsingResult options, List targets) + private BuildResult fetchTargets(OptionsParsingResult options, List targets) throws TargetFetcherException { BuildRequest request = BuildRequest.builder() @@ -59,6 +59,7 @@ private void fetchTargets(OptionsParsingResult options, List targets) "Fetching some target dependencies failed with errors: " + result.getDetailedExitCode().getFailureDetail().getMessage()); } + return result; } static void injectNoBuildOption(OptionsParser optionsParser) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java index bd90a48dc0071f..0df6a9acba0725 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java @@ -14,7 +14,6 @@ package com.google.devtools.build.lib.bazel.commands; import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -22,8 +21,14 @@ import com.google.devtools.build.lib.analysis.NoBuildEvent; import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent; import com.google.devtools.build.lib.bazel.bzlmod.BazelFetchAllValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionValue; +import com.google.devtools.build.lib.bazel.bzlmod.VendorManager; import com.google.devtools.build.lib.bazel.commands.RepositoryFetcher.RepositoryFetcherException; +import com.google.devtools.build.lib.bazel.commands.TargetFetcher.TargetFetcherException; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; +import com.google.devtools.build.lib.buildtool.BuildResult; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; @@ -38,48 +43,88 @@ import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.runtime.KeepGoingOption; import com.google.devtools.build.lib.runtime.LoadingPhaseThreadsOption; +import com.google.devtools.build.lib.runtime.commands.TestCommand; import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.server.FailureDetails.FetchCommand.Code; +import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.RepositoryMappingValue.RepositoryMappingResolutionException; -import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; -import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.build.lib.vfs.PathFragment; -import com.google.devtools.build.lib.vfs.Symlinks; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.InMemoryGraph; +import com.google.devtools.build.skyframe.NodeEntry; +import com.google.devtools.build.skyframe.QueryableGraph.Reason; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.OptionsParsingResult; import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Optional; +import java.util.Queue; +import java.util.Set; +import java.util.function.Supplier; import javax.annotation.Nullable; -/** Fetches external repositories into a specified directory. */ +/** + * Fetches external repositories into a specified directory. + * + *

This command is used to fetch external repositories into a specified directory. It can be used + * to fetch all external repositories, a specific list of repositories or the repositories needed to + * build a specific list of targets. + * + *

The command is used to create a vendor directory that can be used to build the project + * offline. + */ @Command( name = VendorCommand.NAME, + builds = true, + inherits = {TestCommand.class}, options = { VendorOptions.class, PackageOptions.class, KeepGoingOption.class, LoadingPhaseThreadsOption.class }, + allowResidue = true, + usesConfigurationOptions = true, help = "resource:vendor.txt", shortDescription = - "Fetches external repositories into a specific folder specified by the flag " - + "--vendor_dir.") + "Fetches external repositories into a folder specified by the flag --vendor_dir.") public final class VendorCommand implements BlazeCommand { public static final String NAME = "vendor"; - // TODO(salmasamy) decide on name and format - private static final String VENDOR_IGNORE = ".vendorignore"; + private final DownloadManager downloadManager; + private final Supplier> clientEnvironmentSupplier; + @Nullable private VendorManager vendorManager = null; + + public VendorCommand( + DownloadManager downloadManager, Supplier> clientEnvironmentSupplier) { + this.downloadManager = downloadManager; + this.clientEnvironmentSupplier = clientEnvironmentSupplier; + } + + @Override + public void editOptions(OptionsParser optionsParser) { + // We only need to inject these options with fetch target (when there is a residue) + if (!optionsParser.getResidue().isEmpty()) { + TargetFetcher.injectNoBuildOption(optionsParser); + } + } @Override public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult options) { @@ -106,18 +151,18 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti BlazeCommandResult result; VendorOptions vendorOptions = options.getOptions(VendorOptions.class); - PathFragment vendorDirectory = options.getOptions(RepositoryOptions.class).vendorDirectory; LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); + Path vendorDirectory = + env.getWorkspace().getRelative(options.getOptions(RepositoryOptions.class).vendorDirectory); + this.vendorManager = new VendorManager(vendorDirectory); try { - env.syncPackageLoading(options); - if (!vendorOptions.repos.isEmpty()) { - result = vendorRepos(env, threadsOption, vendorOptions.repos, vendorDirectory); + if (!options.getResidue().isEmpty()) { + result = vendorTargets(env, options, options.getResidue()); + } else if (!vendorOptions.repos.isEmpty()) { + result = vendorRepos(env, threadsOption, vendorOptions.repos); } else { - result = vendorAll(env, threadsOption, vendorDirectory); + result = vendorAll(env, threadsOption); } - } catch (AbruptExitException e) { - return createFailedBlazeCommandResult( - env.getReporter(), e.getMessage(), e.getDetailedExitCode()); } catch (InterruptedException e) { return createFailedBlazeCommandResult( env.getReporter(), "Vendor interrupted: " + e.getMessage()); @@ -144,17 +189,19 @@ private BlazeCommandResult validateOptions(CommandEnvironment env, OptionsParsin return createFailedBlazeCommandResult( env.getReporter(), Code.OPTIONS_INVALID, - "You cannot run vendor without specifying --vendor_dir"); + "You cannot run the vendor command without specifying --vendor_dir"); } if (!options.getOptions(PackageOptions.class).fetch) { return createFailedBlazeCommandResult( - env.getReporter(), Code.OPTIONS_INVALID, "You cannot run vendor with --nofetch"); + env.getReporter(), + Code.OPTIONS_INVALID, + "You cannot run the vendor command with --nofetch"); } return null; } private BlazeCommandResult vendorAll( - CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, PathFragment vendorDirectory) + CommandEnvironment env, LoadingPhaseThreadsOption threadsOption) throws InterruptedException, IOException { EvaluationContext evaluationContext = EvaluationContext.newBuilder() @@ -165,23 +212,22 @@ private BlazeCommandResult vendorAll( SkyKey fetchKey = BazelFetchAllValue.key(/* configureEnabled= */ false); EvaluationResult evaluationResult = env.getSkyframeExecutor().prepareAndGet(ImmutableSet.of(fetchKey), evaluationContext); - if (evaluationResult.hasError()) { - Exception e = evaluationResult.getError().getException(); - return createFailedBlazeCommandResult( - env.getReporter(), - e != null ? e.getMessage() : "Unexpected error during fetching all external deps."); - } + if (evaluationResult.hasError()) { + Exception e = evaluationResult.getError().getException(); + return createFailedBlazeCommandResult( + env.getReporter(), + e != null ? e.getMessage() : "Unexpected error during fetching all external deps."); + } BazelFetchAllValue fetchAllValue = (BazelFetchAllValue) evaluationResult.get(fetchKey); - vendor(env, vendorDirectory, fetchAllValue.getReposToVendor()); + env.getReporter().handle(Event.info("Vendoring all external repositories...")); + vendor(env, fetchAllValue.getReposToVendor()); + env.getReporter().handle(Event.info("All external dependencies vendored successfully.")); return BlazeCommandResult.success(); } private BlazeCommandResult vendorRepos( - CommandEnvironment env, - LoadingPhaseThreadsOption threadsOption, - List repos, - PathFragment vendorDirectory) + CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, List repos) throws InterruptedException, IOException { ImmutableMap repositoryNamesAndValues; try { @@ -207,83 +253,151 @@ private BlazeCommandResult vendorRepos( } } - vendor(env, vendorDirectory, reposToVendor.build()); + env.getReporter().handle(Event.info("Vendoring repositories...")); + vendor(env, reposToVendor.build()); if (!notFoundRepoErrors.isEmpty()) { return createFailedBlazeCommandResult( env.getReporter(), "Vendoring some repos failed with errors: " + notFoundRepoErrors); } + env.getReporter().handle(Event.info("All requested repos vendored successfully.")); return BlazeCommandResult.success(); } - /** - * Copies the fetched repos from the external cache into the vendor directory, unless the repo is - * ignored or was already vendored and up-to-date - */ - private void vendor( - CommandEnvironment env, - PathFragment vendorDirectory, - ImmutableList reposToVendor) - throws IOException { - Path vendorPath = - vendorDirectory.isAbsolute() - ? env.getRuntime().getFileSystem().getPath(vendorDirectory) - : env.getWorkspace().getRelative(vendorDirectory); - Path externalPath = - env.getDirectories() - .getOutputBase() - .getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION); - Path vendorIgnore = vendorPath.getRelative(VENDOR_IGNORE); - - if (!vendorPath.exists()) { - vendorPath.createDirectory(); + private BlazeCommandResult vendorTargets( + CommandEnvironment env, OptionsParsingResult options, List targets) + throws InterruptedException, IOException { + // Call fetch which runs build to have the targets graph and configuration set + BuildResult buildResult; + try { + buildResult = TargetFetcher.fetchTargets(env, options, targets); + } catch (TargetFetcherException e) { + return createFailedBlazeCommandResult( + env.getReporter(), Code.QUERY_EVALUATION_ERROR, e.getMessage()); } - // exclude any ignored repo under .vendorignore - if (vendorIgnore.exists()) { - ImmutableSet ignoredRepos = - ImmutableSet.copyOf(FileSystemUtils.readLines(vendorIgnore, UTF_8)); - reposToVendor = - reposToVendor.stream() - .filter(repo -> !ignoredRepos.contains(repo.getName())) - .collect(toImmutableList()); - } else { - FileSystemUtils.createEmptyFile(vendorIgnore); - } + // Traverse the graph created from build to collect repos and vendor them + ImmutableList targetKeys = + buildResult.getActualTargets().stream() + .map( + target -> + ConfiguredTargetKey.builder() + .setConfigurationKey(target.getConfigurationKey()) + .setLabel(target.getLabel()) + .build()) + .collect(toImmutableList()); + InMemoryGraph inMemoryGraph = env.getSkyframeExecutor().getEvaluator().getInMemoryGraph(); + ImmutableSet reposToVendor = collectReposFromTargets(inMemoryGraph, targetKeys); + + env.getReporter().handle(Event.info("Vendoring dependencies for targets...")); + vendor(env, reposToVendor.asList()); + env.getReporter() + .handle( + Event.info( + "All external dependencies for the requested targets vendored successfully.")); + return BlazeCommandResult.success(); + } - // Update "out-of-date" repos under the vendor directory - for (RepositoryName repo : reposToVendor) { - if (!isRepoUpToDate(repo.getName(), vendorPath, externalPath)) { - Path repoUnderVendor = vendorPath.getRelative(repo.getName()); - if (!repoUnderVendor.exists()) { - repoUnderVendor.createDirectory(); + private ImmutableSet collectReposFromTargets( + InMemoryGraph inMemoryGraph, ImmutableList targetKeys) throws InterruptedException { + ImmutableSet.Builder repos = ImmutableSet.builder(); + Queue nodes = new ArrayDeque<>(targetKeys); + Set visited = new HashSet<>(); + while (!nodes.isEmpty()) { + SkyKey key = nodes.remove(); + visited.add(key); + NodeEntry nodeEntry = inMemoryGraph.get(null, Reason.VENDOR_EXTERNAL_REPOS, key); + if (nodeEntry.getValue() instanceof RepositoryDirectoryValue repoDirValue + && repoDirValue.repositoryExists() + && !repoDirValue.excludeFromVendoring()) { + repos.add((RepositoryName) key.argument()); + } + for (SkyKey depKey : nodeEntry.getDirectDeps()) { + if (!visited.contains(depKey)) { + nodes.add(depKey); } - FileSystemUtils.copyTreesBelow( - externalPath.getRelative(repo.getName()), repoUnderVendor, Symlinks.NOFOLLOW); - FileSystemUtils.copyFile( - externalPath.getChild("@" + repo.getName() + ".marker"), - vendorPath.getChild("@" + repo.getName() + ".marker")); } } + return repos.build(); } /** - * Returns whether the repo under vendor needs to be updated by comparing its marker file with the - * one under /external + * Copies the fetched repos from the external cache into the vendor directory, unless the repo is + * ignored or was already vendored and up-to-date */ - private boolean isRepoUpToDate(String repoName, Path vendorPath, Path externalPath) - throws IOException { - Path vendorMarkerFile = vendorPath.getChild("@" + repoName + ".marker"); - if (!vendorMarkerFile.exists()) { - return false; + private void vendor(CommandEnvironment env, ImmutableList reposToVendor) + throws IOException, InterruptedException { + Objects.requireNonNull(vendorManager); + + // 1. Vendor registry files + BazelModuleResolutionValue moduleResolutionValue = + (BazelModuleResolutionValue) + env.getSkyframeExecutor() + .getEvaluator() + .getExistingValue(BazelModuleResolutionValue.KEY); + ImmutableMap> registryFiles = + Objects.requireNonNull(moduleResolutionValue).getRegistryFileHashes(); + + // vendorPathToURL is a map of + // key: a vendor path string converted to lower case + // value: a URL string + // This map is for detecting potential rare vendor path conflicts, such as: + // http://foo.bar.com/BCR vs http://foo.bar.com/bcr => conflict vendor paths on + // case-insensitive system + // http://foo.bar.com/bcr vs http://foo.bar.com:8081/bcr => conflict vendor path because port + // number is ignored in vendor path + // The user has to update the Bazel registries this if such conflicts occur. + Map vendorPathToUrl = new HashMap<>(); + for (Entry> entry : registryFiles.entrySet()) { + URL url = URI.create(entry.getKey()).toURL(); + if (url.getProtocol().equals("file")) { + continue; + } + + String outputPath = vendorManager.getVendorPathForUrl(url).getPathString(); + String outputPathLowerCase = outputPath.toLowerCase(Locale.ROOT); + if (vendorPathToUrl.containsKey(outputPathLowerCase)) { + String previousUrl = vendorPathToUrl.get(outputPathLowerCase); + throw new IOException( + String.format( + "Vendor paths conflict detected for registry URLs:\n" + + " %s => %s\n" + + " %s => %s\n" + + "Their output paths are either the same or only differ by case, which will" + + " cause conflict on case insensitive file systems, please fix by changing the" + + " registry URLs!", + previousUrl, + vendorManager.getVendorPathForUrl(URI.create(previousUrl).toURL()).getPathString(), + entry.getKey(), + outputPath)); + } + + Optional checksum = entry.getValue(); + if (!vendorManager.isUrlVendored(url) + // Only vendor a registry URL when its checksum exists, otherwise the URL should be + // recorded as "not found" in moduleResolutionValue.getRegistryFileHashes() + && checksum.isPresent()) { + try { + vendorManager.vendorRegistryUrl( + url, + downloadManager.downloadAndReadOneUrlForBzlmod( + url, env.getReporter(), clientEnvironmentSupplier.get(), checksum)); + } catch (IOException e) { + throw new IOException( + String.format( + "Failed to vendor registry URL %s at %s: %s", url, outputPath, e.getMessage()), + e.getCause()); + } + } + + vendorPathToUrl.put(outputPathLowerCase, entry.getKey()); } - // Since this runs after BazelFetchAllFunction, its guaranteed that the marker files - // under $OUTPUT_BASE/external are up-to-date. We just need to compare it against the marker - // under vendor. - Path externalMarkerFile = externalPath.getChild("@" + repoName + ".marker"); - String vendorMarkerContent = FileSystemUtils.readContent(vendorMarkerFile, UTF_8); - String externalMarkerContent = FileSystemUtils.readContent(externalMarkerFile, UTF_8); - return Objects.equals(vendorMarkerContent, externalMarkerContent); + // 2. Vendor repos + Path externalPath = + env.getDirectories() + .getOutputBase() + .getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION); + vendorManager.vendorRepos(externalPath, reposToVendor); } private static BlazeCommandResult createFailedBlazeCommandResult( diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java b/src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java index 44f81ff7022b41..8a3a1a80f1683b 100644 --- a/src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java +++ b/src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java @@ -45,6 +45,7 @@ public class LabelConstants { PathFragment.create("WORKSPACE.bazel"); public static final PathFragment MODULE_DOT_BAZEL_FILE_NAME = PathFragment.create("MODULE.bazel"); public static final PathFragment REPO_FILE_NAME = PathFragment.create("REPO.bazel"); + public static final PathFragment VENDOR_FILE_NAME = PathFragment.create("VENDOR.bazel"); public static final PathFragment MODULE_LOCKFILE_NAME = PathFragment.create("MODULE.bazel.lock"); diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java b/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java index 6e046b583cdbb8..5a386b3d0ef8be 100644 --- a/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java +++ b/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java @@ -182,6 +182,11 @@ public String getName() { return name; } + /** Returns the marker file name for this repository. */ + public String getMarkerFileName() { + return "@" + name + ".marker"; + } + /** * Create a {@link RepositoryName} instance that indicates the requested repository name is * actually not visible from the owner repository and should fail in {@code diff --git a/src/main/java/com/google/devtools/build/lib/packages/StarlarkGlobals.java b/src/main/java/com/google/devtools/build/lib/packages/StarlarkGlobals.java index 9f51c51493db6e..b79a94e11a6cca 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/StarlarkGlobals.java +++ b/src/main/java/com/google/devtools/build/lib/packages/StarlarkGlobals.java @@ -70,4 +70,7 @@ public interface StarlarkGlobals { /** Returns the top-levels for REPO.bazel files. */ ImmutableMap getRepoToplevels(); + + /** Returns the top-levels for VENDOR.bazel files. */ + ImmutableMap getVendorToplevels(); } diff --git a/src/main/java/com/google/devtools/build/lib/packages/VendorFileGlobals.java b/src/main/java/com/google/devtools/build/lib/packages/VendorFileGlobals.java new file mode 100644 index 00000000000000..40aa36e944a411 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/VendorFileGlobals.java @@ -0,0 +1,78 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.devtools.build.docgen.annot.GlobalMethods; +import com.google.devtools.build.docgen.annot.GlobalMethods.Environment; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import net.starlark.java.annot.Param; +import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.Sequence; +import net.starlark.java.eval.Starlark; +import net.starlark.java.eval.StarlarkThread; +import net.starlark.java.eval.Tuple; + +/** Definition of the functions used in VENDOR.bazel file. */ +@GlobalMethods(environment = Environment.VENDOR) +public final class VendorFileGlobals { + private VendorFileGlobals() {} + + public static final VendorFileGlobals INSTANCE = new VendorFileGlobals(); + + @StarlarkMethod( + name = "ignore", + doc = + "Ignore this repo from vendoring. Bazel will never vendor it or use the corresponding" + + " directory (if exists) while building in vendor mode.", + extraPositionals = + @Param(name = "args", doc = "The canonical repo names of the repos to ignore."), + useStarlarkThread = true) + public void ignore(Tuple args, StarlarkThread thread) throws EvalException { + VendorThreadContext context = VendorThreadContext.fromOrFail(thread, "ignore()"); + for (String repoName : Sequence.cast(args, String.class, "args")) { + context.addIgnoredRepo(getRepositoryName(repoName)); + } + } + + @StarlarkMethod( + name = "pin", + doc = + "Pin the contents of this repo under the vendor directory. Bazel will not update this" + + " repo while vendoring, and will use the vendored source as if there is a" + + " --override_repository flag when building in vendor mode", + extraPositionals = + @Param(name = "args", doc = "The canonical repo names of the repos to pin."), + useStarlarkThread = true) + public void pin(Tuple args, StarlarkThread thread) throws EvalException { + VendorThreadContext context = VendorThreadContext.fromOrFail(thread, "pin()"); + for (String repoName : Sequence.cast(args, String.class, "args")) { + context.addPinnedRepo(getRepositoryName(repoName)); + } + } + + private RepositoryName getRepositoryName(String repoName) throws EvalException { + if (!repoName.startsWith("@@")) { + throw Starlark.errorf("the canonical repository name must start with `@@`"); + } + try { + repoName = repoName.substring(2); + return RepositoryName.create(repoName); + } catch (LabelSyntaxException e) { + throw Starlark.errorf("Invalid canonical repo name: %s", e.getMessage()); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/packages/VendorThreadContext.java b/src/main/java/com/google/devtools/build/lib/packages/VendorThreadContext.java new file mode 100644 index 00000000000000..f668f4301a9232 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/VendorThreadContext.java @@ -0,0 +1,61 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import java.util.ArrayList; +import java.util.List; +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.Starlark; +import net.starlark.java.eval.StarlarkThread; + +/** Context object for a Starlark thread evaluating the VENDOR.bazel file. */ +public class VendorThreadContext { + + private final List ignoredRepos = new ArrayList<>(); + private final List pinnedRepos = new ArrayList<>(); + + public static VendorThreadContext fromOrFail(StarlarkThread thread, String what) + throws EvalException { + VendorThreadContext context = thread.getThreadLocal(VendorThreadContext.class); + if (context == null) { + throw Starlark.errorf("%s can only be called from VENDOR.bazel", what); + } + return context; + } + + public void storeInThread(StarlarkThread thread) { + thread.setThreadLocal(VendorThreadContext.class, this); + } + + public VendorThreadContext() {} + + public ImmutableList getIgnoredRepos() { + return ImmutableList.copyOf(ignoredRepos); + } + + public ImmutableList getPinnedRepos() { + return ImmutableList.copyOf(pinnedRepos); + } + + public void addIgnoredRepo(RepositoryName repoName) { + ignoredRepos.add(repoName); + } + + public void addPinnedRepo(RepositoryName repoName) { + pinnedRepos.add(repoName); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java index 2024b49cfe2ce9..3444eb5bea1b06 100644 --- a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java +++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java @@ -99,6 +99,7 @@ public enum ProfilerTask { CONFLICT_CHECK("Conflict checking"), DYNAMIC_LOCK("Acquiring dynamic execution output lock", Threshold.FIFTY_MILLIS), REPOSITORY_FETCH("Fetching repository"), + REPOSITORY_VENDOR("Vendoring repository"), UNKNOWN("Unknown event"); private static class Threshold { diff --git a/src/main/java/com/google/devtools/build/lib/rules/BUILD b/src/main/java/com/google/devtools/build/lib/rules/BUILD index 9ffaae384abbd2..9e426ad5dfd20c 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/BUILD +++ b/src/main/java/com/google/devtools/build/lib/rules/BUILD @@ -428,6 +428,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories", "//src/main/java/com/google/devtools/build/lib/bazel:resolved_event", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_value", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/concurrent", "//src/main/java/com/google/devtools/build/lib/events", diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java index 020e375951a921..6ac3f346914e09 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; +import com.google.devtools.build.lib.bazel.bzlmod.VendorFileValue; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.packages.Rule; @@ -147,9 +148,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) .getRelative(repositoryName.getName()); Map overrides = REPOSITORY_OVERRIDES.get(env); if (Preconditions.checkNotNull(overrides).containsKey(repositoryName)) { - DigestWriter.clearMarkerFile(directories, repositoryName); - return setupOverride( - overrides.get(repositoryName), env, repoRoot, repositoryName.getName()); + return setupOverride(overrides.get(repositoryName), env, repoRoot, repositoryName); } Rule rule = getRepositoryRule(env, repositoryName, starlarkSemantics); @@ -170,17 +169,31 @@ public SkyValue compute(SkyKey skyKey, Environment env) DigestWriter digestWriter = new DigestWriter(directories, repositoryName, rule, starlarkSemantics); - if (shouldUseVendorRepos(env, handler, rule)) { - RepositoryDirectoryValue repositoryDirectoryValue = - tryGettingValueUsingVendoredRepo( - env, rule, repoRoot, repositoryName, handler, digestWriter); + + boolean excludeRepoFromVendoring = true; + if (VENDOR_DIRECTORY.get(env).isPresent()) { // If vendor mode is on + VendorFileValue vendorFile = (VendorFileValue) env.getValue(VendorFileValue.KEY); if (env.valuesMissing()) { return null; } - if (repositoryDirectoryValue != null) { - return repositoryDirectoryValue; + boolean excludeRepoByDefault = isRepoExcludedFromVendoringByDefault(handler, rule); + if (!excludeRepoByDefault && !vendorFile.getIgnoredRepos().contains(repositoryName)) { + RepositoryDirectoryValue repositoryDirectoryValue = + tryGettingValueUsingVendoredRepo( + env, rule, repoRoot, repositoryName, handler, digestWriter, vendorFile); + if (env.valuesMissing()) { + return null; + } + if (repositoryDirectoryValue != null) { + return repositoryDirectoryValue; + } } + excludeRepoFromVendoring = + excludeRepoByDefault + || vendorFile.getIgnoredRepos().contains(repositoryName) + || vendorFile.getPinnedRepos().contains(repositoryName); } + if (shouldUseCachedRepos(env, handler, repoRoot, rule)) { // Make sure marker file is up-to-date; correctly describes the current repository state byte[] markerHash = digestWriter.areRepositoryAndMarkerFileConsistent(handler, env); @@ -191,7 +204,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) return RepositoryDirectoryValue.builder() .setPath(repoRoot) .setDigest(markerHash) - .setExcludeFromVendoring(shouldExcludeRepoFromVendoring(handler, rule)) + .setExcludeFromVendoring(excludeRepoFromVendoring) .build(); } } @@ -216,10 +229,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) // restart thus calling the possibly very slow (networking, decompression...) fetch() // operation again. So we write the marker file here immediately. byte[] digest = digestWriter.writeMarkerFile(); - return builder - .setDigest(digest) - .setExcludeFromVendoring(shouldExcludeRepoFromVendoring(handler, rule)) - .build(); + return builder.setDigest(digest).setExcludeFromVendoring(excludeRepoFromVendoring).build(); } if (!repoRoot.exists()) { @@ -246,7 +256,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) .setPath(repoRoot) .setFetchingDelayed() .setDigest(new Fingerprint().digestAndReset()) - .setExcludeFromVendoring(shouldExcludeRepoFromVendoring(handler, rule)) + .setExcludeFromVendoring(excludeRepoFromVendoring) .build(); } } @@ -258,12 +268,25 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( Path repoRoot, RepositoryName repositoryName, RepositoryFunction handler, - DigestWriter digestWriter) + DigestWriter digestWriter, + VendorFileValue vendorFile) throws RepositoryFunctionException, InterruptedException { Path vendorPath = VENDOR_DIRECTORY.get(env).get(); Path vendorRepoPath = vendorPath.getRelative(repositoryName.getName()); if (vendorRepoPath.exists()) { - Path vendorMarker = vendorPath.getChild("@" + repositoryName.getName() + ".marker"); + Path vendorMarker = vendorPath.getChild(repositoryName.getMarkerFileName()); + if (vendorFile.getPinnedRepos().contains(repositoryName)) { + // pinned repos are used as they are without checking their marker file + try { + // delete the marker as it may become out-of-date while it's pinned (old version or + // manual changes) + vendorMarker.delete(); + } catch (IOException e) { + throw new RepositoryFunctionException(e, Transience.TRANSIENT); + } + return setupOverride(vendorRepoPath.asFragment(), env, repoRoot, repositoryName); + } + boolean isVendorRepoUpToDate = digestWriter.areRepositoryAndMarkerFileConsistent(handler, env, vendorMarker) != null; if (env.valuesMissing()) { @@ -280,10 +303,10 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( String.format( "Vendored repository '%s' is out-of-date and fetching is disabled." + " Run build without the '--nofetch' option or run" - + " `bazel vendor` to update it", + + " the bazel vendor command to update it", rule.getName()))); } - return setupOverride(vendorRepoPath.asFragment(), env, repoRoot, repositoryName.getName()); + return setupOverride(vendorRepoPath.asFragment(), env, repoRoot, repositoryName); } else if (!IS_VENDOR_COMMAND.get(env).booleanValue()) { // build command & fetch enabled // We will continue fetching but warn the user that we are not using the vendored repo env.getListener() @@ -293,16 +316,23 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( String.format( "Vendored repository '%s' is out-of-date. The up-to-date version will" + " be fetched into the external cache and used. To update the repo" - + " in the vendor directory, run 'bazel vendor'", + + " in the vendor directory, run the bazel vendor command", rule.getName()))); } + } else if (vendorFile.getPinnedRepos().contains(repositoryName)) { + throw new RepositoryFunctionException( + new IOException( + "Pinned repository " + + repositoryName.getName() + + " not found under the vendor directory"), + Transience.PERSISTENT); } else if (!isFetch.get()) { // repo not vendored & fetching is disabled (--nofetch) throw new RepositoryFunctionException( new IOException( "Vendored repository " + repositoryName.getName() + " not found under the vendor directory and fetching is disabled." - + " To fix run 'bazel vendor' or build without the '--nofetch'"), + + " To fix, run the bazel vendor command or build without the '--nofetch'"), Transience.TRANSIENT); } return null; @@ -382,21 +412,7 @@ private boolean shouldUseCachedRepos( return true; } - /* Determines whether we should use the vendored repositories */ - private boolean shouldUseVendorRepos(Environment env, RepositoryFunction handler, Rule rule) - throws InterruptedException { - if (VENDOR_DIRECTORY.get(env).isEmpty()) { // If vendor mode is off - return false; - } - - if (shouldExcludeRepoFromVendoring(handler, rule)) { - return false; - } - - return true; - } - - private boolean shouldExcludeRepoFromVendoring(RepositoryFunction handler, Rule rule) { + private boolean isRepoExcludedFromVendoringByDefault(RepositoryFunction handler, Rule rule) { return handler.isLocal(rule) || handler.isConfigure(rule) || RepositoryFunction.isWorkspaceRepo(rule); @@ -467,15 +483,16 @@ private Rule getRepoRuleFromWorkspace(RepositoryName repositoryName, Environment @Nullable private RepositoryDirectoryValue setupOverride( - PathFragment sourcePath, Environment env, Path repoRoot, String pathAttr) + PathFragment sourcePath, Environment env, Path repoRoot, RepositoryName repoName) throws RepositoryFunctionException, InterruptedException { + DigestWriter.clearMarkerFile(directories, repoName); RepositoryFunction.setupRepoRoot(repoRoot); RepositoryDirectoryValue.Builder directoryValue = symlinkRepoRoot( directories, repoRoot, directories.getWorkspace().getRelative(sourcePath), - pathAttr, + repoName.getName(), env); if (directoryValue == null) { return null; @@ -604,7 +621,7 @@ private static class DigestWriter { StarlarkSemantics starlarkSemantics) { this.directories = directories; ruleKey = computeRuleKey(rule, starlarkSemantics); - markerPath = getMarkerPath(directories, repositoryName.getName()); + markerPath = getMarkerPath(directories, repositoryName); this.rule = rule; recordedInputValues = Maps.newTreeMap(); } @@ -714,15 +731,15 @@ private String computeRuleKey(Rule rule, StarlarkSemantics starlarkSemantics) { .hexDigestAndReset(); } - private static Path getMarkerPath(BlazeDirectories directories, String ruleName) { + private static Path getMarkerPath(BlazeDirectories directories, RepositoryName repo) { return RepositoryFunction.getExternalRepositoryDirectory(directories) - .getChild("@" + ruleName + ".marker"); + .getChild(repo.getMarkerFileName()); } - static void clearMarkerFile(BlazeDirectories directories, RepositoryName repoName) + static void clearMarkerFile(BlazeDirectories directories, RepositoryName repo) throws RepositoryFunctionException { try { - getMarkerPath(directories, repoName.getName()).delete(); + getMarkerPath(directories, repo).delete(); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java index 12729b26469ce9..5270da498cd83b 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java @@ -167,6 +167,8 @@ public final class SkyFunctions { public static final SkyFunctionName MODULE_EXTENSION_REPO_MAPPING_ENTRIES = SkyFunctionName.createHermetic("MODULE_EXTENSION_REPO_MAPPING_ENTRIES"); + public static final SkyFunctionName VENDOR_FILE = SkyFunctionName.createHermetic("VENDOR_FILE"); + ; public static Predicate isSkyFunction(SkyFunctionName functionName) { return key -> key.functionName().equals(functionName); diff --git a/src/main/java/com/google/devtools/build/skyframe/QueryableGraph.java b/src/main/java/com/google/devtools/build/skyframe/QueryableGraph.java index e42b03ef8ef3f7..98948869bc152d 100644 --- a/src/main/java/com/google/devtools/build/skyframe/QueryableGraph.java +++ b/src/main/java/com/google/devtools/build/skyframe/QueryableGraph.java @@ -221,6 +221,9 @@ enum Reason { /** The node is being looked up to service another "graph lookup" function. */ WALKABLE_GRAPH_OTHER, + /** The node is being looked up to vendor external repos from its dependencies. */ + VENDOR_EXTERNAL_REPOS, + /** Some other reason than one of the above that needs the node's value and deps. */ OTHER_NEEDING_VALUE_AND_DEPS, diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java index 128a0adc029d0e..a75be1e2584f1b 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java @@ -256,6 +256,8 @@ protected void useRuleClassProvider(ConfiguredRuleClassProvider ruleClassProvide PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), + PrecomputedValue.injected( + RepositoryDelegatorFunction.VENDOR_DIRECTORY, Optional.empty()), PrecomputedValue.injected( BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE))) .build(ruleClassProvider, fileSystem); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 378fff3ced078c..9f45c76c154d01 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -121,6 +121,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", + "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/net/starlark/java/eval", "//src/main/java/net/starlark/java/syntax", "//third_party:guava", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 9c98532eec437b..763eeec7b74044 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -37,6 +37,7 @@ import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; import com.google.devtools.build.lib.skyframe.BzlmodRepoRuleFunction; import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; @@ -62,6 +63,7 @@ import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionName; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import net.starlark.java.eval.StarlarkSemantics; import org.junit.Before; @@ -158,6 +160,7 @@ public void setup() throws Exception { BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); + RepositoryDelegatorFunction.VENDOR_DIRECTORY.set(differencer, Optional.empty()); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java index 9fe40a91905fd4..d1d20aedba16c9 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java @@ -23,6 +23,7 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.vfs.Path; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.HashMap; import java.util.Map; @@ -136,7 +137,8 @@ public Registry createRegistry( String url, LockfileMode lockfileMode, ImmutableMap> fileHashes, - ImmutableMap previouslySelectedYankedVersions) { + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) { return Preconditions.checkNotNull(registries.get(url), "unknown registry url: %s", url); } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java index 9e20b46e83c65d..2608a2fb8ee40a 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java @@ -98,7 +98,11 @@ public void testHttpUrl() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( @@ -116,7 +120,11 @@ public void testHttpUrlWithNetrcCreds() throws Exception { "machine [::1] login rinne password rinnepass\n".getBytes(UTF_8))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); var e = assertThrows( @@ -149,7 +157,8 @@ public void testFileUrl() throws Exception { new File(tempFolder.getRoot(), "fakereg").toURI().toString(), LockfileMode.UPDATE, ImmutableMap.of(), - ImmutableMap.of()); + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue(ModuleFile.create("lol".getBytes(UTF_8), file.toURI().toString())); assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); @@ -198,7 +207,11 @@ public void testGetArchiveRepoSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -266,7 +279,11 @@ public void testGetLocalPathRepoSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( RepoSpec.builder() @@ -290,7 +307,11 @@ public void testGetRepoInvalidRegistryJsonSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -324,7 +345,11 @@ public void testGetRepoInvalidModuleJsonSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); } @@ -353,7 +378,11 @@ public void testGetYankedVersion() throws Exception { server.start(); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); Optional> yankedVersion = registry.getYankedVersions("red-pill", reporter); assertThat(yankedVersion) @@ -376,7 +405,11 @@ public void testArchiveWithExplicitType() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("archive_type", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -406,7 +439,11 @@ public void testGetModuleFileChecksums() throws Exception { Optional.of(sha256("unused"))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + knownFiles, + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( @@ -432,7 +469,11 @@ public void testGetModuleFileChecksums() throws Exception { registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + recordedChecksums, + ImmutableMap.of(), + Optional.empty()); // Test that the recorded hashes are used for repo cache hits even when the server content // changes. server.unserve("/myreg/modules/foo/1.0/MODULE.bazel"); @@ -462,7 +503,11 @@ public void testGetModuleFileChecksumMismatch() throws Exception { Optional.of(sha256("original"))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + knownFiles, + ImmutableMap.of(), + Optional.empty()); var e = assertThrows( IOException.class, @@ -503,7 +548,7 @@ public void testGetRepoSpecChecksum() throws Exception { server.getUrl() + "/modules/foo/2.0/source.json", Optional.of(sha256("unused"))); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( RepoSpec.builder() @@ -523,7 +568,11 @@ public void testGetRepoSpecChecksum() throws Exception { registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + recordedChecksums, + ImmutableMap.of(), + Optional.empty()); // Test that the recorded hashes are used for repo cache hits even when the server content // changes. server.unserve("/bazel_registry.json"); @@ -567,7 +616,7 @@ public void testGetRepoSpecChecksumMismatch() throws Exception { Optional.of(sha256(sourceJson))); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); var e = assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); @@ -611,7 +660,7 @@ public void testBazelRegistryChecksumMismatch() throws Exception { Optional.of(sha256(sourceJson))); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); var e = assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java index cc963f233e5e3e..fb52c7fe216416 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import java.net.URISyntaxException; +import java.util.Optional; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -44,14 +45,22 @@ public void badSchemes() { URISyntaxException.class, () -> registryFactory.createRegistry( - "/home/www", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of())); + "/home/www", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty())); assertThat(exception).hasMessageThat().contains("Registry URL has no scheme"); exception = assertThrows( URISyntaxException.class, () -> registryFactory.createRegistry( - "foo://bar", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of())); + "foo://bar", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty())); assertThat(exception).hasMessageThat().contains("Unrecognized registry URL protocol"); } @@ -69,7 +78,8 @@ public void badPath() { "file:c:/path/to/workspace/registry", LockfileMode.UPDATE, ImmutableMap.of(), - ImmutableMap.of())); + ImmutableMap.of(), + Optional.empty())); assertThat(exception).hasMessageThat().contains("Registry URL path is not valid"); } } diff --git a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java index 8f60ae25a85157..9a70266f7e2e42 100644 --- a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java +++ b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java @@ -391,6 +391,8 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), + PrecomputedValue.injected( + RepositoryDelegatorFunction.VENDOR_DIRECTORY, Optional.empty()), PrecomputedValue.injected( BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.UPDATE))) .build(ruleClassProvider, fileSystem); diff --git a/src/test/py/bazel/bzlmod/bazel_vendor_test.py b/src/test/py/bazel/bzlmod/bazel_vendor_test.py index b647a59c472f94..813c32d7025f5d 100644 --- a/src/test/py/bazel/bzlmod/bazel_vendor_test.py +++ b/src/test/py/bazel/bzlmod/bazel_vendor_test.py @@ -79,7 +79,7 @@ def generateBuiltinModules(self): def testBasicVendoring(self): self.main_registry.createCcModule('aaa', '1.0').createCcModule( 'bbb', '1.0', {'aaa': '1.0'} - ) + ).createCcModule('bbb', '2.0') self.ScratchFile( 'MODULE.bazel', [ @@ -93,21 +93,53 @@ def testBasicVendoring(self): self.RunBazel(['vendor', '--vendor_dir=vendor']) - # Assert repos are vendored with marker files and .vendorignore is created - repos_vendored = os.listdir(self._test_cwd + '/vendor') + # Assert repos are vendored with marker files and VENDOR.bazel is created + vendor_dir = self._test_cwd + '/vendor' + repos_vendored = os.listdir(vendor_dir) self.assertIn('aaa~', repos_vendored) self.assertIn('bbb~', repos_vendored) self.assertIn('@aaa~.marker', repos_vendored) self.assertIn('@bbb~.marker', repos_vendored) - self.assertIn('.vendorignore', repos_vendored) + self.assertIn('VENDOR.bazel', repos_vendored) + + # Update bbb to 2.0 and re-vendor + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "bbb", version = "2.0")', + 'local_path_override(module_name="bazel_tools", path="tools_mock")', + 'local_path_override(module_name="local_config_platform", ', + 'path="platforms_mock")', + ], + ) + self.ScratchFile('vendor/bbb~/foo') + self.RunBazel(['vendor', '--vendor_dir=vendor']) + bbb_module_bazel = os.path.join(vendor_dir, 'bbb~/MODULE.bazel') + self.AssertFileContentContains(bbb_module_bazel, 'version = "2.0"') + foo = os.path.join(vendor_dir, 'bbb~/foo') + self.assertFalse( + os.path.exists(foo) + ) # foo should be removed due to re-vendor def testVendorFailsWithNofetch(self): - self.ScratchFile('MODULE.bazel') + self.ScratchFile( + 'MODULE.bazel', + [ + 'local_path_override(module_name="bazel_tools", path="tools_mock")', + 'local_path_override(module_name="local_config_platform", ', + 'path="platforms_mock")', + ], + ) self.ScratchFile('BUILD') + # We need to fetch first so that it won't fail while creating the initial + # repo mapping because of --nofetch + self.RunBazel(['fetch', '--all']) _, _, stderr = self.RunBazel( ['vendor', '--vendor_dir=vendor', '--nofetch'], allow_failure=True ) - self.assertIn('ERROR: You cannot run vendor with --nofetch', stderr) + self.assertIn( + 'ERROR: You cannot run the vendor command with --nofetch', stderr + ) def testVendoringMultipleTimes(self): self.main_registry.createCcModule('aaa', '1.0') @@ -132,10 +164,7 @@ def testVendoringMultipleTimes(self): _, stdout, _ = self.RunBazel(['info', 'output_base']) repo_path = stdout[0] + '/external/aaa~' - if self.IsWindows(): - self.assertTrue(self.IsJunction(repo_path)) - else: - self.assertTrue(os.path.islink(repo_path)) + self.AssertPathIsSymlink(repo_path) def testVendorRepo(self): self.main_registry.createCcModule('aaa', '1.0').createCcModule( @@ -248,38 +277,9 @@ def testVendorDirIsNotCheckedForWorkspaceRepos(self): "Vendored repository 'dummyRepo' is out-of-date.", '\n'.join(stderr) ) - def testBuildingWithVendoredRepos(self): - self.main_registry.createCcModule('aaa', '1.0') - self.ScratchFile( - 'MODULE.bazel', - [ - 'bazel_dep(name = "aaa", version = "1.0")', - ], - ) - self.ScratchFile('BUILD') - self.RunBazel(['vendor', '--vendor_dir=vendor']) - self.assertIn('aaa~', os.listdir(self._test_cwd + '/vendor')) - - # Empty external & build with vendor - self.RunBazel(['clean', '--expunge']) - _, _, stderr = self.RunBazel(['build', '@aaa//:all', '--vendor_dir=vendor']) - self.assertNotIn( - "Vendored repository '_main~ext~justRepo' is out-of-date.", - '\n'.join(stderr), - ) - - # Assert repo aaa in {OUTPUT_BASE}/external is a symlink (junction on - # windows, this validates it was created from vendor and not fetched)= - _, stdout, _ = self.RunBazel(['info', 'output_base']) - repo_path = stdout[0] + '/external/aaa~' - if self.IsWindows(): - self.assertTrue(self.IsJunction(repo_path)) - else: - self.assertTrue(os.path.islink(repo_path)) - def testIgnoreFromVendoring(self): # Repos should be excluded from vendoring: - # 1.Local Repos, 2.Config Repos, 3.Repos declared in .vendorignore file + # 1.Local Repos, 2.Config Repos, 3.Repos declared in VENDOR.bazel file self.main_registry.createCcModule('aaa', '1.0').createCcModule( 'bbb', '1.0', {'aaa': '1.0'} ) @@ -319,24 +319,106 @@ def testIgnoreFromVendoring(self): ) os.makedirs(self._test_cwd + '/vendor', exist_ok=True) - with open(self._test_cwd + '/vendor/.vendorignore', 'w') as f: - f.write('aaa~\n') + with open(self._test_cwd + '/vendor/VENDOR.bazel', 'w') as f: + f.write("ignore('@@_main~ext~regularRepo')\n") self.RunBazel(['vendor', '--vendor_dir=vendor']) repos_vendored = os.listdir(self._test_cwd + '/vendor') - # Assert bbb and the regularRepo are vendored with marker files + # Assert aaa & bbb are vendored with marker files + self.assertIn('aaa~', repos_vendored) self.assertIn('bbb~', repos_vendored) self.assertIn('@bbb~.marker', repos_vendored) - self.assertIn('_main~ext~regularRepo', repos_vendored) - self.assertIn('@_main~ext~regularRepo.marker', repos_vendored) + self.assertIn('@aaa~.marker', repos_vendored) - # Assert aaa (from .vendorignore), local and config repos are not vendored - self.assertNotIn('aaa~', repos_vendored) + # Assert regular repo (from VENDOR.bazel), local and config repos are + # not vendored self.assertNotIn('bazel_tools', repos_vendored) self.assertNotIn('local_config_platform', repos_vendored) self.assertNotIn('_main~ext~localRepo', repos_vendored) self.assertNotIn('_main~ext~configRepo', repos_vendored) + self.assertNotIn('_main~ext~regularRepo', repos_vendored) + + def testBuildingWithPinnedRepo(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'ext = use_extension("extension.bzl", "ext")', + 'use_repo(ext, "venRepo")', + ], + ) + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + '', + 'def _ext_impl(ctx):', + ' repo_rule(name="venRepo")', + 'ext = module_extension(implementation=_ext_impl)', + ], + ) + self.ScratchFile('BUILD') + + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@venRepo']) + self.assertIn('_main~ext~venRepo', os.listdir(self._test_cwd + '/vendor')) + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'IhaveChanged\')")', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + '', + 'def _ext_impl(ctx):', + ' repo_rule(name="venRepo")', + 'ext = module_extension(implementation=_ext_impl)', + ], + ) + + # Pin the repo then build, should build what is under vendor + # directory with no warning + with open(self._test_cwd + '/vendor/VENDOR.bazel', 'w') as f: + f.write("pin('@@_main~ext~venRepo')\n") + _, _, stderr = self.RunBazel( + ['build', '@venRepo//:all', '--vendor_dir=vendor'], + ) + self.assertNotIn( + "Vendored repository '_main~ext~venRepo' is out-of-date.", + '\n'.join(stderr), + ) + self.assertIn( + 'Target @@_main~ext~venRepo//:lala up-to-date (nothing to build)', + stderr, + ) + + # Unpin the repo, clean the cache and assert updates are applied + with open(self._test_cwd + '/vendor/VENDOR.bazel', 'w') as f: + f.write('') + _, _, stderr = self.RunBazel( + ['build', '@venRepo//:all', '--vendor_dir=vendor'], + ) + self.assertIn( + 'Target @@_main~ext~venRepo//:IhaveChanged up-to-date (nothing to' + ' build)', + stderr, + ) + self.assertIn( + "Vendored repository '_main~ext~venRepo' is out-of-date.", + '\n'.join(stderr), + ) + + # Re-vendor & build make sure the repo is successfully updated + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@venRepo']) + _, _, stderr = self.RunBazel( + ['build', '@venRepo//:all', '--vendor_dir=vendor'], + ) + self.assertNotIn( + "Vendored repository '_main~ext~venRepo' is out-of-date.", + '\n'.join(stderr), + ) def testBuildingOutOfDateVendoredRepo(self): self.ScratchFile( @@ -362,7 +444,7 @@ def testBuildingOutOfDateVendoredRepo(self): ) # Vendor, assert and build with no problems - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@justRepo']) self.assertIn('_main~ext~justRepo', os.listdir(self._test_cwd + '/vendor')) _, _, stderr = self.RunBazel( ['build', '@justRepo//:all', '--vendor_dir=vendor'] @@ -370,8 +452,8 @@ def testBuildingOutOfDateVendoredRepo(self): self.assertNotIn( "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' - ' cache and used. To update the repo in the vendor directory, run' - " 'bazel vendor'", + ' cache and used. To update the repo in the vendor directory, run' + ' the bazel vendor command', stderr, ) @@ -400,15 +482,15 @@ def testBuildingOutOfDateVendoredRepo(self): self.assertIn( "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' - ' cache and used. To update the repo in the vendor directory, run' - " 'bazel vendor'", + ' cache and used. To update the repo in the vendor directory, run' + ' the bazel vendor command', stderr, ) _, stdout, _ = self.RunBazel(['info', 'output_base']) self.assertFalse(os.path.islink(stdout[0] + '/external/bbb~1.0')) # Assert vendoring again solves the problem - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@justRepo']) self.RunBazel(['clean', '--expunge']) _, _, stderr = self.RunBazel( ['build', '@justRepo//:all', '--vendor_dir=vendor'] @@ -416,12 +498,12 @@ def testBuildingOutOfDateVendoredRepo(self): self.assertNotIn( "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' - ' cache and used. To update the repo in the vendor directory, run' - " 'bazel vendor'", + ' cache and used. To update the repo in the vendor directory, run' + ' the bazel vendor command', stderr, ) - def testBuildingVendoredRepoInOfflineMode(self): + def testBuildingVendoredRepoWithNoFetch(self): self.ScratchFile( 'MODULE.bazel', [ @@ -445,7 +527,7 @@ def testBuildingVendoredRepoInOfflineMode(self): self.ScratchFile('BUILD') # Vendor, assert and build with no problems - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '@venRepo//:all']) self.assertIn('_main~ext~venRepo', os.listdir(self._test_cwd + '/vendor')) # Make updates in repo definition @@ -479,8 +561,8 @@ def testBuildingVendoredRepoInOfflineMode(self): ) self.assertIn( 'ERROR: Vendored repository _main~ext~noVenRepo not found under the' - " vendor directory and fetching is disabled. To fix run 'bazel" - " vendor' or build without the '--nofetch'", + ' vendor directory and fetching is disabled. To fix, run the bazel' + " vendor command or build without the '--nofetch'", stderr, ) @@ -492,7 +574,7 @@ def testBuildingVendoredRepoInOfflineMode(self): self.assertIn( "WARNING: : Vendored repository '_main~ext~venRepo' is" ' out-of-date and fetching is disabled. Run build without the' - " '--nofetch' option or run `bazel vendor` to update it", + " '--nofetch' option or run the bazel vendor command to update it", stderr, ) # Assert the out-dated repo is the one built with @@ -501,6 +583,198 @@ def testBuildingVendoredRepoInOfflineMode(self): stderr, ) + def testBasicVendorTarget(self): + self.main_registry.createCcModule('aaa', '1.0').createCcModule( + 'bbb', '1.0' + ).createCcModule('ccc', '1.0') + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "aaa", version = "1.0")', + 'bazel_dep(name = "bbb", version = "1.0")', + 'bazel_dep(name = "ccc", version = "1.0")', + ], + ) + self.ScratchFile('BUILD') + + self.RunBazel( + ['vendor', '@aaa//:lib_aaa', '@bbb//:lib_bbb', '--vendor_dir=vendor'] + ) + # Assert aaa & bbb and are vendored + self.assertIn('aaa~', os.listdir(self._test_cwd + '/vendor')) + self.assertIn('bbb~', os.listdir(self._test_cwd + '/vendor')) + self.assertNotIn('ccc~', os.listdir(self._test_cwd + '/vendor')) + + def testBuildVendoredTargetOffline(self): + self.main_registry.createCcModule('aaa', '1.0').createCcModule( + 'bbb', '1.0', {'aaa': '1.0'} + ) + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "bbb", version = "1.0")', + ], + ) + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = [', + ' "@bbb//:lib_bbb",', + ' ],', + ')', + ], + ) + self.ScratchFile( + 'main.cc', + [ + '#include "bbb.h"', + 'int main() {', + ' hello_bbb("Hello there!");', + '}', + ], + ) + + self.RunBazel(['vendor', '//:main', '--vendor_dir=vendor']) + + # Build and run the target in a clean build with internet blocked and make + # sure it works + _, _, _ = self.RunBazel(['clean', '--expunge']) + _, stdout, _ = self.RunBazel( + ['run', '//:main', '--vendor_dir=vendor', '--repository_cache='], + env_add={ + 'HTTP_PROXY': 'internet_blocked', + 'HTTPS_PROXY': 'internet_blocked', + }, + ) + self.assertIn('Hello there! => bbb@1.0', stdout) + + # Assert repos in {OUTPUT_BASE}/external are symlinks (junction on + # windows, this validates it was created from vendor and not fetched) + _, stdout, _ = self.RunBazel(['info', 'output_base']) + for repo in ['aaa~', 'bbb~']: + repo_path = stdout[0] + '/external/' + repo + self.AssertPathIsSymlink(repo_path) + + def testVendorConflictRegistryFile(self): + self.main_registry.createCcModule('aaa', '1.0').createCcModule( + 'bbb', '1.0', {'aaa': '1.0'} + ) + # The registry URLs of main_registry and another_registry only differ by the + # port number + another_registry = BazelRegistry( + os.path.join(self.registries_work_dir, 'MAIN'), + ) + another_registry.start() + another_registry.createCcModule('aaa', '1.0') + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "bbb", version = "1.0")', + 'local_path_override(module_name="bazel_tools", path="tools_mock")', + 'local_path_override(module_name="local_config_platform", ', + 'path="platforms_mock")', + 'single_version_override(', + ' module_name = "aaa",', + ' registry = "%s",' % another_registry.getURL(), + ')', + ], + ) + self.ScratchFile('BUILD') + exit_code, _, stderr = self.RunBazel( + ['vendor', '--vendor_dir=vendor'], allow_failure=True + ) + self.AssertExitCode(exit_code, 8, stderr) + self.assertIn( + 'ERROR: Error while vendoring repos: Vendor paths conflict detected for' + ' registry URLs:', + stderr, + ) + + def testVendorRepoWithSymlinks(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'ext = use_extension("extension.bzl", "ext")', + 'use_repo(ext, "foo", "bar")', + ], + ) + abs_foo = self.ScratchFile('abs', ['Hello from abs!']).replace('\\', '/') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_foo_impl(ctx):', + ' ctx.file("REPO.bazel")', + ' ctx.file("data", "Hello from foo!\\n")', + # Symlink to an absolute path outside of external root + f' ctx.symlink("{abs_foo}", "sym_abs")', + # Symlink to a file in the same repo + ' ctx.symlink("data", "sym_foo")', + # Symlink to a file in another repo + ' ctx.symlink(ctx.path(Label("@bar//:data")), "sym_bar")', + # Symlink to a directory in another repo + ' ctx.symlink("../_main~ext~bar/pkg", "sym_pkg")', + ( + ' ctx.file("BUILD", "exports_files([\'sym_abs\',' + " 'sym_foo','sym_bar', 'sym_pkg/data'])\")" + ), + 'repo_foo = repository_rule(implementation=_repo_foo_impl)', + '', + 'def _repo_bar_impl(ctx):', + ' ctx.file("REPO.bazel")', + ' ctx.file("data", "Hello from bar!\\n")', + ' ctx.file("pkg/data", "Hello from pkg bar!\\n")', + ' ctx.file("BUILD", "exports_files([\'data\'])")', + 'repo_bar = repository_rule(implementation=_repo_bar_impl)', + '', + 'def _ext_impl(ctx):', + ' repo_foo(name="foo")', + ' repo_bar(name="bar")', + 'ext = module_extension(implementation=_ext_impl)', + ], + ) + self.ScratchFile( + 'BUILD', + [ + 'genrule(', + ' name = "print_paths",', + ( + ' srcs = ["@foo//:sym_abs", "@foo//:sym_foo",' + ' "@foo//:sym_bar", "@foo//:sym_pkg/data"],' + ), + ' outs = ["output.txt"],', + ' cmd = "cat $(SRCS) > $@",', + ')', + ], + ) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@foo']) + self.RunBazel(['clean', '--expunge']) + self.AssertPathIsSymlink(self._test_cwd + '/vendor/bazel-external') + + # Move the vendor directory to a new location and use a new output base, + # it should still work + os.rename(self._test_cwd + '/vendor', self._test_cwd + '/vendor_new') + output_base = tempfile.mkdtemp(dir=self._tests_root) + self.RunBazel([ + f'--output_base={output_base}', + 'build', + '//:print_paths', + '--vendor_dir=vendor_new', + '--verbose_failures', + ]) + _, stdout, _ = self.RunBazel( + [f'--output_base={output_base}', 'info', 'output_base'] + ) + self.AssertPathIsSymlink(stdout[0] + '/external/_main~ext~foo') + output = os.path.join(self._test_cwd, './bazel-bin/output.txt') + self.AssertFileContentContains( + output, + 'Hello from abs!\nHello from foo!\nHello from bar!\nHello from pkg' + ' bar!\n', + ) + if __name__ == '__main__': absltest.main() diff --git a/src/test/py/bazel/test_base.py b/src/test/py/bazel/test_base.py index fc0936b1729612..691817a3cde342 100644 --- a/src/test/py/bazel/test_base.py +++ b/src/test/py/bazel/test_base.py @@ -212,6 +212,15 @@ def AssertFileContentNotContains(self, file_path, entry): if entry in f.read(): self.fail('File "%s" does contain "%s"' % (file_path, entry)) + def AssertPathIsSymlink(self, path): + if self.IsWindows(): + self.assertTrue( + self.IsReparsePoint(path), + "Path '%s' is not a symlink or junction" % path, + ) + else: + self.assertTrue(os.path.islink(path), "Path '%s' is not a symlink" % path) + def CreateWorkspaceWithDefaultRepos(self, path, lines=None): rule_definition = [ 'load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")' @@ -274,8 +283,8 @@ def IsLinux(): """Returns true if the current platform is Linux.""" return sys.platform.startswith('linux') - def IsJunction(self, path): - """Returns whether a folder is a junction or not. Used with Windows folders. + def IsReparsePoint(self, path): + """Returns whether a path is a reparse point (symlink or junction) on Windows. Args: path: string; an absolute path to a folder e.g. "C://foo/bar/aaa" diff --git a/src/test/shell/bazel/bazel_repository_cache_test.sh b/src/test/shell/bazel/bazel_repository_cache_test.sh index 4a15bfc898670a..e86ca1194364a2 100755 --- a/src/test/shell/bazel/bazel_repository_cache_test.sh +++ b/src/test/shell/bazel/bazel_repository_cache_test.sh @@ -192,7 +192,7 @@ function test_fetch() { bazel fetch --repository_cache="$repo_cache_dir" //zoo:breeding-program >& $TEST_log \ || echo "Expected fetch to succeed" - expect_log "All external dependencies for these targets fetched successfully" + expect_log "All external dependencies for the requested targets fetched successfully." } function test_directory_structure() { @@ -231,7 +231,7 @@ function test_fetch_value_with_existing_cache_and_no_network() { bazel fetch --repository_cache="$repo_cache_dir" //zoo:breeding-program >& $TEST_log \ || echo "Expected fetch to succeed" - expect_log "All external dependencies for these targets fetched successfully" + expect_log "All external dependencies for the requested targets fetched successfully." } @@ -249,7 +249,7 @@ function test_load_cached_value() { bazel fetch --repository_cache="$repo_cache_dir" //zoo:breeding-program >& $TEST_log \ || echo "Expected fetch to succeed" - expect_log "All external dependencies for these targets fetched successfully" + expect_log "All external dependencies for the requested targets fetched successfully." } function test_write_cache_without_hash() { @@ -388,7 +388,7 @@ EOF bazel fetch --repository_cache="$repo_cache_dir" @foo//:all >& $TEST_log \ || echo "Expected fetch to succeed" - expect_log "All external dependencies for these targets fetched successfully" + expect_log "All external dependencies for the requested targets fetched successfully." } function test_starlark_download_fail_without_cache() { @@ -441,7 +441,7 @@ EOF bazel fetch --repository_cache="$repo_cache_dir" @foo//:all >& $TEST_log \ || echo "Expected fetch to succeed" - expect_log "All external dependencies for these targets fetched successfully" + expect_log "All external dependencies for the requested targets fetched successfully." } function test_starlark_download_and_extract_fail_without_cache() { diff --git a/src/test/shell/bazel/jdeps_test.sh b/src/test/shell/bazel/jdeps_test.sh index 2776ebee9c4c88..ec58c651d3a114 100755 --- a/src/test/shell/bazel/jdeps_test.sh +++ b/src/test/shell/bazel/jdeps_test.sh @@ -72,7 +72,8 @@ function test_jdeps() { # src/test/shell/bazel/jdeps_class_denylist.txt. find . -type f -iname \*class | \ grep -vFf "$denylist" | \ - xargs -s 262144 ../jdk/reduced/bin/jdeps --list-reduced-deps --ignore-missing-deps | \ + sort -r | \ + xargs -s 400000 ../jdk/reduced/bin/jdeps --list-reduced-deps --ignore-missing-deps | \ grep -v "unnamed module" > ../jdeps \ || fail "Failed to run jdeps on non denylisted class files." cd ..