From b48dc31e4203927c0c5cf9ab578dfcbc6dec144d Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Tue, 1 Oct 2024 06:33:20 -0700 Subject: [PATCH 1/3] Automatically ignore vendor dir for finding packages - Adding `--vendor_dir` to ignored package prefixes as if it's added in the `.bazelignore` file. - This allows users to do `bazel build //...` after vendoring external deps in the source root without manually adding the vendor dir in `.bazelignore`. Fixes https://github.com/bazelbuild/bazel/issues/23521 Closes #23771. PiperOrigin-RevId: 680994986 Change-Id: I9c8b76ca20b9060232dfe05460a8fc857c92e93f --- .../google/devtools/build/lib/skyframe/BUILD | 1 + .../IgnoredPackagePrefixesFunction.java | 12 ++++++ .../lib/analysis/util/BuildViewTestCase.java | 4 +- .../packages/util/PackageLoadingTestCase.java | 4 ++ .../lib/pkgcache/IncrementalLoadingTest.java | 5 ++- src/test/py/bazel/bzlmod/bazel_vendor_test.py | 43 +++++++++++++++++++ 6 files changed, 66 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD index 35c48e95f9b22a..9ad032e1becfc0 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD @@ -798,6 +798,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/io:inconsistent_filesystem_exception", "//src/main/java/com/google/devtools/build/lib/pkgcache", "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_directory_value", + "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/skyframe", diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/IgnoredPackagePrefixesFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/IgnoredPackagePrefixesFunction.java index f5ffb25d802ad4..c935316d34a8c8 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/IgnoredPackagePrefixesFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/IgnoredPackagePrefixesFunction.java @@ -13,6 +13,8 @@ // limitations under the License. package com.google.devtools.build.lib.skyframe; +import static com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction.VENDOR_DIRECTORY; + import com.google.common.collect.ImmutableSet; import com.google.common.io.CharStreams; import com.google.common.io.LineProcessor; @@ -82,7 +84,17 @@ public SkyValue compute(SkyKey key, Environment env) } if (repositoryName.isMain()) { + PathFragment vendorDir = null; + if (VENDOR_DIRECTORY.get(env).isPresent()) { + vendorDir = VENDOR_DIRECTORY.get(env).get().asFragment(); + } + for (Root packagePathEntry : pkgLocator.getPathEntries()) { + PathFragment workspaceRoot = packagePathEntry.asPath().asFragment(); + if (vendorDir != null && vendorDir.startsWith(workspaceRoot)) { + ignoredPackagePrefixesBuilder.add(vendorDir.relativeTo(workspaceRoot)); + } + RootedPath rootedPatternFile = RootedPath.toRootedPath(packagePathEntry, ignoredPackagePrefixesFile); FileValue patternFileValue = (FileValue) env.getValue(FileValue.key(rootedPatternFile)); diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java index 01b67f677d395f..8890b3f433ddf7 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java @@ -526,7 +526,9 @@ private void setUpSkyframe() { skyframeExecutor.injectExtraPrecomputedValues( ImmutableList.of( PrecomputedValue.injected( - RepositoryDelegatorFunction.RESOLVED_FILE_INSTEAD_OF_WORKSPACE, Optional.empty()))); + RepositoryDelegatorFunction.RESOLVED_FILE_INSTEAD_OF_WORKSPACE, Optional.empty()), + PrecomputedValue.injected( + RepositoryDelegatorFunction.VENDOR_DIRECTORY, Optional.empty()))); } protected void setPackageOptions(String... options) throws Exception { diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/PackageLoadingTestCase.java b/src/test/java/com/google/devtools/build/lib/packages/util/PackageLoadingTestCase.java index ee66f058002e16..2029dc2e0c6d59 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/util/PackageLoadingTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/packages/util/PackageLoadingTestCase.java @@ -149,6 +149,10 @@ private SkyframeExecutor createSkyframeExecutor() { ImmutableList.of( PrecomputedValue.injected( RepositoryDelegatorFunction.RESOLVED_FILE_INSTEAD_OF_WORKSPACE, Optional.empty()))); + skyframeExecutor.injectExtraPrecomputedValues( + ImmutableList.of( + PrecomputedValue.injected( + RepositoryDelegatorFunction.VENDOR_DIRECTORY, Optional.empty()))); SkyframeExecutorTestHelper.process(skyframeExecutor); return skyframeExecutor; } diff --git a/src/test/java/com/google/devtools/build/lib/pkgcache/IncrementalLoadingTest.java b/src/test/java/com/google/devtools/build/lib/pkgcache/IncrementalLoadingTest.java index c52733ba27cbe5..777cfad5442ec6 100644 --- a/src/test/java/com/google/devtools/build/lib/pkgcache/IncrementalLoadingTest.java +++ b/src/test/java/com/google/devtools/build/lib/pkgcache/IncrementalLoadingTest.java @@ -474,8 +474,9 @@ public DiffAwareness maybeCreate(Root pathEntry, ImmutableSet ignoredPaths skyframeExecutor.injectExtraPrecomputedValues( ImmutableList.of( PrecomputedValue.injected( - RepositoryDelegatorFunction.RESOLVED_FILE_INSTEAD_OF_WORKSPACE, - Optional.empty()))); + RepositoryDelegatorFunction.RESOLVED_FILE_INSTEAD_OF_WORKSPACE, Optional.empty()), + PrecomputedValue.injected( + RepositoryDelegatorFunction.VENDOR_DIRECTORY, Optional.empty()))); skyframeExecutor.preparePackageLoading( new PathPackageLocator( outputBase, diff --git a/src/test/py/bazel/bzlmod/bazel_vendor_test.py b/src/test/py/bazel/bzlmod/bazel_vendor_test.py index 12ee461e124adb..b8ad9535b03779 100644 --- a/src/test/py/bazel/bzlmod/bazel_vendor_test.py +++ b/src/test/py/bazel/bzlmod/bazel_vendor_test.py @@ -671,6 +671,49 @@ def testVendorWithTargetPatternFile(self): self.assertIn('bbb~', os.listdir(self._test_cwd + '/vendor')) self.assertNotIn('ccc~', os.listdir(self._test_cwd + '/vendor')) + def testVendorDirIsIgnored(self): + self.main_registry.createCcModule('aaa', '1.0') + self.ScratchFile( + 'MODULE.bazel', + ['bazel_dep(name = "aaa", version = "1.0")'], + ) + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = [', + ' "@aaa//:lib_aaa",', + ' ],', + ')', + ], + ) + self.ScratchFile( + 'main.cc', + [ + '#include "aaa.h"', + 'int main() {', + ' hello_aaa("Hello there!");', + '}', + ], + ) + + self.RunBazel([ + 'vendor', + '//...', + '--vendor_dir=vendor', + ]) + # Assert aaa is vendored + self.assertIn('aaa+', os.listdir(self._test_cwd + '/vendor')) + + # bazel build //... should succeed because vendor dir is ignored. + self.RunBazel([ + 'build', + '//...', + '--vendor_dir=vendor', + ]) + def testBuildVendoredTargetOffline(self): self.main_registry.createCcModule('aaa', '1.0').createCcModule( 'bbb', '1.0', {'aaa': '1.0'} From 37b830ed33bac099338a14447f47bbae6eaf6291 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Wed, 9 Oct 2024 15:23:34 +0200 Subject: [PATCH 2/3] Fix tests --- .../google/devtools/build/lib/skyframe/GlobFunctionTest.java | 2 ++ src/test/py/bazel/bzlmod/bazel_vendor_test.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/GlobFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/GlobFunctionTest.java index 320b8e810f0073..ed68ca157f4ad9 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/GlobFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/GlobFunctionTest.java @@ -126,6 +126,8 @@ public final void setUp() throws Exception { PrecomputedValue.STARLARK_SEMANTICS.set(differencer, StarlarkSemantics.DEFAULT); RepositoryDelegatorFunction.RESOLVED_FILE_INSTEAD_OF_WORKSPACE.set( differencer, Optional.empty()); + RepositoryDelegatorFunction.VENDOR_DIRECTORY.set( + differencer, Optional.empty()); createTestFiles(); } diff --git a/src/test/py/bazel/bzlmod/bazel_vendor_test.py b/src/test/py/bazel/bzlmod/bazel_vendor_test.py index b8ad9535b03779..35f63e4feaf2f2 100644 --- a/src/test/py/bazel/bzlmod/bazel_vendor_test.py +++ b/src/test/py/bazel/bzlmod/bazel_vendor_test.py @@ -705,7 +705,7 @@ def testVendorDirIsIgnored(self): '--vendor_dir=vendor', ]) # Assert aaa is vendored - self.assertIn('aaa+', os.listdir(self._test_cwd + '/vendor')) + self.assertIn('aaa~', os.listdir(self._test_cwd + '/vendor')) # bazel build //... should succeed because vendor dir is ignored. self.RunBazel([ From 60176ce5038d01299357fb0b2fc497c6f130e844 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Wed, 9 Oct 2024 15:50:41 +0200 Subject: [PATCH 3/3] Add mirror --- bazel_downloader.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bazel_downloader.cfg b/bazel_downloader.cfg index 99a15d355ebec0..95b28a5df5f3c6 100644 --- a/bazel_downloader.cfg +++ b/bazel_downloader.cfg @@ -4,3 +4,5 @@ rewrite (maven.google.com)/(.*) https://mirror.bazel.build/$1/$2 rewrite (maven.google.com)/(.*) https://$1/$2 rewrite (cdn.azul.com)/(.*) https://mirror.bazel.build/$1/$2 rewrite (cdn.azul.com)/(.*) https://$1/$2 +rewrite (aka.ms)/(.*) https://mirror.bazel.build/$1/$2 +rewrite (aka.ms)/(.*) https://$1/$2