diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 5c4f4adbab73bf..4b45705309b979 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -15,6 +15,7 @@ java_library( srcs = [ "ArchiveRepoSpecBuilder.java", "AttributeValues.java", + "ModuleFile.java", "ModuleKey.java", "RepoSpec.java", "Version.java", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index 46be96ffef45f4..55ed3eacdb165f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -88,12 +88,12 @@ private Optional grabFile(String url, ExtendedEventHandler eventHandler) } @Override - public Optional getModuleFile(ModuleKey key, ExtendedEventHandler eventHandler) + public Optional getModuleFile(ModuleKey key, ExtendedEventHandler eventHandler) throws IOException, InterruptedException { - return grabFile( + String url = constructUrl( - getUrl(), "modules", key.getName(), key.getVersion().toString(), "MODULE.bazel"), - eventHandler); + getUrl(), "modules", key.getName(), key.getVersion().toString(), "MODULE.bazel"); + return grabFile(url, eventHandler).map(content -> ModuleFile.create(content, url)); } /** Represents fields available in {@code bazel_registry.json} for the registry. */ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFile.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFile.java new file mode 100644 index 00000000000000..797bed387066a3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFile.java @@ -0,0 +1,33 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; + +/** A MODULE.bazel file's content and location. */ +@AutoValue +public abstract class ModuleFile { + /** The raw content of the module file. */ + @SuppressWarnings("mutable") + public abstract byte[] getContent(); + + /** The user-facing location of the module file, e.g. a file system path or URL. */ + public abstract String getLocation(); + + public static ModuleFile create(byte[] content, String location) { + return new AutoValue_ModuleFile(content, location); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java index b8406793a7ffd9..f9c1e0022e2c85 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java @@ -110,11 +110,11 @@ public SkyValue compute(SkyKey skyKey, Environment env) return null; } String moduleFileHash = - new Fingerprint().addBytes(getModuleFileResult.moduleFileContents).hexDigestAndReset(); + new Fingerprint().addBytes(getModuleFileResult.moduleFile.getContent()).hexDigestAndReset(); ModuleFileGlobals moduleFileGlobals = execModuleFile( - getModuleFileResult.moduleFileContents, + getModuleFileResult.moduleFile, getModuleFileResult.registry, moduleKey, // Dev dependencies should always be ignored if the current module isn't the root module @@ -151,8 +151,8 @@ private SkyValue computeForRootModule(StarlarkSemantics starlarkSemantics, Envir if (env.getValue(FileValue.key(moduleFilePath)) == null) { return null; } - byte[] moduleFile = readFile(moduleFilePath.asPath()); - String moduleFileHash = new Fingerprint().addBytes(moduleFile).hexDigestAndReset(); + ModuleFile moduleFile = readModuleFile(moduleFilePath.asPath()); + String moduleFileHash = new Fingerprint().addBytes(moduleFile.getContent()).hexDigestAndReset(); ModuleFileGlobals moduleFileGlobals = execModuleFile( moduleFile, @@ -189,7 +189,7 @@ private SkyValue computeForRootModule(StarlarkSemantics starlarkSemantics, Envir } private ModuleFileGlobals execModuleFile( - byte[] moduleFile, + ModuleFile moduleFile, @Nullable Registry registry, ModuleKey moduleKey, boolean ignoreDevDeps, @@ -197,7 +197,7 @@ private ModuleFileGlobals execModuleFile( Environment env) throws ModuleFileFunctionException, InterruptedException { StarlarkFile starlarkFile = - StarlarkFile.parse(ParserInput.fromUTF8(moduleFile, moduleKey + "/MODULE.bazel")); + StarlarkFile.parse(ParserInput.fromUTF8(moduleFile.getContent(), moduleFile.getLocation())); if (!starlarkFile.ok()) { Event.replayEventsOn(env.getListener(), starlarkFile.errors()); throw errorf(Code.BAD_MODULE, "error parsing MODULE.bazel file for %s", moduleKey); @@ -224,7 +224,7 @@ private ModuleFileGlobals execModuleFile( } private static class GetModuleFileResult { - byte[] moduleFileContents; + ModuleFile moduleFile; // `registry` can be null if this module has a non-registry override. @Nullable Registry registry; } @@ -249,7 +249,7 @@ private GetModuleFileResult getModuleFile( return null; } GetModuleFileResult result = new GetModuleFileResult(); - result.moduleFileContents = readFile(moduleFilePath.asPath()); + result.moduleFile = readModuleFile(moduleFilePath.asPath()); return result; } @@ -285,11 +285,11 @@ private GetModuleFileResult getModuleFile( GetModuleFileResult result = new GetModuleFileResult(); for (Registry registry : registryObjects) { try { - Optional moduleFile = registry.getModuleFile(key, env.getListener()); - if (!moduleFile.isPresent()) { + Optional moduleFile = registry.getModuleFile(key, env.getListener()); + if (moduleFile.isEmpty()) { continue; } - result.moduleFileContents = moduleFile.get(); + result.moduleFile = moduleFile.get(); result.registry = registry; return result; } catch (IOException e) { @@ -301,9 +301,10 @@ private GetModuleFileResult getModuleFile( throw errorf(Code.MODULE_NOT_FOUND, "module not found in registries: %s", key); } - private static byte[] readFile(Path path) throws ModuleFileFunctionException { + private static ModuleFile readModuleFile(Path path) throws ModuleFileFunctionException { try { - return FileSystemUtils.readWithKnownFileSize(path, path.getFileSize()); + return ModuleFile.create( + FileSystemUtils.readWithKnownFileSize(path, path.getFileSize()), path.getPathString()); } catch (IOException e) { throw errorf(Code.MODULE_NOT_FOUND, "MODULE.bazel expected but not found at %s", path); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Registry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Registry.java index 547bba7c2b2c5a..19c54d884051fa 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Registry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Registry.java @@ -31,7 +31,7 @@ public interface Registry { * Retrieves the contents of the module file of the module identified by {@code key} from the * registry. Returns {@code Optional.empty()} when the module is not found in this registry. */ - Optional getModuleFile(ModuleKey key, ExtendedEventHandler eventHandler) + Optional getModuleFile(ModuleKey key, ExtendedEventHandler eventHandler) throws IOException, InterruptedException; /** diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java index 593a3c5484da7a..814d7b09e49274 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java @@ -63,8 +63,16 @@ public String getUrl() { } @Override - public Optional getModuleFile(ModuleKey key, ExtendedEventHandler eventHandler) { - return Optional.ofNullable(modules.get(key)).map(value -> value.getBytes(UTF_8)); + public Optional getModuleFile(ModuleKey key, ExtendedEventHandler eventHandler) { + return Optional.ofNullable(modules.get(key)) + .map(value -> value.getBytes(UTF_8)) + .map( + content -> + ModuleFile.create( + content, + String.format( + "%s/modules/%s/%s/MODULE.bazel", + url, key.getName(), key.getVersion().toString()))); } @Override diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java index d18e81248c2afc..98a8d28b165e18 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java @@ -72,7 +72,9 @@ public void testHttpUrl() throws Exception { Registry registry = registryFactory.getRegistryWithUrl(server.getUrl() + "/myreg"); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) - .hasValue("lol".getBytes(UTF_8)); + .hasValue( + ModuleFile.create( + "lol".getBytes(UTF_8), server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel")); assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); } @@ -94,7 +96,9 @@ public void testHttpUrlWithNetrcCreds() throws Exception { downloadManager.setNetrcCreds(new NetrcCredentials(netrc)); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) - .hasValue("lol".getBytes(UTF_8)); + .hasValue( + ModuleFile.create( + "lol".getBytes(UTF_8), server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel")); assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); } @@ -110,7 +114,7 @@ public void testFileUrl() throws Exception { registryFactory.getRegistryWithUrl( new File(tempFolder.getRoot(), "fakereg").toURI().toString()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) - .hasValue("lol".getBytes(UTF_8)); + .hasValue(ModuleFile.create("lol".getBytes(UTF_8), file.toURI().toString())); assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 9e115fe0596632..d94389a6f7ecac 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -1183,7 +1183,7 @@ public void importNonExistentRepo() throws Exception { .contains( "module extension \"ext\" from \"//:defs.bzl\" does not generate repository" + " \"missing_repo\", yet it is imported as \"my_repo\" in the usage at" - + " /MODULE.bazel:1:20"); + + " /ws/MODULE.bazel:1:20"); } @Test @@ -1654,7 +1654,7 @@ public void extensionMetadata() throws Exception { assertEventCount(1, eventCollector); assertContainsEvent( - "WARNING /MODULE.bazel:3:20: The module extension ext defined in @ext//:defs.bzl" + "WARNING /ws/MODULE.bazel:3:20: The module extension ext defined in @ext//:defs.bzl" + " reported incorrect imports of repositories via use_repo():\n" + "\n" + "Imported, but not created by the extension (will cause the build to fail):\n" @@ -1732,11 +1732,11 @@ public void extensionMetadata_all() throws Exception { .isEqualTo( "module extension \"ext\" from \"@ext~1.0//:defs.bzl\" does not generate repository " + "\"invalid_dep\", yet it is imported as \"invalid_dep\" in the usage at " - + "/MODULE.bazel:3:20"); + + "/ws/MODULE.bazel:3:20"); assertEventCount(1, eventCollector); assertContainsEvent( - "WARNING /MODULE.bazel:3:20: The module extension ext defined in @ext//:defs.bzl" + "WARNING /ws/MODULE.bazel:3:20: The module extension ext defined in @ext//:defs.bzl" + " reported incorrect imports of repositories via use_repo():\n" + "\n" + "Imported, but not created by the extension (will cause the build to fail):\n" @@ -1811,11 +1811,11 @@ public void extensionMetadata_allDev() throws Exception { .isEqualTo( "module extension \"ext\" from \"@ext~1.0//:defs.bzl\" does not generate repository " + "\"invalid_dep\", yet it is imported as \"invalid_dep\" in the usage at " - + "/MODULE.bazel:3:20"); + + "/ws/MODULE.bazel:3:20"); assertEventCount(1, eventCollector); assertContainsEvent( - "WARNING /MODULE.bazel:3:20: The module extension ext defined in @ext//:defs.bzl" + "WARNING /ws/MODULE.bazel:3:20: The module extension ext defined in @ext//:defs.bzl" + " reported incorrect imports of repositories via use_repo():\n" + "\n" + "Imported, but not created by the extension (will cause the build to fail):\n" diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java index d654cd8427b83a..1e29e61d276022 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java @@ -477,7 +477,9 @@ public void testModuleExtensions_good() throws Exception { .setExtensionBzlFile("@mymod//:defs.bzl") .setExtensionName("myext1") .setUsingModule(myMod) - .setLocation(Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 2, 23)) + .setLocation( + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 2, 23)) .setImports(ImmutableBiMap.of("repo1", "repo1")) .setDevImports(ImmutableSet.of()) .addTag( @@ -490,7 +492,8 @@ public void testModuleExtensions_good() throws Exception { .buildImmutable())) .setDevDependency(false) .setLocation( - Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 4, 11)) + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 4, 11)) .build()) .build()) .addExtensionUsage( @@ -498,7 +501,9 @@ public void testModuleExtensions_good() throws Exception { .setExtensionBzlFile("@mymod//:defs.bzl") .setExtensionName("myext2") .setUsingModule(myMod) - .setLocation(Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 5, 23)) + .setLocation( + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 5, 23)) .setImports(ImmutableBiMap.of("other_repo1", "repo1", "repo2", "repo2")) .setDevImports(ImmutableSet.of()) .addTag( @@ -511,7 +516,8 @@ public void testModuleExtensions_good() throws Exception { .buildImmutable())) .setDevDependency(false) .setLocation( - Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 7, 12)) + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 7, 12)) .build()) .addTag( Tag.builder() @@ -523,7 +529,8 @@ public void testModuleExtensions_good() throws Exception { .buildImmutable())) .setDevDependency(false) .setLocation( - Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 8, 12)) + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 8, 12)) .build()) .build()) .addExtensionUsage( @@ -531,7 +538,9 @@ public void testModuleExtensions_good() throws Exception { .setExtensionBzlFile("@rules_jvm_external//:defs.bzl") .setExtensionName("maven") .setUsingModule(myMod) - .setLocation(Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 10, 22)) + .setLocation( + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 10, 22)) .setImports( ImmutableBiMap.of("mvn", "maven", "junit", "junit", "guava", "guava")) .setDevImports(ImmutableSet.of()) @@ -545,7 +554,8 @@ public void testModuleExtensions_good() throws Exception { .buildImmutable())) .setDevDependency(false) .setLocation( - Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 12, 10)) + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 12, 10)) .build()) .addTag( Tag.builder() @@ -557,7 +567,8 @@ public void testModuleExtensions_good() throws Exception { .buildImmutable())) .setDevDependency(false) .setLocation( - Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 14, 10)) + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 14, 10)) .build()) .build()) .build()); @@ -597,7 +608,7 @@ public void testModuleExtensions_duplicateProxy_asRoot() throws Exception { .setExtensionBzlFile("@//:defs.bzl") .setExtensionName("myext") .setUsingModule(ModuleKey.ROOT) - .setLocation(Location.fromFileLineColumn("/MODULE.bazel", 1, 23)) + .setLocation(Location.fromFileLineColumn("/workspace/MODULE.bazel", 1, 23)) .setImports( ImmutableBiMap.of( "alpha", "alpha", "beta", "beta", "gamma", "gamma", "delta", @@ -613,7 +624,7 @@ public void testModuleExtensions_duplicateProxy_asRoot() throws Exception { .buildImmutable())) .setDevDependency(true) .setLocation( - Location.fromFileLineColumn("/MODULE.bazel", 2, 11)) + Location.fromFileLineColumn("/workspace/MODULE.bazel", 2, 11)) .build()) .addTag( Tag.builder() @@ -625,7 +636,7 @@ public void testModuleExtensions_duplicateProxy_asRoot() throws Exception { .buildImmutable())) .setDevDependency(false) .setLocation( - Location.fromFileLineColumn("/MODULE.bazel", 5, 11)) + Location.fromFileLineColumn("/workspace/MODULE.bazel", 5, 11)) .build()) .addTag( Tag.builder() @@ -637,7 +648,7 @@ public void testModuleExtensions_duplicateProxy_asRoot() throws Exception { .buildImmutable())) .setDevDependency(true) .setLocation( - Location.fromFileLineColumn("/MODULE.bazel", 8, 11)) + Location.fromFileLineColumn("/workspace/MODULE.bazel", 8, 11)) .build()) .addTag( Tag.builder() @@ -649,7 +660,7 @@ public void testModuleExtensions_duplicateProxy_asRoot() throws Exception { .buildImmutable())) .setDevDependency(false) .setLocation( - Location.fromFileLineColumn("/MODULE.bazel", 11, 11)) + Location.fromFileLineColumn("/workspace/MODULE.bazel", 11, 11)) .build()) .build()) .build()); @@ -694,7 +705,9 @@ public void testModuleExtensions_duplicateProxy_asDep() throws Exception { .setExtensionBzlFile("@mymod//:defs.bzl") .setExtensionName("myext") .setUsingModule(myMod) - .setLocation(Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 5, 23)) + .setLocation( + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 5, 23)) .setImports(ImmutableBiMap.of("beta", "beta", "delta", "delta")) .setDevImports(ImmutableSet.of()) .addTag( @@ -707,7 +720,8 @@ public void testModuleExtensions_duplicateProxy_asDep() throws Exception { .buildImmutable())) .setDevDependency(false) .setLocation( - Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 6, 11)) + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 6, 11)) .build()) .addTag( Tag.builder() @@ -719,7 +733,8 @@ public void testModuleExtensions_duplicateProxy_asDep() throws Exception { .buildImmutable())) .setDevDependency(false) .setLocation( - Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 12, 11)) + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 12, 11)) .build()) .build()) .build()); diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py index 3568ea699e7863..f9fe557bbd1851 100644 --- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -169,8 +169,9 @@ def testChangeFlagWithLockfile(self): allow_failure=True, ) self.AssertExitCode(exit_code, 48, stderr) - self.assertIn( - "ERROR: sss@1.3/MODULE.bazel:1:9: invalid character: '!'", stderr + self.assertRegex( + '\n'.join(stderr), + "ERROR: .*/sss/1.3/MODULE.bazel:1:9: invalid character: '!'", ) def testLockfileErrorMode(self):