From 0e1b8aae660ecaaadbce4e1ba4b5c864398fbafb Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Thu, 20 Jun 2024 19:34:39 +0200 Subject: [PATCH] [7.2.1] Enforce and await cleanup in `StarlarkBaseExternalContext` (#22814) `StarlarkBaseExternalContext` now implements `AutoCloseable` and, in `close()`: 1. Cancels all pending async tasks. 2. Awaits their termination. 3. Cleans up the working directory (always for module extensions, on failure for repo rules). 4. Fails if there were pending async tasks in an otherwise successful evaluation. Previously, module extensions didn't do any of those. Repo rules did 1 and 4 and sometimes 3, but not in all cases. This change required replacing the fixed-size thread pool in `DownloadManager` with virtual threads, thereby resolving a TODO about not using a fixed-size thread pool for the `GrpcRemoteDownloader`. Work towards #22680 Work towards #22748 Closes #22772 PiperOrigin-RevId: 644669599 Change-Id: Ib71e5bf346830b92277ac2bd473e11c834cb2624 Closes #22775 --- .../starlark/StarlarkRepositoryContext.java | 112 ++++++++++++++++++ src/test/py/bazel/bzlmod/bazel_module_test.py | 1 - 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java index 05a98da4e115f7..5e1076de8bf95c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java @@ -448,6 +448,118 @@ public void patch(Object patchFile, StarlarkInt stripI, String watchPatch, Starl } } + @StarlarkMethod( + name = "extract", + doc = "Extract an archive to the repository directory.", + useStarlarkThread = true, + parameters = { + @Param( + name = "archive", + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = Label.class), + @ParamType(type = StarlarkPath.class) + }, + named = true, + doc = + "path to the archive that will be unpacked," + + " relative to the repository directory."), + @Param( + name = "output", + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = Label.class), + @ParamType(type = StarlarkPath.class) + }, + defaultValue = "''", + named = true, + doc = + "path to the directory where the archive will be unpacked," + + " relative to the repository directory."), + @Param( + name = "stripPrefix", + defaultValue = "''", + named = true, + doc = + "a directory prefix to strip from the extracted files." + + "\nMany archives contain a top-level directory that contains all files in the" + + " archive. Instead of needing to specify this prefix over and over in the" + + " build_file, this field can be used to strip it from extracted" + + " files."), + @Param( + name = "rename_files", + defaultValue = "{}", + named = true, + positional = false, + doc = + "An optional dict specifying files to rename during the extraction. Archive entries" + + " with names exactly matching a key will be renamed to the value, prior to" + + " any directory prefix adjustment. This can be used to extract archives that" + + " contain non-Unicode filenames, or which have files that would extract to" + + " the same path on case-insensitive filesystems."), + @Param( + name = "watch_archive", + defaultValue = "'auto'", + positional = false, + named = true, + doc = + "whether to watch the archive file. Can be the string " + + "'yes', 'no', or 'auto'. Passing 'yes' is equivalent to immediately invoking " + + "the watch() method; passing 'no' does " + + "not attempt to watch the file; passing 'auto' will only attempt to watch " + + "the file when it is legal to do so (see watch() docs for more " + + "information."), + }) + public void extract( + Object archive, + Object output, + String stripPrefix, + Dict renameFiles, // expected + String watchArchive, + StarlarkThread thread) + throws RepositoryFunctionException, InterruptedException, EvalException { + StarlarkPath archivePath = getPath("extract()", archive); + + if (!archivePath.exists()) { + throw new RepositoryFunctionException( + Starlark.errorf("Archive path '%s' does not exist.", archivePath), Transience.TRANSIENT); + } + if (archivePath.isDir()) { + throw Starlark.errorf("attempting to extract a directory: %s", archivePath); + } + maybeWatch(archivePath, ShouldWatch.fromString(watchArchive)); + + StarlarkPath outputPath = getPath("extract()", output); + checkInOutputDirectory("write", outputPath); + + Map renameFilesMap = + Dict.cast(renameFiles, String.class, String.class, "rename_files"); + + WorkspaceRuleEvent w = + WorkspaceRuleEvent.newExtractEvent( + archive.toString(), + output.toString(), + stripPrefix, + renameFilesMap, + identifyingStringForLogging, + thread.getCallerLocation()); + env.getListener().post(w); + + env.getListener() + .post( + new ExtractProgress( + outputPath.getPath().toString(), "Extracting " + archivePath.getBasename())); + DecompressorValue.decompress( + DecompressorDescriptor.builder() + .setContext(identifyingStringForLogging) + .setArchivePath(archivePath.getPath()) + .setDestinationPath(outputPath.getPath()) + .setPrefix(stripPrefix) + .setRenameFiles(renameFilesMap) + .build()); + env.getListener().post(new ExtractProgress(outputPath.getPath().toString())); + } + @StarlarkMethod( name = "watch_tree", doc = diff --git a/src/test/py/bazel/bzlmod/bazel_module_test.py b/src/test/py/bazel/bzlmod/bazel_module_test.py index 6cbc5b503e49f5..2d3cf21d2e5e4e 100644 --- a/src/test/py/bazel/bzlmod/bazel_module_test.py +++ b/src/test/py/bazel/bzlmod/bazel_module_test.py @@ -1069,6 +1069,5 @@ def testRegression22754(self): self.ScratchFile('testdata/WORKSPACE') self.RunBazel(['build', ':all']) - if __name__ == '__main__': absltest.main()