repository_ctx: Allow renaming archive entries during extraction.

Adds a `rename_files=` parameter to the `extract()` and `download_and_extract()` methods of `repository_ctx`. This new parameter takes a dict of archive entries to rename, and is applied prior to any prefix adjustment.

Closes bazelbuild#16052.

PiperOrigin-RevId: 469686270
Change-Id: Ia8bc37ba1150ee1ece8ef4f613c26d0f93ca988b

src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleEvent.java

@@ -110,12 +110,19 @@ public static WorkspaceRuleEvent newDownloadEvent(
 
   /** Creates a new WorkspaceRuleEvent for an extract event. */
   public static WorkspaceRuleEvent newExtractEvent(
-      String archive, String output, String stripPrefix, String ruleLabel, Location location) {
+      String archive,
+      String output,
+      String stripPrefix,
+      Map<String, String> renameFiles,
+      String ruleLabel,
+      Location location) {
+
     ExtractEvent e =
         WorkspaceLogProtos.ExtractEvent.newBuilder()
             .setArchive(archive)
             .setOutput(output)
             .setStripPrefix(stripPrefix)
+            .putAllRenameFiles(renameFiles)
             .build();
 
     WorkspaceLogProtos.WorkspaceEvent.Builder result =
@@ -138,6 +145,7 @@ public static WorkspaceRuleEvent newDownloadAndExtractEvent(
       String integrity,
       String type,
       String stripPrefix,
+      Map<String, String> renameFiles,
       String ruleLabel,
       Location location) {
     WorkspaceLogProtos.DownloadAndExtractEvent.Builder e =
@@ -146,7 +154,8 @@ public static WorkspaceRuleEvent newDownloadAndExtractEvent(
             .setSha256(sha256)
             .setIntegrity(integrity)
             .setType(type)
-            .setStripPrefix(stripPrefix);
+            .setStripPrefix(stripPrefix)
+            .putAllRenameFiles(renameFiles);
     for (URL u : urls) {
       e.addUrl(u.toString());
     }

src/main/java/com/google/devtools/build/lib/bazel/debug/workspace_log.proto

@@ -59,6 +59,8 @@ message ExtractEvent {
   string output = 2;
   // A directory prefix to strip from extracted files.
   string strip_prefix = 3;
+  // Files to rename during extraction.
+  map<string, string> rename_files = 4;
 }
 
 message DownloadAndExtractEvent {
@@ -74,6 +76,8 @@ message DownloadAndExtractEvent {
   string strip_prefix = 5;
   // checksum in Subresource Integrity format, if specified
   string integrity = 6;
+  // Files to rename during extraction.
+  map<string, string> rename_files = 7;
 }
 
 // Information on "file" event in repository_ctx.

src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java

@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.Map;
 import org.apache.commons.compress.archivers.ar.ArArchiveEntry;
 import org.apache.commons.compress.archivers.ar.ArArchiveInputStream;
 
@@ -49,11 +50,15 @@ public Path decompress(DecompressorDescriptor descriptor)
       throw new InterruptedException();
     }
 
+    Map<String, String> renameFiles = descriptor.renameFiles();
+
     try (InputStream decompressorStream = getDecompressorStream(descriptor)) {
       ArArchiveInputStream arStream = new ArArchiveInputStream(decompressorStream);
       ArArchiveEntry entry;
       while ((entry = arStream.getNextArEntry()) != null) {
-        Path filePath = descriptor.repositoryPath().getRelative(entry.getName());
+        String entryName = entry.getName();
+        entryName = renameFiles.getOrDefault(entryName, entryName);
+        Path filePath = descriptor.repositoryPath().getRelative(entryName);
         filePath.getParentDirectory().createDirectoryAndParents();
         if (entry.isDirectory()) {
           // ar archives don't contain any directory information, so this should never

src/main/java/com/google/devtools/build/lib/bazel/repository/CompressedTarFunction.java

@@ -47,6 +47,7 @@ public Path decompress(DecompressorDescriptor descriptor)
       throw new InterruptedException();
     }
     Optional<String> prefix = descriptor.prefix();
+    Map<String, String> renameFiles = descriptor.renameFiles();
     boolean foundPrefix = false;
     Set<String> availablePrefixes = new HashSet<>();
     // Store link, target info of symlinks, we create them after regular files are extracted.
@@ -56,7 +57,9 @@ public Path decompress(DecompressorDescriptor descriptor)
       TarArchiveInputStream tarStream = new TarArchiveInputStream(decompressorStream);
       TarArchiveEntry entry;
       while ((entry = tarStream.getNextTarEntry()) != null) {
-        StripPrefixedPath entryPath = StripPrefixedPath.maybeDeprefix(entry.getName(), prefix);
+        String entryName = entry.getName();
+        entryName = renameFiles.getOrDefault(entryName, entryName);
+        StripPrefixedPath entryPath = StripPrefixedPath.maybeDeprefix(entryName, prefix);
         foundPrefix = foundPrefix || entryPath.foundPrefix();
 
         if (prefix.isPresent() && !foundPrefix) {

src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorDescriptor.java

@@ -15,10 +15,13 @@
 package com.google.devtools.build.lib.bazel.repository;
 
 import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor;
 import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import java.util.Collections;
+import java.util.Map;
 import java.util.Objects;
 import javax.annotation.Nullable;
 
@@ -33,17 +36,25 @@ public class DecompressorDescriptor {
   private final Path repositoryPath;
   private final Optional<String> prefix;
   private final boolean executable;
+  private final Map<String, String> renameFiles;
   private final Decompressor decompressor;
 
   private DecompressorDescriptor(
-      String targetKind, String targetName, Path archivePath, Path repositoryPath,
-      @Nullable String prefix, boolean executable, Decompressor decompressor) {
+      String targetKind,
+      String targetName,
+      Path archivePath,
+      Path repositoryPath,
+      @Nullable String prefix,
+      boolean executable,
+      Map<String, String> renameFiles,
+      Decompressor decompressor) {
     this.targetKind = targetKind;
     this.targetName = targetName;
     this.archivePath = archivePath;
     this.repositoryPath = repositoryPath;
     this.prefix = Optional.fromNullable(prefix);
     this.executable = executable;
+    this.renameFiles = renameFiles;
     this.decompressor = decompressor;
   }
 
@@ -71,6 +82,10 @@ public boolean executable() {
     return executable;
   }
 
+  public Map<String, String> renameFiles() {
+    return renameFiles;
+  }
+
   public Decompressor getDecompressor() {
     return decompressor;
   }
@@ -92,12 +107,13 @@ public boolean equals(Object other) {
         && Objects.equals(repositoryPath, descriptor.repositoryPath)
         && Objects.equals(prefix, descriptor.prefix)
         && Objects.equals(executable, descriptor.executable)
+        && Objects.equals(renameFiles, descriptor.renameFiles)
         && decompressor == descriptor.decompressor;
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(targetKind, targetName, archivePath, repositoryPath, prefix);
+    return Objects.hash(targetKind, targetName, archivePath, repositoryPath, prefix, renameFiles);
   }
 
   public static Builder builder() {
@@ -115,6 +131,7 @@ public static class Builder {
     private Path repositoryPath;
     private String prefix;
     private boolean executable;
+    private Map<String, String> renameFiles;
     private Decompressor decompressor;
 
     private Builder() {
@@ -124,8 +141,18 @@ public DecompressorDescriptor build() throws RepositoryFunctionException {
       if (decompressor == null) {
         decompressor = DecompressorValue.getDecompressor(archivePath);
       }
+      if (renameFiles == null) {
+        renameFiles = Collections.emptyMap();
+      }
       return new DecompressorDescriptor(
-          targetKind, targetName, archivePath, repositoryPath, prefix, executable, decompressor);
+          targetKind,
+          targetName,
+          archivePath,
+          repositoryPath,
+          prefix,
+          executable,
+          renameFiles,
+          decompressor);
     }
 
     @CanIgnoreReturnValue
@@ -164,6 +191,12 @@ public Builder setExecutable(boolean executable) {
       return this;
     }
 
+    @CanIgnoreReturnValue
+    public Builder setRenameFiles(Map<String, String> renameFiles) {
+      this.renameFiles = ImmutableMap.copyOf(renameFiles);
+      return this;
+    }
+
     @CanIgnoreReturnValue
     public Builder setDecompressor(Decompressor decompressor) {
       this.decompressor = decompressor;

src/main/java/com/google/devtools/build/lib/bazel/repository/ZipDecompressor.java

@@ -79,14 +79,17 @@ public Path decompress(DecompressorDescriptor descriptor)
       throws IOException, InterruptedException {
     Path destinationDirectory = descriptor.repositoryPath();
     Optional<String> prefix = descriptor.prefix();
+    Map<String, String> renameFiles = descriptor.renameFiles();
     boolean foundPrefix = false;
     // Store link, target info of symlinks, we create them after regular files are extracted.
     Map<Path, PathFragment> symlinks = new HashMap<>();
 
     try (ZipReader reader = new ZipReader(descriptor.archivePath().getPathFile())) {
       Collection<ZipFileEntry> entries = reader.entries();
       for (ZipFileEntry entry : entries) {
-        StripPrefixedPath entryPath = StripPrefixedPath.maybeDeprefix(entry.getName(), prefix);
+        String entryName = entry.getName();
+        entryName = renameFiles.getOrDefault(entryName, entryName);
+        StripPrefixedPath entryPath = StripPrefixedPath.maybeDeprefix(entryName, prefix);
         foundPrefix = foundPrefix || entryPath.foundPrefix();
         if (entryPath.skip()) {
           continue;

src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java

@@ -749,8 +749,24 @@ public StructImpl download(
                     + " archive. Instead of needing to specify this prefix over and over in the"
                     + " <code>build_file</code>, this field can be used to strip it from extracted"
                     + " files."),
+        @Param(
+            name = "rename_files",
+            defaultValue = "{}",
+            named = true,
+            positional = false,
+            doc =
+                "An optional dict specifying files to rename during the extraction. Archive entries"
+                    + " with names exactly matching a key will be renamed to the value, prior to"
+                    + " any directory prefix adjustment. This can be used to extract archives that"
+                    + " contain non-Unicode filenames, or which have files that would extract to"
+                    + " the same path on case-insensitive filesystems."),
       })
-  public void extract(Object archive, Object output, String stripPrefix, StarlarkThread thread)
+  public void extract(
+      Object archive,
+      Object output,
+      String stripPrefix,
+      Dict<?, ?> renameFiles, // <String, String> expected
+      StarlarkThread thread)
       throws RepositoryFunctionException, InterruptedException, EvalException {
     StarlarkPath archivePath = getPath("extract()", archive);
 
@@ -762,11 +778,15 @@ public void extract(Object archive, Object output, String stripPrefix, StarlarkT
     StarlarkPath outputPath = getPath("extract()", output);
     checkInOutputDirectory("write", outputPath);
 
+    Map<String, String> renameFilesMap =
+        Dict.cast(renameFiles, String.class, String.class, "rename_files");
+
     WorkspaceRuleEvent w =
         WorkspaceRuleEvent.newExtractEvent(
             archive.toString(),
             output.toString(),
             stripPrefix,
+            renameFilesMap,
             rule.getLabel().toString(),
             thread.getCallerLocation());
     env.getListener().post(w);
@@ -782,6 +802,7 @@ public void extract(Object archive, Object output, String stripPrefix, StarlarkT
             .setArchivePath(archivePath.getPath())
             .setRepositoryPath(outputPath.getPath())
             .setPrefix(stripPrefix)
+            .setRenameFiles(renameFilesMap)
             .build());
     env.getListener().post(new ExtractProgress(outputPath.getPath().toString()));
   }
@@ -881,6 +902,17 @@ public void extract(Object archive, Object output, String stripPrefix, StarlarkT
                     + " risk to omit the checksum as remote files can change. At best omitting this"
                     + " field will make your build non-hermetic. It is optional to make development"
                     + " easier but should be set before shipping."),
+        @Param(
+            name = "rename_files",
+            defaultValue = "{}",
+            named = true,
+            positional = false,
+            doc =
+                "An optional dict specifying files to rename during the extraction. Archive entries"
+                    + " with names exactly matching a key will be renamed to the value, prior to"
+                    + " any directory prefix adjustment. This can be used to extract archives that"
+                    + " contain non-Unicode filenames, or which have files that would extract to"
+                    + " the same path on case-insensitive filesystems."),
       })
   public StructImpl downloadAndExtract(
       Object url,
@@ -892,6 +924,7 @@ public StructImpl downloadAndExtract(
       String canonicalId,
       Dict<?, ?> auth, // <String, Dict> expected
       String integrity,
+      Dict<?, ?> renameFiles, // <String, String> expected
       StarlarkThread thread)
       throws RepositoryFunctionException, InterruptedException, EvalException {
     Map<URI, Map<String, String>> authHeaders = getAuthHeaders(getAuthContents(auth, "auth"));
@@ -911,6 +944,9 @@ public StructImpl downloadAndExtract(
       checksumValidation = e;
     }
 
+    Map<String, String> renameFilesMap =
+        Dict.cast(renameFiles, String.class, String.class, "rename_files");
+
     WorkspaceRuleEvent w =
         WorkspaceRuleEvent.newDownloadAndExtractEvent(
             urls,
@@ -919,6 +955,7 @@ public StructImpl downloadAndExtract(
             integrity,
             type,
             stripPrefix,
+            renameFilesMap,
             rule.getLabel().toString(),
             thread.getCallerLocation());
 
@@ -977,6 +1014,7 @@ public StructImpl downloadAndExtract(
               .setArchivePath(downloadedPath)
               .setRepositoryPath(outputPath.getPath())
               .setPrefix(stripPrefix)
+              .setRenameFiles(renameFilesMap)
               .build());
       env.getListener().post(new ExtractProgress(outputPath.getPath().toString()));
     }

src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java

@@ -27,6 +27,7 @@
 import com.google.devtools.build.runfiles.Runfiles;
 import java.io.File;
 import java.io.IOException;
+import java.util.HashMap;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -62,6 +63,22 @@ public void testDecompress() throws Exception {
     assertThat(secondFile.isSymbolicLink()).isFalse();
   }
 
+  /**
+   * Test decompressing an ar file, with some entries being renamed during the extraction process.
+   */
+  @Test
+  public void testDecompressWithRenamedFiles() throws Exception {
+    HashMap<String, String> renameFiles = new HashMap<>();
+    renameFiles.put("archived_first.txt", "renamed_file.txt");
+    DecompressorDescriptor.Builder descriptorBuilder =
+        createDescriptorBuilder().setRenameFiles(renameFiles);
+    Path outputDir = decompress(descriptorBuilder);
+
+    assertThat(outputDir.exists()).isTrue();
+    Path renamedFile = outputDir.getRelative("renamed_file.txt");
+    assertThat(renamedFile.exists()).isTrue();
+  }
+
   private Path decompress(DecompressorDescriptor.Builder descriptorBuilder) throws Exception {
     descriptorBuilder.setDecompressor(ArFunction.INSTANCE);
     return new ArFunction().decompress(descriptorBuilder.build());