Implement ignoring directories based on wildcards.

src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkGlobalsImpl.java

@@ -23,7 +23,7 @@
 import com.google.devtools.build.lib.collect.nestedset.Depset;
 import com.google.devtools.build.lib.packages.BuildGlobals;
 import com.google.devtools.build.lib.packages.Proto;
-import com.google.devtools.build.lib.packages.RepoCallable;
+import com.google.devtools.build.lib.packages.RepoFileGlobals;
 import com.google.devtools.build.lib.packages.SelectorList;
 import com.google.devtools.build.lib.packages.StarlarkGlobals;
 import com.google.devtools.build.lib.packages.StarlarkNativeModule;
@@ -133,7 +133,7 @@ public ImmutableMap<String, Object> getModuleToplevels() {
   @Override
   public ImmutableMap<String, Object> getRepoToplevels() {
     ImmutableMap.Builder<String, Object> env = ImmutableMap.builder();
-    Starlark.addMethods(env, RepoCallable.INSTANCE);
+    Starlark.addMethods(env, RepoFileGlobals.INSTANCE);
     return env.buildOrThrow();
   }
 

src/main/java/com/google/devtools/build/lib/cmdline/BUILD

@@ -51,6 +51,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
         "//src/main/java/com/google/devtools/build/lib/util:hash_codes",
         "//src/main/java/com/google/devtools/build/lib/util:string",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:ospathpolicy",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",

src/main/java/com/google/devtools/build/lib/cmdline/IgnoredSubdirectories.java

@@ -14,57 +14,132 @@
 
 package com.google.devtools.build.lib.cmdline;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.skyframe.serialization.DeserializationContext;
+import com.google.devtools.build.lib.skyframe.serialization.ObjectCodec;
+import com.google.devtools.build.lib.skyframe.serialization.SerializationContext;
+import com.google.devtools.build.lib.skyframe.serialization.SerializationException;
 import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.CodedOutputStream;
+import java.io.IOException;
 import java.util.Objects;
 import javax.annotation.Nullable;
 
 /**
  * A set of subdirectories to ignore during target pattern matching or globbing.
- *
- * <p>This is currently just a prefix, but will eventually support glob-style wildcards.
  */
 public final class IgnoredSubdirectories {
-  public static final IgnoredSubdirectories EMPTY = new IgnoredSubdirectories(ImmutableSet.of());
+  public static final IgnoredSubdirectories EMPTY =
+      new IgnoredSubdirectories(ImmutableSet.of(), ImmutableList.of());
+
+  private static final Splitter SLASH_SPLITTER = Splitter.on("/");
 
   private final ImmutableSet<PathFragment> prefixes;
 
-  private IgnoredSubdirectories(ImmutableSet<PathFragment> prefixes) {
-    for (PathFragment prefix : prefixes) {
-      Preconditions.checkArgument(!prefix.isAbsolute());
+  // String[] is mutable; we keep the split version because that's faster to match and the non-split
+  // one because that allows for simpler equality checking and then matchingEntry() doesn't need to
+  // allocate new objects.
+  private final ImmutableList<String> patterns;
+  private final ImmutableList<String[]> splitPatterns;
+
+  private static class Codec implements ObjectCodec<IgnoredSubdirectories> {
+    private static final Codec INSTANCE = new Codec();
+
+    @Override
+    public Class<? extends IgnoredSubdirectories> getEncodedClass() {
+      return IgnoredSubdirectories.class;
+    }
+
+    @Override
+    public void serialize(
+        SerializationContext context, IgnoredSubdirectories obj, CodedOutputStream codedOut)
+        throws SerializationException, IOException {
+      context.serialize(obj.prefixes, codedOut);
+      context.serialize(obj.patterns, codedOut);
     }
+
+    @Override
+    public IgnoredSubdirectories deserialize(
+        DeserializationContext context, CodedInputStream codedIn)
+        throws SerializationException, IOException {
+      ImmutableSet<PathFragment> prefixes = context.deserialize(codedIn);
+      ImmutableList<String> patterns = context.deserialize(codedIn);
+
+      return new IgnoredSubdirectories(prefixes, patterns);
+    }
+  }
+
+  private IgnoredSubdirectories(
+      ImmutableSet<PathFragment> prefixes, ImmutableList<String> patterns) {
     this.prefixes = prefixes;
+    this.patterns = patterns;
+    this.splitPatterns =
+        patterns.stream()
+            .map(p -> Iterables.toArray(SLASH_SPLITTER.split(p), String.class))
+            .collect(toImmutableList());
   }
 
   public static IgnoredSubdirectories of(ImmutableSet<PathFragment> prefixes) {
-    if (prefixes.isEmpty()) {
+    return of(prefixes, ImmutableList.of());
+  }
+
+  public static IgnoredSubdirectories of(
+      ImmutableSet<PathFragment> prefixes, ImmutableList<String> patterns) {
+    if (prefixes.isEmpty() && patterns.isEmpty()) {
       return EMPTY;
-    } else {
-      return new IgnoredSubdirectories(prefixes);
     }
+
+    for (PathFragment prefix : prefixes) {
+      Preconditions.checkArgument(!prefix.isAbsolute());
+    }
+
+    return new IgnoredSubdirectories(prefixes, patterns);
   }
 
   public IgnoredSubdirectories withPrefix(PathFragment prefix) {
-    ImmutableSet<PathFragment> prefixed =
+    Preconditions.checkArgument(!prefix.isAbsolute());
+
+    ImmutableSet<PathFragment> prefixedPrefixes =
         prefixes.stream().map(prefix::getRelative).collect(toImmutableSet());
-    return new IgnoredSubdirectories(prefixed);
+
+    ImmutableList<String> prefixedPatterns =
+        patterns.stream().map(p -> prefix + "/" + p).collect(toImmutableList());
+
+    return new IgnoredSubdirectories(prefixedPrefixes, prefixedPatterns);
   }
 
   public IgnoredSubdirectories union(IgnoredSubdirectories other) {
     return new IgnoredSubdirectories(
-        ImmutableSet.<PathFragment>builder().addAll(prefixes).addAll(other.prefixes).build());
+        ImmutableSet.<PathFragment>builder().addAll(prefixes).addAll(other.prefixes).build(),
+        ImmutableList.copyOf(
+            ImmutableSet.<String>builder().addAll(patterns).addAll(other.patterns).build()));
   }
 
   /** Filters out entries that cannot match anything under {@code directory}. */
   public IgnoredSubdirectories filterForDirectory(PathFragment directory) {
     ImmutableSet<PathFragment> filteredPrefixes =
         prefixes.stream().filter(p -> p.startsWith(directory)).collect(toImmutableSet());
 
-    return new IgnoredSubdirectories(filteredPrefixes);
+    String[] splitDirectory = Iterables.toArray(
+        SLASH_SPLITTER.split(directory.getPathString()), String.class);
+    ImmutableList.Builder<String> filteredPatterns = ImmutableList.builder();
+    for (int i = 0; i < patterns.size(); i++) {
+      if (UnixGlob.canMatchChild(splitPatterns.get(i), splitDirectory)) {
+        filteredPatterns.add(patterns.get(i));
+      }
+    }
+
+    return new IgnoredSubdirectories(filteredPrefixes, filteredPatterns.build());
   }
 
   public ImmutableSet<PathFragment> prefixes() {
@@ -95,10 +170,17 @@ public boolean allPathsAreUnder(PathFragment directory) {
 
   /** Returns the entry that matches a given directory or {@code null} if none. */
   @Nullable
-  public PathFragment matchingEntry(PathFragment directory) {
+  public String matchingEntry(PathFragment directory) {
     for (PathFragment prefix : prefixes) {
       if (directory.startsWith(prefix)) {
-        return prefix;
+        return prefix.getPathString();
+      }
+    }
+
+    String[] segmentArray = Iterables.toArray(directory.segments(), String.class);
+    for (int i = 0; i < patterns.size(); i++) {
+      if (UnixGlob.matchesPrefix(splitPatterns.get(i), segmentArray)) {
+        return patterns.get(i);
       }
     }
 
@@ -111,17 +193,22 @@ public boolean equals(Object other) {
       return false;
     }
 
+    // splitPatterns is a function of patterns so it's enough to check if patterns is equal
     IgnoredSubdirectories that = (IgnoredSubdirectories) other;
-    return Objects.equals(this.prefixes, that.prefixes);
+    return Objects.equals(this.prefixes, that.prefixes)
+        && Objects.equals(this.patterns, that.patterns);
   }
 
   @Override
   public int hashCode() {
-    return prefixes.hashCode();
+    return Objects.hash(prefixes, patterns);
   }
 
   @Override
   public String toString() {
-    return MoreObjects.toStringHelper("IgnoredSubdirectories").add("prefixes", prefixes).toString();
+    return MoreObjects.toStringHelper("IgnoredSubdirectories")
+        .add("prefixes", prefixes)
+        .add("patterns", patterns)
+        .toString();
   }
 }

src/main/java/com/google/devtools/build/lib/cmdline/TargetPattern.java

@@ -595,8 +595,7 @@ public <T, E extends Exception & QueryExceptionMarkerInterface> void eval(
           this,
           excludedSubdirectories);
       IgnoredSubdirectories ignoredSubdirectories = ignoredSubdirectoriesSupplier.get();
-      PathFragment matchingEntry =
-          ignoredSubdirectories.matchingEntry(directory.getPackageFragment());
+      String matchingEntry = ignoredSubdirectories.matchingEntry(directory.getPackageFragment());
       if (warnIfFiltered(matchingEntry, resolver)) {
         return;
       }
@@ -632,8 +631,7 @@ ListenableFuture<Void> evalAsync(
       IgnoredSubdirectories filteredIgnoredSubdirectories;
       try {
         IgnoredSubdirectories ignoredSubdirectories = ignoredSubdirectoriesSupplier.get();
-        PathFragment matchingEntry =
-            ignoredSubdirectories.matchingEntry(directory.getPackageFragment());
+        String matchingEntry = ignoredSubdirectories.matchingEntry(directory.getPackageFragment());
         if (warnIfFiltered(matchingEntry, resolver)) {
           return immediateVoidFuture();
         }
@@ -654,7 +652,7 @@ ListenableFuture<Void> evalAsync(
           executor);
     }
 
-    private boolean warnIfFiltered(PathFragment matchingEntry, TargetPatternResolver<?> resolver) {
+    private boolean warnIfFiltered(String matchingEntry, TargetPatternResolver<?> resolver) {
       if (matchingEntry != null) {
         resolver.warn(
             "Pattern '"

src/main/java/com/google/devtools/build/lib/packages/RepoFileGlobals.java

@@ -0,0 +1,81 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import java.util.Map;
+import net.starlark.java.annot.Param;
+import net.starlark.java.annot.ParamType;
+import net.starlark.java.annot.StarlarkMethod;
+import net.starlark.java.eval.EvalException;
+import net.starlark.java.eval.Sequence;
+import net.starlark.java.eval.Starlark;
+import net.starlark.java.eval.StarlarkThread;
+
+/** Definition of the {@code repo()} function used in REPO.bazel files. */
+public final class RepoFileGlobals {
+  private RepoFileGlobals() {}
+
+  public static final RepoFileGlobals INSTANCE = new RepoFileGlobals();
+
+  @StarlarkMethod(
+      name = "ignore_directories",
+      doc = "The list of directories to ignore in this repository. <p>This function takes a list"
+          + " of strings and a directory is ignored if any of the given strings matches its"
+          + " repository-relative path according to the semantics of the <code>glob()</code>"
+          + " function. This function can be used to ignore directories that are implementation"
+          + " details of source control systems, output files of other build systems, etc.",
+      useStarlarkThread = true,
+      parameters = {
+        @Param(
+            name = "dirs",
+            allowedTypes = {
+              @ParamType(type = Sequence.class, generic1 = String.class),
+            })
+      })
+  public void ignoreDirectories(Iterable<?> dirsUnchecked, StarlarkThread thread)
+      throws EvalException {
+    Sequence<String> dirs = Sequence.cast(dirsUnchecked, String.class, "dirs");
+    RepoThreadContext context = RepoThreadContext.fromOrFail(thread, "repo()");
+
+    if (context.isIgnoredDirectoriesSet()) {
+      throw new EvalException("'ignored_directories()' can only be called once");
+    }
+
+    context.setIgnoredDirectories(dirs);
+  }
+
+  @StarlarkMethod(
+      name = "repo",
+      documented = false, // documented separately
+      extraKeywords = @Param(name = "kwargs"),
+      useStarlarkThread = true)
+  public void repoCallable(Map<String, Object> kwargs, StarlarkThread thread)
+      throws EvalException {
+    RepoThreadContext context = RepoThreadContext.fromOrFail(thread, "repo()");
+    if (context.isRepoFunctionCalled()) {
+      throw Starlark.errorf("'repo' can only be called once in the REPO.bazel file");
+    }
+
+    if (context.isIgnoredDirectoriesSet()) {
+      throw Starlark.errorf("if repo() is called, it must be called before any other functions");
+    }
+
+    if (kwargs.isEmpty()) {
+      throw Starlark.errorf("at least one argument must be given to the 'repo' function");
+    }
+
+    context.setPackageArgsMap(kwargs);
+  }
+}