Replace all globs in IREE core with enforce_glob

build_tools/bazel/deep_copy.bzl

@@ -43,6 +43,7 @@ def _deep_copy_recursion_depth_1(x):
 
 def deep_copy(x):
     """Returns a copy of the argument, making a deep copy if it is a container.
+
     Args:
       x: (object) value to copy. If it is a container with nested containers as
          elements, the maximum nesting depth is restricted to three (e.g.,

build_tools/bazel/enforce_glob.bzl

@@ -0,0 +1,50 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A utility to enforce that a list matches a glob expression.
+
+We use this primarily to enable the error-checking capabilities of globs of test
+files in IREE while still allowing our Bazel to CMake conversion to not create
+CMake globs (which are discouraged for collecting source files, see
+https://cmake.org/cmake/help/latest/command/file.html#glob) and not be dependent
+on any information outside of the BUILD file.
+"""
+
+def enforce_glob(files, **kwargs):
+    """A utility to enforce that a list matches a glob expression.
+
+    Note that the comparison is done in an order-independent fashion.
+
+    Args:
+        files: a list that is expected to contain the same files as the
+            specified glob expression.
+        **kwargs: keyword arguments forwarded to the glob.
+
+    Returns:
+        files. The input argument unchanged
+    """
+    glob_result = native.glob(**kwargs)
+
+    # glob returns a sorted list.
+    if sorted(files) != glob_result:
+        glob_result_dict = {k: None for k in glob_result}
+        result_dict = {k: None for k in files}
+        missing = [k for k in glob_result if k not in files]
+        extra = [k for k in files if k not in glob_result]
+        fail(("Error in enforce_glob." +
+              "\nExpected {}." +
+              "\nGot {}." +
+              "\nMissing {}." +
+              "\nExtra {}").format(glob_result, files, missing, extra))
+    return files

build_tools/bazel_to_cmake/bazel_to_cmake_converter.py

@@ -251,6 +251,9 @@ def filegroup(self, name, **kwargs):
   def sh_binary(self, name, **kwargs):
     self._convert_unimplemented_function("sh_binary", name)
 
+  def enforce_glob(self, files, **kwargs):
+    return files
+
   def glob(self, include, exclude=None, exclude_directories=1):
     if exclude_directories != 1:
       self._convert_unimplemented_function("glob", "with exclude_directories")

integrations/tensorflow/iree_tf_compiler/TF/test/BUILD

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 load("@iree//iree:lit_test.bzl", "iree_lit_test_suite")
+load("@iree//build_tools/bazel:enforce_glob.bzl", "enforce_glob")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -22,7 +23,18 @@ package(
 
 iree_lit_test_suite(
     name = "lit",
-    srcs = glob(["*.mlir"]),
+    srcs = enforce_glob(
+        [
+            "convert_to_mhlo.mlir",
+            "lower_global_tensors.mlir",
+            "lower_global_tensors_complex.mlir",
+            "lower_global_tensors_invalid.mlir",
+            "propagate_resource_casts.mlir",
+            "strip_metadata.mlir",
+            "verify_fully_converted.mlir",
+        ],
+        include = ["*.mlir"],
+    ),
     data = [
         "//iree_tf_compiler:iree-tf-opt",
         "@iree//iree/tools:IreeFileCheck",

integrations/tensorflow/iree_tf_compiler/TFL/test/BUILD

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 load("@iree//iree:lit_test.bzl", "iree_lit_test_suite")
+load("@iree//build_tools/bazel:enforce_glob.bzl", "enforce_glob")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -22,7 +23,14 @@ package(
 
 iree_lit_test_suite(
     name = "lit",
-    srcs = glob(["*.mlir"]),
+    srcs = enforce_glob(
+        [
+            "convert_metadata.mlir",
+            "strip_metadata.mlir",
+            "verify_fully_converted.mlir",
+        ],
+        include = ["*.mlir"],
+    ),
     data = [
         "//iree_tf_compiler:iree-opt-tflite",
         "@iree//iree/tools:IreeFileCheck",

integrations/tensorflow/iree_tf_compiler/TFL/test/import/BUILD

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 load("@iree//iree:lit_test.bzl", "iree_lit_test_suite")
+load("@iree//build_tools/bazel:enforce_glob.bzl", "enforce_glob")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -22,7 +23,13 @@ package(
 
 iree_lit_test_suite(
     name = "lit",
-    srcs = glob(["*.mlir"]),
+    srcs = enforce_glob(
+        [
+            "add.mlir",
+            "multi_add.mlir",
+        ],
+        include = ["*.mlir"],
+    ),
     data = glob(["*.tflite"]) + [
         "//iree_tf_compiler:iree-import-tflite",
         "@iree//iree/tools:IreeFileCheck",

integrations/tensorflow/iree_tf_compiler/dialect/tf_strings/conversion/test/BUILD

@@ -15,6 +15,7 @@
 # Tests for lowering MLIR in various dialects to IREE interpreter bytecode.
 
 load("@iree//iree:lit_test.bzl", "iree_lit_test_suite")
+load("@iree//build_tools/bazel:enforce_glob.bzl", "enforce_glob")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -24,7 +25,13 @@ package(
 
 iree_lit_test_suite(
     name = "lit",
-    srcs = glob(["*.mlir"]),
+    srcs = enforce_glob(
+        [
+            "tf_strings_to_strings.mlir",
+            "tf_to_tf_strings.mlir",
+        ],
+        include = ["*.mlir"],
+    ),
     data = [
         "//iree_tf_compiler:iree-tf-opt",
         "@iree//iree/tools:IreeFileCheck",

integrations/tensorflow/iree_tf_compiler/dialect/tf_strings/ir/BUILD

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 load("@iree//build_tools/bazel:tblgen.bzl", "gentbl")
+load("@iree//build_tools/bazel:enforce_glob.bzl", "enforce_glob")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -22,7 +23,13 @@ package(
 
 filegroup(
     name = "td_files",
-    srcs = glob(["*.td"]),
+    srcs = enforce_glob(
+        [
+            "base.td",
+            "ops.td",
+        ],
+        include = ["*.td"],
+    ),
 )
 
 gentbl(

integrations/tensorflow/iree_tf_compiler/dialect/tf_tensorlist/conversion/test/BUILD

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 load("@iree//iree:lit_test.bzl", "iree_lit_test_suite")
+load("@iree//build_tools/bazel:enforce_glob.bzl", "enforce_glob")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -22,7 +23,13 @@ package(
 
 iree_lit_test_suite(
     name = "lit",
-    srcs = glob(["*.mlir"]),
+    srcs = enforce_glob(
+        [
+            "convert_tf_tensorlist_to_tensorlist.mlir",
+            "convert_tf_to_tf_tensorlist.mlir",
+        ],
+        include = ["*.mlir"],
+    ),
     data = [
         "//iree_tf_compiler:iree-tf-opt",
         "@iree//iree/tools:IreeFileCheck",

integrations/tensorflow/iree_tf_compiler/dialect/tf_tensorlist/ir/BUILD

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 load("@iree//build_tools/bazel:tblgen.bzl", "gentbl")
+load("@iree//build_tools/bazel:enforce_glob.bzl", "enforce_glob")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -24,7 +25,13 @@ exports_files(["tf_tensorlist_base.td"])
 
 filegroup(
     name = "td_files",
-    srcs = glob(["*.td"]),
+    srcs = enforce_glob(
+        [
+            "tf_tensorlist_base.td",
+            "tf_tensorlist_ops.td",
+        ],
+        include = ["*.td"],
+    ),
 )
 
 cc_library(

integrations/tensorflow/iree_tf_compiler/dialect/tf_tensorlist/ir/test/BUILD

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 load("@iree//iree:lit_test.bzl", "iree_lit_test_suite")
+load("@iree//build_tools/bazel:enforce_glob.bzl", "enforce_glob")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -22,7 +23,10 @@ package(
 
 iree_lit_test_suite(
     name = "lit",
-    srcs = glob(["*.mlir"]),
+    srcs = enforce_glob(
+        ["ops.mlir"],
+        include = ["*.mlir"],
+    ),
     data = [
         "//iree_tf_compiler:iree-tf-opt",
         "@iree//iree/tools:IreeFileCheck",

iree/compiler/Bindings/TFLite/Transforms/test/BUILD

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 load("//iree:lit_test.bzl", "iree_lit_test_suite")
+load("//build_tools/bazel:enforce_glob.bzl", "enforce_glob")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -22,7 +23,13 @@ package(
 
 iree_lit_test_suite(
     name = "lit",
-    srcs = glob(["*.mlir"]),
+    srcs = enforce_glob(
+        [
+            "materialize_shape_support.mlir",
+            "wrap_entry_points.mlir",
+        ],
+        include = ["*.mlir"],
+    ),
     data = [
         "//iree/tools:IreeFileCheck",
         "//iree/tools:iree-opt",

iree/compiler/Bindings/TFLite/Transforms/test/CMakeLists.txt

@@ -10,12 +10,12 @@
 
 iree_add_all_subdirs()
 
-file(GLOB _GLOB_X_MLIR LIST_DIRECTORIES false RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS *.mlir)
 iree_lit_test_suite(
   NAME
     lit
   SRCS
-    "${_GLOB_X_MLIR}"
+    "materialize_shape_support.mlir"
+    "wrap_entry_points.mlir"
   DATA
     iree::tools::IreeFileCheck
     iree::tools::iree-opt

iree/compiler/Conversion/Common/test/BUILD

@@ -15,6 +15,7 @@
 # Tests for common transforms.
 
 load("//iree:lit_test.bzl", "iree_lit_test_suite")
+load("//build_tools/bazel:enforce_glob.bzl", "enforce_glob")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -24,7 +25,14 @@ package(
 
 iree_lit_test_suite(
     name = "lit",
-    srcs = glob(["*.mlir"]),
+    srcs = enforce_glob(
+        [
+            "linalg_bufferize.mlir",
+            "linalg_rewrite_destructive_updates.mlir",
+            "remove_dead_allocs.mlir",
+        ],
+        include = ["*.mlir"],
+    ),
     data = [
         "//iree/tools:IreeFileCheck",
         "//iree/tools:iree-opt",

iree/compiler/Conversion/Common/test/CMakeLists.txt

@@ -10,12 +10,13 @@
 
 iree_add_all_subdirs()
 
-file(GLOB _GLOB_X_MLIR LIST_DIRECTORIES false RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS *.mlir)
 iree_lit_test_suite(
   NAME
     lit
   SRCS
-    "${_GLOB_X_MLIR}"
+    "linalg_bufferize.mlir"
+    "linalg_rewrite_destructive_updates.mlir"
+    "remove_dead_allocs.mlir"
   DATA
     iree::tools::IreeFileCheck
     iree::tools::iree-opt

iree/compiler/Conversion/HLOToHLO/test/BUILD

@@ -15,6 +15,7 @@
 # Tests for common transforms.
 
 load("//iree:lit_test.bzl", "iree_lit_test_suite")
+load("//build_tools/bazel:enforce_glob.bzl", "enforce_glob")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -24,7 +25,13 @@ package(
 
 iree_lit_test_suite(
     name = "lit",
-    srcs = glob(["*.mlir"]),
+    srcs = enforce_glob(
+        [
+            "conv1x12dot.mlir",
+            "f32Tof16.mlir",
+        ],
+        include = ["*.mlir"],
+    ),
     data = [
         "//iree/tools:IreeFileCheck",
         "//iree/tools:iree-opt",

iree/compiler/Conversion/HLOToHLO/test/CMakeLists.txt

@@ -10,12 +10,12 @@
 
 iree_add_all_subdirs()
 
-file(GLOB _GLOB_X_MLIR LIST_DIRECTORIES false RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS *.mlir)
 iree_lit_test_suite(
   NAME
     lit
   SRCS
-    "${_GLOB_X_MLIR}"
+    "conv1x12dot.mlir"
+    "f32Tof16.mlir"
   DATA
     iree::tools::IreeFileCheck
     iree::tools::iree-opt