Refactor integration testing

test/apply_fixes/execution_logic.py

@@ -17,48 +17,30 @@
 TEST_CASES_DIR = Path("test/apply_fixes")
 TEST_WORKSPACES_DIR = TEST_CASES_DIR / "workspaces"
 
-WORKSPACE_FILE_TEMPLATE = """
-local_repository(
-    name = "depend_on_what_you_use",
-    path = "{dwyu_path}",
-)
-
-load("@depend_on_what_you_use//:setup_step_1.bzl", "setup_step_1")
-setup_step_1()
-
-load("@depend_on_what_you_use//:setup_step_2.bzl", "setup_step_2")
-setup_step_2()
-
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+MODULE_FILE_TEMPLATE = """
+bazel_dep(name = "depend_on_what_you_use")
+local_path_override(module_name = "depend_on_what_you_use", path = "{dwyu_path}")
 
 #
-# One can use DWYU without the content below. Those are dependencies to enable testing.
+# Setup dependencies for test orchestration
 #
 
-http_archive(
-    name = "bazel_skylib",
-    sha256 = "9f38886a40548c6e96c106b752f242130ee11aaa068a56ba7e56f4511f33e4f2",
-    urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/1.6.1/bazel-skylib-1.6.1.tar.gz"],
-)
-
-maybe(
-    http_archive,
-    name = "rules_python",
-    sha256 = "e85ae30de33625a63eca7fc40a94fea845e641888e52f32b6beea91e8b1b2793",
-    strip_prefix = "rules_python-0.27.1",
-    urls = ["https://github.com/bazelbuild/rules_python/releases/download/0.27.1/rules_python-0.27.1.tar.gz"],
-)
-
-load("@rules_python//python:repositories.bzl", "python_register_toolchains")
-python_register_toolchains(
-    name = "python",
-    python_version = "3.8",
-)
+bazel_dep(name = "bazel_skylib", version = "1.6.1")
+
+bazel_dep(name = "rules_python", version = "0.27.1")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(python_version = "3.8")
 
 {extra_content}
 """
 
+BAZEL_RC_FILE = """
+common --nolegacy_external_runfiles
+common --lockfile_mode=off
+"""
+
+BAZEL_VERSION = "7.0.0"
+
 
 def dwyu_path_as_string(dwyu_path: Path) -> str:
     """
@@ -71,16 +53,16 @@ def setup_test_workspace(
     origin_workspace: Path, test_sources: Path, extra_workspace_file_content: str, temporary_workspace: Path
 ) -> None:
     copytree(src=test_sources, dst=str(temporary_workspace), dirs_exist_ok=True)
-    with temporary_workspace.joinpath("WORKSPACE").open(mode="w", encoding="utf-8") as ws_file:
+    with temporary_workspace.joinpath("MODULE.bazel").open(mode="w", encoding="utf-8") as ws_file:
         ws_file.write(
-            WORKSPACE_FILE_TEMPLATE.format(
+            MODULE_FILE_TEMPLATE.format(
                 dwyu_path=dwyu_path_as_string(origin_workspace), extra_content=extra_workspace_file_content
             )
         )
     with temporary_workspace.joinpath(".bazelversion").open(mode="w", encoding="utf-8") as ws_file:
-        ws_file.write("7.0.0")
+        ws_file.write(BAZEL_VERSION)
     with temporary_workspace.joinpath(".bazelrc").open(mode="w", encoding="utf-8") as ws_file:
-        ws_file.write("common --nolegacy_external_runfiles\ncommon --noenable_bzlmod")
+        ws_file.write(BAZEL_RC_FILE)
 
 
 def cleanup(test_workspace: Path) -> None:

test/apply_fixes/missing_dependency/test_find_external_dependency.py

@@ -9,7 +9,10 @@ def test_target(self) -> str:
 
     @property
     def extra_workspace_file_content(self) -> str:
-        return 'load("//:load_external_dep.bzl", "load_external_dep")\nload_external_dep()'
+        return """
+bazel_dep(name = "external_dep")
+local_path_override(module_name = "external_dep", path = "external_dep")
+"""
 
     def execute_test_logic(self) -> Result:
         self._create_reports()

test/apply_fixes/missing_dependency/workspace/external_dep/MODULE.bazel

@@ -0,0 +1 @@
+module(name = "external_dep")

test/aspect/MODULE.bazel

@@ -1,6 +1,6 @@
 module(name = "dwyu_aspect_integration_tests")
 
-bazel_dep(name = "depend_on_what_you_use", dev_dependency = True)
+bazel_dep(name = "depend_on_what_you_use")
 local_path_override(
     module_name = "depend_on_what_you_use",
     path = "../../",
@@ -10,26 +10,16 @@ local_path_override(
 ## Python toolchains for testing compatibility to various Python versions
 ##
 
-# Keep in sync with //test/aspect/WORKSPACE
-# TODO version >= 0.30.0 allows choosing the toolchain without caring for the patch version. Unfortunately, this is not
-# compatible to using WORKSPACE and Bazel 5.x
-bazel_dep(name = "rules_python", version = "0.27.1", dev_dependency = True)
+# We use a newer rules_python version than the one DWYU depends on to profit from the ability to select a toolchain
+# without specifying the patch version, which is possible from >= 0.30.0 onward.
+# Furthermore, this allows ensuring that we are compatible to new rules_python versions. The most crucial parts of DWYU
+# do not depend on rules_python and therefore we are confident in the example integration tests and unit tests finding
+# all issues related to the rules_python version DWYU depends on.
+bazel_dep(name = "rules_python", version = "0.32.2")
 
-# Update with each rules_python update for easy lookup until specifying the patch version for toolchain is possible again
-# Keep in syc with execute_tests.py
-# PATCH_MAPPING:
-#  "3.8": "3.8.18"
-#  "3.9": "3.9.18"
-#  "3.10": "3.10.13"
-#  "3.11": "3.11.6"
-#  "3.12": "3.12.0"
-# Choose different version via: --@rules_python//python/config_settings:python_version=Major.Minor.Patch mathing one of
-# the registered toolchains below.
-python = use_extension(
-    "@rules_python//python/extensions:python.bzl",
-    "python",
-    dev_dependency = True,
-)
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+
+# Choose different version via: --@rules_python//python/config_settings:python_version=X
 python.toolchain(
     is_default = True,
     python_version = "3.8",
@@ -43,19 +33,19 @@ python.toolchain(python_version = "3.12")
 ## Workspaces for test purposes
 ##
 
-bazel_dep(name = "external_test_repo", dev_dependency = True)
+bazel_dep(name = "external_test_repo")
 local_path_override(
     module_name = "external_test_repo",
     path = "external_repo/repo",
 )
 
-bazel_dep(name = "complex_includes_test_repo", dev_dependency = True)
+bazel_dep(name = "complex_includes_test_repo")
 local_path_override(
     module_name = "complex_includes_test_repo",
     path = "complex_includes/ext_repo",
 )
 
-bazel_dep(name = "skip_external_deps_test_repo", dev_dependency = True)
+bazel_dep(name = "skip_external_deps_test_repo")
 local_path_override(
     module_name = "skip_external_deps_test_repo",
     path = "skip_external_targets/external_dep",
@@ -68,4 +58,4 @@ local_path_override(
 ## Reference https://github.com/bazelbuild/bazel/blob/master/src/MODULE.tools
 ##
 
-bazel_dep(name = "platforms", version = "0.0.7", dev_dependency = True)
+bazel_dep(name = "platforms", version = "0.0.7")

test/aspect/WORKSPACE

@@ -16,18 +16,8 @@ setup_step_2()
 ##
 ## Python toolchains for testing compatibility to various Python versions
 ##
-
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
-
-# Keep in sync with //test/aspect/MODULE.bazel
-maybe(
-    http_archive,
-    name = "rules_python",
-    sha256 = "e85ae30de33625a63eca7fc40a94fea845e641888e52f32b6beea91e8b1b2793",
-    strip_prefix = "rules_python-0.27.1",
-    urls = ["https://github.com/bazelbuild/rules_python/releases/download/0.27.1/rules_python-0.27.1.tar.gz"],
-)
+## We rely on rules_python provided by DWYU as dependency, since we we do not profit from using a custom version.
+##
 
 load("@rules_python//python:repositories.bzl", "python_register_multi_toolchains")
 

test/aspect/execute_tests.py

@@ -12,13 +12,13 @@
 
 # Test matrix. We don't combine each Bazel version with each Python version as there is no significant benefit. We
 # manually define pairs which make sure each Bazel and Python version we care about is used at least once.
-# Keep in sync with MODULE.bazel
+# For versions using the legacy WORKSPACE setup we have to specify the patch version for Python
 TESTED_VERSIONS = [
     TestedVersions(bazel="5.4.1", python="3.8.18"),
     TestedVersions(bazel="6.5.0", python="3.9.18"),
-    TestedVersions(bazel="7.0.0", python="3.10.13"),
-    TestedVersions(bazel="7.1.1", python="3.11.6"),
-    TestedVersions(bazel="8.0.0-pre.20240401.3", python="3.12.0"),
+    TestedVersions(bazel="7.0.0", python="3.10"),
+    TestedVersions(bazel="7.1.1", python="3.11", is_default=True),
+    TestedVersions(bazel="8.0.0-pre.20240516.1", python="3.12"),
 ]
 
 VERSION_SPECIFIC_ARGS = {
@@ -35,7 +35,7 @@
     "--incompatible_enforce_config_setting_visibility": CompatibleVersions(minimum="5.0.0"),
     "--incompatible_config_setting_private_default_visibility": CompatibleVersions(minimum="5.0.0"),
     "--incompatible_disable_target_provider_fields": CompatibleVersions(minimum="5.0.0"),
-    "--incompatible_struct_has_no_methods": CompatibleVersions(minimum="5.0.0"),
+    "--incompatible_struct_has_no_methods": CompatibleVersions(minimum="5.0.0", before="8.0.0"),
     "--incompatible_use_platforms_repo_for_constraints": CompatibleVersions(minimum="5.0.0", before="7.0.0"),
     "--incompatible_disallow_empty_glob": CompatibleVersions(minimum="5.0.0"),
     "--incompatible_no_implicit_file_export": CompatibleVersions(minimum="5.0.0"),
@@ -49,12 +49,10 @@
     "--incompatible_check_visibility_for_toolchains": CompatibleVersions(minimum="7.0.0"),
     "--incompatible_auto_exec_groups": CompatibleVersions(minimum="7.0.0"),
     "--incompatible_disable_non_executable_java_binary": CompatibleVersions(minimum="7.0.0"),
+    "--incompatible_python_disallow_native_rules": CompatibleVersions(minimum="7.0.0"),
+    "--incompatible_disallow_struct_provider_syntax": CompatibleVersions(minimum="7.0.0"),
     # Theoretically of interest for us, but rules_python does not comply to this.
     # "--incompatible_stop_exporting_language_modules": CompatibleVersions(minimum="6.0.0"),
-    # Theoretically interesting for our project, but Bazel itself does not adhere to it
-    # "--incompatible_python_disallow_native_rules": CompatibleVersions(minimum="7.0.0"),
-    # Theoretically of interest for us, but rules_python does not comply to this.
-    # "--incompatible_disallow_struct_provider_syntax": CompatibleVersions(minimum="7.0.0"),
 }
 
 
@@ -74,6 +72,12 @@ def cli() -> Namespace:
         help="Run tests with the specified Python version."
         " Has to be one of the versions for which we register a hermetic toolchain. Also requires setting '--bazel'.",
     )
+    parser.add_argument(
+        "--only-default-version",
+        "-d",
+        action="store_true",
+        help="Execute tests only for the default Bazel and Python version.",
+    )
     parser.add_argument("--list", "-l", action="store_true", help="List all available test cases and return.")
     parser.add_argument(
         "--test",
@@ -106,5 +110,6 @@ def cli() -> Namespace:
             python=args.python,
             requested_tests=args.test,
             list_tests=args.list,
+            only_default_version=args.only_default_version,
         )
     )

test/aspect/execution_logic.py

@@ -54,6 +54,7 @@ def main(
     python: str | None = None,
     requested_tests: list[str] | None = None,
     list_tests: bool = False,
+    only_default_version: bool = False,
 ) -> int:
     workspace_path = get_current_workspace()
     test_files = sorted([Path(x) for x in workspace_path.glob("*/test_*.py")])
@@ -71,7 +72,12 @@ def main(
             module = SourceFileLoader("", str(test.resolve())).load_module()
             tests.append(module.TestCase(name))
 
-    versions = [TestedVersions(bazel=bazel, python=python)] if bazel and python else tested_versions
+    if bazel and python:
+        versions = [TestedVersions(bazel=bazel, python=python)]
+    elif only_default_version:
+        versions = [version for version in tested_versions if version.is_default]
+    else:
+        versions = tested_versions
 
     failed_tests = []
     output_root = Path.home() / ".cache" / "bazel" / "dwyu"

test/aspect/version.py

@@ -5,6 +5,7 @@
 class TestedVersions:
     bazel: str
     python: str
+    is_default: bool = False
 
 
 @dataclass