Improve test coverage on macOS

- Add `--test_tag_filters=-no_macos` for macOS to filter out tests that are not suppose to run on Mac, just like the `no_windows` tag. - Fixed PatchApiBlackBoxTest and GitRepositoryBlackBoxTest on newer macOS versions. - Enable previously disabled tests on arm64 macOS platform in presubmit - Disable slow tests on Intel macOS platform in presubmit since they are often the bottleneck. Tests fixed: - Fixes bazelbuild#16525 - Fixes bazelbuild#17447 - Fixes bazelbuild#17411 - Related: bazelbuild#18776 (fixed on macos_arm64) Closes bazelbuild#21381. PiperOrigin-RevId: 608993052 Change-Id: I03bdf954c358bb736a866ce3a02f6dac4473e47e
meteorcloudy · Feb 21, 2024 · 9f3f465 · 9f3f465
1 parent 70452ff
commit 9f3f465
Show file tree

Hide file tree

Showing 8 changed files with 74 additions and 128 deletions.
diff --git a/.bazelci/postsubmit.yml b/.bazelci/postsubmit.yml
@@ -188,10 +188,6 @@ tasks:
       - "//tools/aquery_differ/..."
       - "//tools/python/..."
       - "//tools/bash/..."
-      # C++ coverage is not supported on macOS yet.
-      - "-//src/test/shell/bazel:bazel_cc_code_coverage_test"
-      # MacOS does not have cgroups so it can't support hardened sandbox
-      - "-//src/test/shell/integration:bazel_hardened_sandboxed_worker_test"
       # https://github.com/bazelbuild/bazel/issues/16526
       - "-//src/test/shell/bazel:starlark_repository_test"
       # https://github.com/bazelbuild/bazel/issues/17407
@@ -200,10 +196,6 @@ tasks:
       - "-//src/test/shell/bazel/apple:bazel_objc_test"
       # https://github.com/bazelbuild/bazel/issues/17410
       - "-//src/test/java/com/google/devtools/build/lib/platform:SystemMemoryPressureEventTest"
-      # https://github.com/bazelbuild/bazel/issues/17411
-      - "-//src/test/java/com/google/devtools/build/lib/blackbox/tests/workspace:PatchApiBlackBoxTest"
-      # https://github.com/bazelbuild/bazel/issues/17447
-      - "-//src/test/java/com/google/devtools/build/lib/blackbox/tests/workspace:GitRepositoryBlackBoxTest"
       # https://github.com/bazelbuild/bazel/issues/17456
       - "-//src/test/shell/bazel:bazel_determinism_test"
       # https://github.com/bazelbuild/bazel/issues/17457
@@ -246,34 +238,10 @@ tasks:
       - "//tools/aquery_differ/..."
       - "//tools/python/..."
       - "//tools/bash/..."
-      # C++ coverage is not supported on macOS yet.
-      - "-//src/test/shell/bazel:bazel_cc_code_coverage_test"
-      # MacOS does not have cgroups so it can't support hardened sandbox
-      - "-//src/test/shell/integration:bazel_hardened_sandboxed_worker_test"
-      # https://github.com/bazelbuild/bazel/issues/16525
-      - "-//src/test/java/com/google/devtools/build/lib/buildtool:KeepGoingTest"
-      - "-//src/test/java/com/google/devtools/build/lib/buildtool:DanglingSymlinkTest"
-      - "-//src/test/java/com/google/devtools/build/lib/buildtool:CompileOneDependencyIntegrationTest"
-      - "-//src/test/java/com/google/devtools/build/lib/buildtool:SkymeldBuildIntegrationTest"
-      - "-//src/test/java/com/google/devtools/build/lib/rules/objc:BazelJ2ObjcLibraryTest"
-      - "-//src/test/java/com/google/devtools/build/lib/skyframe/rewinding:RewindingTest"
-      - "-//src/test/java/com/google/devtools/build/lib/buildtool:MiscAnalysisTest"
-      - "-//src/test/java/com/google/devtools/build/lib/rules/objc:ObjcRulesTests"
-      # https://github.com/bazelbuild/bazel/issues/17007
-      - "-//src/test/java/com/google/devtools/build/lib/platform:SystemMemoryPressureEventTest"
       # https://github.com/bazelbuild/bazel/issues/16521 & https://github.com/bazelbuild/bazel/issues/18776
       - "-//src/test/shell/bazel/android/..."
       - "-//src/tools/android/java/com/google/devtools/build/android/..."
       - "-//src/test/java/com/google/devtools/build/android/dexer:AllTests"
-      # Macs can't find python, so these fail: https://github.com/bazelbuild/bazel/issues/18776
-      - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test"
-      - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test_with_head_android_tools"
-      - "-//src/test/shell/bazel/android:aapt_integration_test"
-      - "-//src/test/shell/bazel/android:aapt_integration_test_with_head_android_tools"
-      # https://github.com/bazelbuild/bazel/issues/17411
-      - "-//src/test/java/com/google/devtools/build/lib/blackbox/tests/workspace:PatchApiBlackBoxTest"
-      # https://github.com/bazelbuild/bazel/issues/17447
-      - "-//src/test/java/com/google/devtools/build/lib/blackbox/tests/workspace:GitRepositoryBlackBoxTest"
     include_json_profile:
       - build
       - test

diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
@@ -191,10 +191,6 @@ tasks:
       - "//tools/aquery_differ/..."
       - "//tools/python/..."
       - "//tools/bash/..."
-      # C++ coverage is not supported on macOS yet.
-      - "-//src/test/shell/bazel:bazel_cc_code_coverage_test"
-      # MacOS does not have cgroups so it can't support hardened sandbox
-      - "-//src/test/shell/integration:bazel_hardened_sandboxed_worker_test"
       # https://github.com/bazelbuild/bazel/issues/16526
       - "-//src/test/shell/bazel:starlark_repository_test"
       # https://github.com/bazelbuild/bazel/issues/17407
@@ -204,10 +200,6 @@ tasks:
       # https://github.com/bazelbuild/bazel/issues/17410
       - "-//src/test/java/com/google/devtools/build/lib/platform:SystemMemoryPressureEventTest"
       # https://github.com/bazelbuild/bazel/issues/17411
-      - "-//src/test/java/com/google/devtools/build/lib/blackbox/tests/workspace:PatchApiBlackBoxTest"
-      # https://github.com/bazelbuild/bazel/issues/17447
-      - "-//src/test/java/com/google/devtools/build/lib/blackbox/tests/workspace:GitRepositoryBlackBoxTest"
-      # https://github.com/bazelbuild/bazel/issues/17456
       - "-//src/test/shell/bazel:bazel_determinism_test"
       # https://github.com/bazelbuild/bazel/issues/17457
       - "-//src/test/shell/bazel:jdeps_test"
@@ -216,6 +208,60 @@ tasks:
       - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test_with_head_android_tools"
       - "-//src/test/shell/bazel/android:aapt_integration_test"
       - "-//src/test/shell/bazel/android:aapt_integration_test_with_head_android_tools"
+      # Disable the top 50 most time-consuming tests on macOS Intel platform in presubmit.
+      # Those tests are still covered in postsubmit and by macOS arm64 platform in presubmit.
+      # To run any of the following test in presubmit, just comment out the corresponding line.
+      # TODO(pcloudy): Disable the android tests after enabling them on Apple Silicon platform.
+      - "-//src/test/shell/bazel:bazel_bootstrap_distfile_test"
+      - "-//src/test/shell/bazel:bazel_bootstrap_distfile_tar_test"
+      # - "-//src/test/shell/bazel/android:android_integration_test"
+      # - "-//src/test/shell/bazel/android:android_integration_test_with_head_android_tools"
+      - "-//src/test/shell/bazel:bazel_proto_library_test"
+      - "-//src/test/py/bazel:runfiles_test"
+      - "-//src/test/shell/integration:loading_phase_tests"
+      - "-//src/test/py/bazel:bazel_module_test"
+      - "-//src/test/shell/integration:target_compatible_with_test"
+      - "-//src/test/shell/integration:bazel_json_worker_test"
+      - "-//src/test/shell/bazel:bazel_coverage_java_jdk17_toolchain_head_test"
+      - "-//src/test/shell/bazel:bazel_java_test_jdk11_toolchain_head"
+      # - "-//src/test/shell/bazel/android:aar_integration_test"
+      - "-//src/test/shell/bazel:bazel_coverage_java_test"
+      - "-//src/test/shell/bazel:bazel_coverage_java_jdk11_toolchain_head_test"
+      - "-//src/test/shell/bazel:bazel_java_test_jdk17_toolchain_head"
+      - "-//src/test/shell/bazel:starlark_git_repository_test"
+      - "-//src/test/shell/integration:build_event_stream_test"
+      - "-//third_party/ijar/test:zip_test"
+      - "-//src/test/java/com/google/devtools/build/lib/rules/java:JavaStarlarkApiTest"
+      - "-//src/test/shell/bazel:bazel_java_tools_test"
+      - "-//src/test/shell/bazel/remote:build_without_the_bytes_test"
+      - "-//src/test/shell/integration:bazel_sandboxed_worker_test"
+      - "-//src/test/shell/integration:client_test"
+      - "-//src/test/shell/bazel/remote:remote_execution_test"
+      - "-//src/test/shell/bazel:bazel_coverage_java_jdk17_toolchain_released_test"
+      - "-//src/test/py/bazel:bazel_vendor_test"
+      - "-//src/test/shell/integration:sandboxing_test"
+      - "-//src/test/shell/bazel:bazel_java_test"
+      - "-//src/test/py/bazel:bzlmod_query_test"
+      # - "-//src/test/shell/bazel/android:desugarer_integration_test"
+      # - "-//src/test/shell/bazel/android:android_local_test_integration_test"
+      # - "-//src/test/shell/bazel/android:desugarer_integration_test_with_head_android_tools"
+      - "-//src/test/py/bazel:bazel_external_repository_test"
+      # - "-//src/test/shell/bazel/android:resource_processing_integration_test"
+      - "-//src/test/shell/bazel:bazel_coverage_java_jdk11_toolchain_released_test"
+      # - "-//src/test/shell/bazel/android:aar_integration_test_with_head_android_tools"
+      # - "-//src/test/shell/bazel/android:android_local_test_integration_test_with_head_android_tools"
+      - "-//src/tools/singlejar:zip64_test"
+      - "-//src/test/py/bazel:launcher_test"
+      - "-//src/test/shell/integration:bazel_worker_test"
+      - "-//src/test/py/bazel:py_test"
+      - "-//src/test/shell/bazel:bazel_test_test"
+      - "-//src/test/py/bazel:bazel_overrides_test"
+      - "-//src/test/shell/bazel:path_mapping_test"
+      - "-//src/test/shell/integration:toolchain_test"
+      - "-//src/test/shell/integration:execution_phase_tests"
+      # - "-//src/test/shell/bazel/android:resource_processing_integration_test_with_head_android_tools"
+      - "-//src/test/shell/integration:aquery_test"
+      - "-//src/test/py/bazel:mod_command_test"
     include_json_profile:
       - build
       - test
@@ -250,87 +296,10 @@ tasks:
       - "//tools/aquery_differ/..."
       - "//tools/python/..."
       - "//tools/bash/..."
-      # C++ coverage is not supported on macOS yet.
-      - "-//src/test/shell/bazel:bazel_cc_code_coverage_test"
-      # MacOS does not have cgroups so it can't support hardened sandbox
-      - "-//src/test/shell/integration:bazel_hardened_sandboxed_worker_test"
-      # https://github.com/bazelbuild/bazel/issues/16525
-      - "-//src/test/java/com/google/devtools/build/lib/buildtool:KeepGoingTest"
-      - "-//src/test/java/com/google/devtools/build/lib/buildtool:DanglingSymlinkTest"
-      - "-//src/test/java/com/google/devtools/build/lib/buildtool:CompileOneDependencyIntegrationTest"
-      - "-//src/test/java/com/google/devtools/build/lib/buildtool:SkymeldBuildIntegrationTest"
-      - "-//src/test/java/com/google/devtools/build/lib/rules/objc:BazelJ2ObjcLibraryTest"
-      - "-//src/test/java/com/google/devtools/build/lib/skyframe/rewinding:RewindingTest"
-      - "-//src/test/java/com/google/devtools/build/lib/buildtool:MiscAnalysisTest"
-      - "-//src/test/java/com/google/devtools/build/lib/rules/objc:ObjcRulesTests"
-      # https://github.com/bazelbuild/bazel/issues/17007
-      - "-//src/test/java/com/google/devtools/build/lib/platform:SystemMemoryPressureEventTest"
-      # Disable the most time-consuming tests on macOS arm64 platform in presubmit.
-      # To run any of the following test in presubmit, just comment out the corresponding line.
-      # TODO(pcloudy): Re-enable when Apple Silicon machine waiting time is no longer the major
-      # bottleneck for presubmit.
-      - "-//src/test/shell/bazel:bazel_determinism_test"
-      - "-//src/test/shell/bazel:jdeps_test"
-      - "-//src/test/shell/bazel:starlark_git_repository_test"
-      - "-//src/test/shell/bazel:bazel_bootstrap_distfile_tar_test"
-      - "-//src/test/shell/integration:bazel_json_worker_test"
-      - "-//src/test/py/bazel:runfiles_test"
-      - "-//src/test/py/bazel:launcher_test"
-      - "-//src/test/py/bazel:bazel_module_test"
-      - "-//src/test/shell/bazel:bazel_java_test_jdk11_toolchain_head"
-      - "-//src/test/shell/integration:target_compatible_with_test"
-      - "-//src/test/py/bazel:py_test"
-      - "-//src/test/py/bazel:bzlmod_query_test"
-      - "-//src/test/py/bazel:cc_import_test"
-      - "-//src/test/shell/bazel/remote:build_without_the_bytes_test"
-      - "-//src/test/shell/bazel:bazel_coverage_java_test"
-      - "-//src/test/py/bazel:bazel_yanked_versions_test"
-      - "-//src/test/shell/bazel:bazel_coverage_java_jdk17_toolchain_head_test"
-      - "-//src/test/shell/bazel:bazel_proto_library_test"
-      - "-//src/test/shell/bazel:bazel_java_tools_test"
-      - "-//src/test/shell/integration:build_event_stream_test"
-      - "-//src/test/py/bazel:bazel_overrides_test"
-      - "-//src/test/shell/bazel:cc_integration_test"
-      - "-//src/test/shell/bazel:bazel_java_test"
-      - "-//src/test/java/com/google/devtools/build/lib/rules/config:ConfigRulesTests"
-      - "-//src/test/shell/bazel:bazel_java_test_jdk17_toolchain_head"
-      - "-//src/test/shell/integration:bazel_worker_test"
-      - "-//src/test/shell/bazel:tags_propagation_native_test"
-      - "-//src/test/shell/bazel:bazel_coverage_java_jdk11_toolchain_head_test"
-      - "-//src/test/shell/integration:bazel_sandboxed_worker_test"
-      - "-//src/test/py/bazel:bazel_repo_mapping_test"
-      - "-//src/test/shell/bazel:bazel_coverage_java_jdk11_toolchain_released_test"
-      - "-//src/test/py/bazel:bazel_workspace_test"
-      - "-//src/test/shell/bazel:execroot_test"
-      - "-//src/test/shell/bazel:starlark_repository_test"
-      - "-//src/test/shell/bazel:bazel_coverage_java_jdk17_toolchain_released_test"
-      - "-//src/test/shell/bazel/remote:remote_execution_test"
-      - "-//src/test/shell/integration:toolchain_test"
-      - "-//src/test/shell/bazel:bazel_test_test"
-      - "-//src/test/shell/integration:test_test"
-      - "-//src/test/shell/bazel:bazel_execlog_test"
-      - "-//src/test/shell/integration:modify_execution_info_test"
-      - "-//src/test/shell/bazel:external_integration_test"
-      - "-//src/test/py/bazel:bazel_external_repository_test"
-      - "-//src/test/shell/bazel:bazel_workspaces_test"
-      - "-//src/test/shell/integration:client_test"
-      - "-//src/test/shell/bazel:workspace_resolved_test"
-      - "-//src/test/shell/bazel:bazel_repository_cache_test"
-      - "-//src/test/shell/integration:aquery_test"
-      - "-//src/test/shell/integration:py_args_escaping_test"
       # https://github.com/bazelbuild/bazel/issues/16521 & https://github.com/bazelbuild/bazel/issues/18776
       - "-//src/test/shell/bazel/android/..."
       - "-//src/tools/android/java/com/google/devtools/build/android/..."
       - "-//src/test/java/com/google/devtools/build/android/dexer:AllTests"
-      # Macs can't find python, so these fail: https://github.com/bazelbuild/bazel/issues/18776
-      - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test"
-      - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test_with_head_android_tools"
-      - "-//src/test/shell/bazel/android:aapt_integration_test"
-      - "-//src/test/shell/bazel/android:aapt_integration_test_with_head_android_tools"
-      # https://github.com/bazelbuild/bazel/issues/17411
-      - "-//src/test/java/com/google/devtools/build/lib/blackbox/tests/workspace:PatchApiBlackBoxTest"
-      # https://github.com/bazelbuild/bazel/issues/17447
-      - "-//src/test/java/com/google/devtools/build/lib/blackbox/tests/workspace:GitRepositoryBlackBoxTest"
     include_json_profile:
       - build
       - test

diff --git a/.bazelrc b/.bazelrc
@@ -95,6 +95,7 @@ test:ci-macos --test_env=REPOSITORY_CACHE=/Users/buildkite/bazeltest/repo_cache
 test:ci-macos --test_env=REMOTE_NETWORK_ADDRESS=bazel.build:80
 test:ci-macos --sandbox_writable_path=/Users/buildkite/bazeltest
 test:ci-macos --sandbox_default_allow_network=false
+test:ci-macos --test_tag_filters=-no_macos
 
 ## For Windows
 common:ci-windows --config=ci-common

diff --git a/...ava/com/google/devtools/build/lib/blackbox/tests/workspace/GitRepositoryBlackBoxTest.java b/...ava/com/google/devtools/build/lib/blackbox/tests/workspace/GitRepositoryBlackBoxTest.java
@@ -40,7 +40,7 @@
  * <p>1. We are creating some git repository with the preset contents, which will be used for
  * fetching contents for the test. We plan to fetch contents specifying either commit hash, tag, or
  * branch. For all test variants, we are creating the same repository, as the same HEAD commit is
- * marked with a tag, and can be addressed with commit hash, master branch, and tag name.
+ * marked with a tag, and can be addressed with commit hash, main branch, and tag name.
  *
  * <p>2. The contents of the git repository working tree is generated by {@link
  * RepoWithRuleWritingTextGenerator}. We pass some certain text to that generator; that exact text
@@ -123,7 +123,7 @@ public void testCloneAtCommit() throws Exception {
    * general approach description in the class javadoc comment.
    */
   @Test
-  public void testCloneAtMaster() throws Exception {
+  public void testCloneAtMain() throws Exception {
     Path repo = context().getTmpDir().resolve("ext_repo");
     setupGitRepository(context(), repo);
 
@@ -138,7 +138,7 @@ public void testCloneAtMaster() throws Exception {
             "new_git_repository(",
             "  name='ext',",
             String.format("  remote='%s',", PathUtils.pathToFileURI(repo.resolve(".git"))),
-            "  branch='master',",
+            "  branch='main',",
             String.format("  build_file_content=\"\"\"%s\"\"\",", buildFileContent),
             ")");
 
@@ -158,7 +158,7 @@ public void testCheckoutOfCommitFromBranch() throws Exception {
     Path repo = context().getTmpDir().resolve("branch_repo");
     GitRepositoryHelper gitRepository = initGitRepository(context(), repo);
 
-    context().write(repo.resolve("master.marker").toString());
+    context().write(repo.resolve("main.marker").toString());
     gitRepository.addAll();
     gitRepository.commit("Initial commit");
 
@@ -171,10 +171,10 @@ public void testCheckoutOfCommitFromBranch() throws Exception {
     gitRepository.commit("Commit in branch");
     String branchCommitHash = gitRepository.getHead();
 
-    gitRepository.checkout("master");
+    gitRepository.checkout("main");
     generator.withOutputText(HELLO_FROM_EXTERNAL_REPOSITORY).setupRepository();
     gitRepository.addAll();
-    gitRepository.commit("Commit in master");
+    gitRepository.commit("Commit in main");
 
     context()
         .write(
@@ -203,7 +203,7 @@ public void testCheckoutOfCommitFromTag() throws Exception {
     Path repo = context().getTmpDir().resolve("tag_repo");
     GitRepositoryHelper gitRepository = initGitRepository(context(), repo);
 
-    context().write(repo.resolve("master.marker").toString());
+    context().write(repo.resolve("main.marker").toString());
     gitRepository.addAll();
     gitRepository.commit("Initial commit");
 
@@ -217,14 +217,14 @@ public void testCheckoutOfCommitFromTag() throws Exception {
     gitRepository.tag("tag1");
     String tagCommitHash = gitRepository.getHead();
 
-    gitRepository.checkout("master");
+    gitRepository.checkout("main");
 
     // delete branch, so that the last commit is not an any branch.
     gitRepository.deleteBranch("demonstrate_branch");
 
     generator.withOutputText(HELLO_FROM_EXTERNAL_REPOSITORY).setupRepository();
     gitRepository.addAll();
-    gitRepository.commit("Commit in master");
+    gitRepository.commit("Commit in main");
 
     context()
         .write(

diff --git a/...test/java/com/google/devtools/build/lib/blackbox/tests/workspace/GitRepositoryHelper.java b/...test/java/com/google/devtools/build/lib/blackbox/tests/workspace/GitRepositoryHelper.java
@@ -48,6 +48,8 @@ void init() throws Exception {
     runGit("init");
     runGit("config", "user.email", "[email protected]");
     runGit("config", "user.name", "E X Ample");
+    runGit("commit", "--allow-empty", "-m", "Initial commit");
+    runGit("branch", "-M", "main");
   }
 
   /**

diff --git a/...est/java/com/google/devtools/build/lib/blackbox/tests/workspace/PatchApiBlackBoxTest.java b/...est/java/com/google/devtools/build/lib/blackbox/tests/workspace/PatchApiBlackBoxTest.java
@@ -113,7 +113,7 @@ private void setUpPatchTestRepo(
             "index 1f4c41e..9d548ff 100644",
             "--- a/foo.sh",
             "+++ b/foo.sh",
-            "@@ -1,3 +1,3 @@",
+            "@@ -1,4 +1,4 @@",
             " #!/usr/bin/env sh",
             "",
             "-echo Here be dragons...",

diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/BUILD b/src/test/java/com/google/devtools/build/lib/sandbox/BUILD
@@ -134,7 +134,11 @@ java_test(
     name = "CgroupsInfoTest",
     size = "small",
     srcs = ["CgroupsInfoTest.java"],
-    tags = ["no_windows"],
+    tags = [
+        # MacOS does not have cgroups so it can't support hardened sandbox
+        "no_macos",
+        "no_windows",
+    ],
     deps = [
         "//src/main/java/com/google/devtools/build/lib/sandbox:linux_sandbox",
         "//src/main/java/com/google/devtools/build/lib/util:pair",

diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
@@ -606,6 +606,8 @@ sh_test(
     srcs = ["bazel_cc_code_coverage_test.sh"],
     data = [":test-deps"],
     tags = [
+        # C++ coverage is not supported on macOS yet.
+        "no_macos",
         "no_windows",
     ],
 )