From 20578a1f68c841a264b72b00591b11ab2fa77b43 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sat, 16 Mar 2024 19:10:37 +0900 Subject: [PATCH] gh-112536: Add TSAN builds on Github Actions (#116872) --- .github/workflows/build.yml | 22 ++++++++++ .github/workflows/reusable-tsan.yml | 51 ++++++++++++++++++++++++ Lib/test/test_concurrent_futures/util.py | 4 ++ Lib/test/test_logging.py | 4 ++ Lib/test/test_threading.py | 8 ++++ Python/thread_pthread.h | 4 ++ 6 files changed, 93 insertions(+) create mode 100644 .github/workflows/reusable-tsan.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d43b83e830e1fb..e36859e728b67f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -484,6 +484,24 @@ jobs: - name: Tests run: xvfb-run make test + build_tsan: + name: 'Thread sanitizer' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: ./configure --config-cache --with-thread-sanitizer --with-pydebug + + build_tsan_free_threading: + name: 'Thread sanitizer (free-threading)' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug + # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: name: CIFuzz @@ -542,6 +560,8 @@ jobs: - build_windows_free_threading - test_hypothesis - build_asan + - build_tsan + - build_tsan_free_threading - cifuzz runs-on: ubuntu-latest @@ -575,6 +595,8 @@ jobs: build_windows, build_windows_free_threading, build_asan, + build_tsan, + build_tsan_free_threading, ' || '' }} diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml new file mode 100644 index 00000000000000..96a9c1b0cda3c3 --- /dev/null +++ b/.github/workflows/reusable-tsan.yml @@ -0,0 +1,51 @@ +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + options: + required: true + type: string + +jobs: + build_tsan_reusable: + name: 'Thread sanitizer' + runs-on: ubuntu-22.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + - name: Install Dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt install -y clang + # Reduce ASLR to avoid TSAN crashing + sudo sysctl -w vm.mmap_rnd_bits=28 + - name: TSAN Option Setup + run: | + echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Configure CPython + run: ${{ inputs.options }} + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: ./python -m test --tsan -j4 diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index 3e855031913042..3b8ec3e205d5aa 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -85,6 +85,8 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") return super().get_context() @@ -111,6 +113,8 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") return super().get_context() diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 32bb5171a3f757..c84eca51b52362 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -80,6 +80,9 @@ skip_if_asan_fork = unittest.skipIf( support.HAVE_ASAN_FORK_BUG, "libasan has a pthread_create() dead lock related to thread+fork") +skip_if_tsan_fork = unittest.skipIf( + support.check_sanitizer(thread=True), + "TSAN doesn't support threads after fork") class BaseTest(unittest.TestCase): @@ -731,6 +734,7 @@ def remove_loop(fname, tries): @support.requires_fork() @threading_helper.requires_working_threading() @skip_if_asan_fork + @skip_if_tsan_fork def test_post_fork_child_no_deadlock(self): """Ensure child logging locks are not held; bpo-6721 & bpo-36533.""" class _OurHandler(logging.Handler): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 3b5c37c948c8c3..9769cb41e3e689 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -50,6 +50,11 @@ def skip_unless_reliable_fork(test): return test +skip_if_tsan_fork = unittest.skipIf( + support.check_sanitizer(thread=True), + "TSAN doesn't support threads after fork") + + def requires_subinterpreters(meth): """Decorator to skip a test if subinterpreters are not supported.""" return unittest.skipIf(interpreters is None, @@ -634,6 +639,7 @@ def test_daemon_param(self): self.assertTrue(t.daemon) @skip_unless_reliable_fork + @skip_if_tsan_fork def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up # the after-fork mechanism. @@ -703,6 +709,7 @@ def f(): @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + @skip_if_tsan_fork def test_main_thread_after_fork(self): code = """if 1: import os, threading @@ -1271,6 +1278,7 @@ def test_2_join_in_forked_process(self): self._run_and_join(script) @skip_unless_reliable_fork + @skip_if_tsan_fork def test_3_join_in_forked_from_thread(self): # Like the test above, but fork() was called from a worker thread # In the forked process, the main Thread object must be marked as stopped. diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 64cc60053e6cf7..65d366e91c322a 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -95,6 +95,10 @@ #endif #endif +/* Thread sanitizer doesn't currently support sem_clockwait */ +#ifdef _Py_THREAD_SANITIZER +#undef HAVE_SEM_CLOCKWAIT +#endif /* Whether or not to use semaphores directly rather than emulating them with * mutexes and condition variables: