From bd5e85acb4a7881fda3130871658e2921820ad97 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 12:59:30 +0900 Subject: [PATCH 01/19] gh-112536: Set up TSAN CI for free-threading --- .github/workflows/build.yml | 58 +++++++++++++++++++++++++++++++++++++ configure | 7 +++++ configure.ac | 6 ++++ pyconfig.h.in | 3 ++ 4 files changed, 74 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20d1fad40ecafe..e3e396d52e4678 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -472,6 +472,64 @@ jobs: - name: Tests run: xvfb-run make test + build_tsan_free_threading: + name: 'Thread sanitizer (free-threading)' + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + env: + OPENSSL_VER: 3.0.13 + PYTHONSTRICTEXTENSIONBUILD: 1 + ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 + steps: + - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Set up GCC-10 for TSAN + uses: egor-tensin/setup-gcc@v1 + with: + version: 10 + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v4 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Configure CPython + run: ./configure --config-cache --with-thread-sanitizer --disable-gil + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: xvfb-run make test + # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: name: CIFuzz diff --git a/configure b/configure index 5e8c9af7befde0..4fd271b2847559 100755 --- a/configure +++ b/configure @@ -12718,6 +12718,13 @@ with_tsan="no" fi +if test "$with_tsan" = "yes" +then + +printf "%s\n" "#define MI_TSAN 1" >>confdefs.h + +fi + # Set info about shared libraries. diff --git a/configure.ac b/configure.ac index e615259dcb1fca..1a35d858c36b2a 100644 --- a/configure.ac +++ b/configure.ac @@ -3264,6 +3264,12 @@ AC_MSG_RESULT([no]) with_tsan="no" ]) +if test "$with_tsan" = "yes" +then + AC_DEFINE([MI_TSAN], [1], + [Define MI_TSAN for thread sanitizer]) +fi + # Set info about shared libraries. AC_SUBST([SHLIB_SUFFIX]) AC_SUBST([LDSHARED]) diff --git a/pyconfig.h.in b/pyconfig.h.in index e28baef51d5737..cff177154dd0bd 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1588,6 +1588,9 @@ . */ #undef MAJOR_IN_SYSMACROS +/* Define MI_TSAN for thread sanitizer */ +#undef MI_TSAN + /* Define if mvwdelch in curses.h is an expression. */ #undef MVWDELCH_IS_EXPRESSION From a7a3b08a278512ba44959e108edc5bc89b327958 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 13:35:49 +0900 Subject: [PATCH 02/19] Reduce test scope --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e3e396d52e4678..ac632960261618 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -524,7 +524,7 @@ jobs: - name: Configure CPython run: ./configure --config-cache --with-thread-sanitizer --disable-gil - name: Build CPython - run: make -j4 + run: ./python -m test --pgo -j4 # Reduce test scope - name: Display build info run: make pythoninfo - name: Tests From a7f482669e04551258631a42d64f3272564e51b0 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 13:43:50 +0900 Subject: [PATCH 03/19] fix --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ac632960261618..4d8eacaa88a7e3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -524,11 +524,11 @@ jobs: - name: Configure CPython run: ./configure --config-cache --with-thread-sanitizer --disable-gil - name: Build CPython - run: ./python -m test --pgo -j4 # Reduce test scope + run: make -j4 - name: Display build info run: make pythoninfo - name: Tests - run: xvfb-run make test + run: ./python -m test --pgo -j4 # Reduce test scope # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: From 84bccfb527b65dc63f7c55685fe3870de6b4241f Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 14:13:34 +0900 Subject: [PATCH 04/19] Add _Py_NO_SANITIZE_THREAD annotation --- Objects/obmalloc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 4fe195b63166c1..44d8c4d7710fe5 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -446,6 +446,7 @@ _PyMem_ArenaFree(void *Py_UNUSED(ctx), void *ptr, /***************************/ static int +_Py_NO_SANITIZE_THREAD set_default_allocator_unlocked(PyMemAllocatorDomain domain, int debug, PyMemAllocatorEx *old_alloc) { From aa15f1612215ba94f2b8eb9e3718add581e8d69e Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 14:17:42 +0900 Subject: [PATCH 05/19] nit --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4d8eacaa88a7e3..46fa881696ba1a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -481,7 +481,6 @@ jobs: env: OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 - ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - uses: actions/checkout@v4 - name: Runner image version From 0f7269007805aeaa3e44a67cea6ce54dc2ce2893 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 14:19:41 +0900 Subject: [PATCH 06/19] nit --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 46fa881696ba1a..368f106b278b60 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -618,6 +618,7 @@ jobs: build_windows, build_windows_free_threading, build_asan, + build_tsan_free_threading, ' || '' }} From 2ce5cdd7f743c4bc8debb33b4af33ee527b2d768 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 14:24:31 +0900 Subject: [PATCH 07/19] fix --- Objects/obmalloc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 44d8c4d7710fe5..7fc146c1ccd4d1 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -848,6 +848,7 @@ get_allocator_unlocked(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) } static void +_Py_NO_SANITIZE_THREAD set_allocator_unlocked(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) { switch(domain) From 05a588281a1928485647dd210b489d937d32a9eb Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 14:32:17 +0900 Subject: [PATCH 08/19] swtich to clang --- .github/workflows/build.yml | 6 +++--- Objects/obmalloc.c | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 368f106b278b60..400489be249368 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -494,10 +494,10 @@ jobs: run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-10 for TSAN - uses: egor-tensin/setup-gcc@v1 + - name: Set up clang-15 for TSAN + uses: egor-tensin/setup-clang@v1 with: - version: 10 + version: 15 - name: Configure OpenSSL env vars run: | echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 7fc146c1ccd4d1..44d8c4d7710fe5 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -848,7 +848,6 @@ get_allocator_unlocked(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) } static void -_Py_NO_SANITIZE_THREAD set_allocator_unlocked(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) { switch(domain) From 0b68814bc25a78127a4fac4dfbffe4c1afc2cc57 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 14:38:45 +0900 Subject: [PATCH 09/19] Update --- .github/workflows/build.yml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 400489be249368..0ecb4b092b58eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -478,9 +478,6 @@ jobs: timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' - env: - OPENSSL_VER: 3.0.13 - PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 - name: Runner image version @@ -490,36 +487,12 @@ jobs: with: path: config.cache key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Set up clang-15 for TSAN uses: egor-tensin/setup-clang@v1 with: version: 15 - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython run: ./configure --config-cache --with-thread-sanitizer --disable-gil - name: Build CPython From d2a44760872441e6d60061021524bc7dfb49d351 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 14:44:24 +0900 Subject: [PATCH 10/19] Update env --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0ecb4b092b58eb..6a5b8bf0b78d6b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -493,6 +493,9 @@ jobs: uses: egor-tensin/setup-clang@v1 with: version: 15 + run: | + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clangxx" >> $GITHUB_ENV - name: Configure CPython run: ./configure --config-cache --with-thread-sanitizer --disable-gil - name: Build CPython From f641cad4886a90390003c51581b159c2b5c9369c Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 14:47:11 +0900 Subject: [PATCH 11/19] fix --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6a5b8bf0b78d6b..43f0418a2a8710 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -493,11 +493,11 @@ jobs: uses: egor-tensin/setup-clang@v1 with: version: 15 + - name: Configure CPython run: | echo "CC=clang" >> $GITHUB_ENV echo "CXX=clangxx" >> $GITHUB_ENV - - name: Configure CPython - run: ./configure --config-cache --with-thread-sanitizer --disable-gil + ./configure --config-cache --with-thread-sanitizer --disable-gil - name: Build CPython run: make -j4 - name: Display build info From dd88d4a2a0ad1e249505f124273e723ba3e47a39 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 14:50:34 +0900 Subject: [PATCH 12/19] fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 43f0418a2a8710..10cb6cff32bf11 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -496,7 +496,7 @@ jobs: - name: Configure CPython run: | echo "CC=clang" >> $GITHUB_ENV - echo "CXX=clangxx" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV ./configure --config-cache --with-thread-sanitizer --disable-gil - name: Build CPython run: make -j4 From 6c96e4174b23b94992fcb007fe38db5b7047f36f Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 14:54:22 +0900 Subject: [PATCH 13/19] fix --- .github/workflows/build.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 10cb6cff32bf11..5b076a73d3116a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -494,10 +494,7 @@ jobs: with: version: 15 - name: Configure CPython - run: | - echo "CC=clang" >> $GITHUB_ENV - echo "CXX=clang++" >> $GITHUB_ENV - ./configure --config-cache --with-thread-sanitizer --disable-gil + run: CC=clang CXX=clang++ ./configure --config-cache --with-thread-sanitizer --disable-gil - name: Build CPython run: make -j4 - name: Display build info From 179bacd06dd5925c5637285d291229680d7f7c3b Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 14:57:06 +0900 Subject: [PATCH 14/19] Update(?) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5b076a73d3116a..72b01d0d2eee5b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -494,7 +494,7 @@ jobs: with: version: 15 - name: Configure CPython - run: CC=clang CXX=clang++ ./configure --config-cache --with-thread-sanitizer --disable-gil + run: CC=clang CXX=clang++ ./configure --config-cache --with-thread-sanitizer --disable-gil --with-mimalloc - name: Build CPython run: make -j4 - name: Display build info From 691d0e4fcbc489cd4a80e002abbd3a23f4f41441 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 15:01:07 +0900 Subject: [PATCH 15/19] fix --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 72b01d0d2eee5b..ed65d5476a3fd6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -493,8 +493,9 @@ jobs: uses: egor-tensin/setup-clang@v1 with: version: 15 + cc: 1 - name: Configure CPython - run: CC=clang CXX=clang++ ./configure --config-cache --with-thread-sanitizer --disable-gil --with-mimalloc + run: ./configure --config-cache --with-thread-sanitizer --disable-gil - name: Build CPython run: make -j4 - name: Display build info From d89f932ab944f765aa8fe8c921700e398b10843e Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 15:03:20 +0900 Subject: [PATCH 16/19] fix(?) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed65d5476a3fd6..7a03a33afb62e8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -495,7 +495,7 @@ jobs: version: 15 cc: 1 - name: Configure CPython - run: ./configure --config-cache --with-thread-sanitizer --disable-gil + run: ./configure --config-cache --with-thread-sanitizer --disable-gil --with-mimalloc - name: Build CPython run: make -j4 - name: Display build info From d833618bb3a5077847f11c058c740b1fce38299f Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 15:04:38 +0900 Subject: [PATCH 17/19] Check order --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7a03a33afb62e8..d70d010bf94bfa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -495,7 +495,7 @@ jobs: version: 15 cc: 1 - name: Configure CPython - run: ./configure --config-cache --with-thread-sanitizer --disable-gil --with-mimalloc + run: ./configure --config-cache --disable-gil --with-thread-sanitizer - name: Build CPython run: make -j4 - name: Display build info From 33317e7a808939d8b11d44d872a710f478b0628a Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 15:06:52 +0900 Subject: [PATCH 18/19] Weired fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d70d010bf94bfa..51ae2ebd359603 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -495,7 +495,7 @@ jobs: version: 15 cc: 1 - name: Configure CPython - run: ./configure --config-cache --disable-gil --with-thread-sanitizer + run: ./configure --config-cache --disable-gil --with-mimalloc --with-thread-sanitizer - name: Build CPython run: make -j4 - name: Display build info From cade95e59f97cccb16d9d5f1259685bf57b1117b Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 15:11:10 +0900 Subject: [PATCH 19/19] Revert to the gcc --- .github/workflows/build.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 51ae2ebd359603..4d7e6a63e656fe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -480,22 +480,14 @@ jobs: if: needs.check_source.outputs.run_tests == 'true' steps: - uses: actions/checkout@v4 - - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up clang-15 for TSAN - uses: egor-tensin/setup-clang@v1 + - name: Set up GCC-10 for ASAN + uses: egor-tensin/setup-gcc@v1 with: - version: 15 - cc: 1 + version: 11 - name: Configure CPython - run: ./configure --config-cache --disable-gil --with-mimalloc --with-thread-sanitizer + run: ./configure --disable-gil --with-thread-sanitizer - name: Build CPython run: make -j4 - name: Display build info