From 9f546fcf68d90da53504272c23bdd2d379962eff Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 30 May 2023 13:35:09 +0200 Subject: [PATCH 1/3] Check that SingleThreadedBlasScope is created and destroyed on the same std::thread --- include/dlaf/common/single_threaded_blas.h | 7 +++++++ src/common/single_threaded_blas.cpp | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/include/dlaf/common/single_threaded_blas.h b/include/dlaf/common/single_threaded_blas.h index d8167cd949..452f14b0a0 100644 --- a/include/dlaf/common/single_threaded_blas.h +++ b/include/dlaf/common/single_threaded_blas.h @@ -10,6 +10,10 @@ #pragma once +#ifdef DLAF_ASSERT_MODERATE_ENABLE +#include +#endif + namespace dlaf::common::internal { class [[nodiscard]] SingleThreadedBlasScope { public: @@ -21,6 +25,9 @@ class [[nodiscard]] SingleThreadedBlasScope { SingleThreadedBlasScope& operator=(SingleThreadedBlasScope const&) = delete; private: +#ifdef DLAF_ASSERT_MODERATE_ENABLE + std::thread::id calling_thread; +#endif #ifdef DLAF_WITH_OPENMP int omp_num_threads; #endif diff --git a/src/common/single_threaded_blas.cpp b/src/common/single_threaded_blas.cpp index be38e2e5bc..8955837d56 100644 --- a/src/common/single_threaded_blas.cpp +++ b/src/common/single_threaded_blas.cpp @@ -17,8 +17,15 @@ #include +#ifdef DLAF_ASSERT_MODERATE_ENABLE +#include +#endif + namespace dlaf::common::internal { SingleThreadedBlasScope::SingleThreadedBlasScope() { +#ifdef DLAF_ASSERT_MODERATE_ENABLE + calling_thread = std::this_thread::get_id(); +#endif #ifdef DLAF_WITH_OPENMP omp_num_threads = omp_get_max_threads(); omp_set_num_threads(1); @@ -29,6 +36,10 @@ SingleThreadedBlasScope::SingleThreadedBlasScope() { } SingleThreadedBlasScope::~SingleThreadedBlasScope() { +#ifdef DLAF_ASSERT_MODERATE_ENABLE + auto current_thread = std::this_thread::get_id(); + DLAF_ASSERT_MODERATE(calling_thread == current_thread, calling_thread, current_thread); +#endif #ifdef DLAF_WITH_OPENMP omp_set_num_threads(omp_num_threads); #endif From 6daae56f823139ea5ba23c328dd505d502e562b6 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 2 Jun 2023 09:20:07 +0200 Subject: [PATCH 2/3] Narrow scope of SingleThreadedBlasScope in test_bt_band_to_tridiag to avoid keeping it alive while yielding --- .../eigensolver/test_bt_band_to_tridiag.cpp | 60 ++++++++++--------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/test/unit/eigensolver/test_bt_band_to_tridiag.cpp b/test/unit/eigensolver/test_bt_band_to_tridiag.cpp index 8ea1aeed02..95083988e1 100644 --- a/test/unit/eigensolver/test_bt_band_to_tridiag.cpp +++ b/test/unit/eigensolver/test_bt_band_to_tridiag.cpp @@ -134,24 +134,26 @@ void testBacktransformation(SizeType m, SizeType n, SizeType mb, SizeType nb, co if (m == 0 || n == 0) return; - dlaf::common::internal::SingleThreadedBlasScope single; + { + dlaf::common::internal::SingleThreadedBlasScope single; - using eigensolver::internal::nrStepsForSweep; - using eigensolver::internal::nrSweeps; - for (SizeType sweep = nrSweeps(m) - 1; sweep >= 0; --sweep) { - for (SizeType step = nrStepsForSweep(sweep, m, b) - 1; step >= 0; --step) { - const SizeType j = sweep; - const SizeType i = j + 1 + step * b; + using eigensolver::internal::nrStepsForSweep; + using eigensolver::internal::nrSweeps; + for (SizeType sweep = nrSweeps(m) - 1; sweep >= 0; --sweep) { + for (SizeType step = nrStepsForSweep(sweep, m, b) - 1; step >= 0; --step) { + const SizeType j = sweep; + const SizeType i = j + 1 + step * b; - const SizeType size = std::min(b, m - i); - const SizeType i_v = (i - 1) / b * b; + const SizeType size = std::min(b, m - i); + const SizeType i_v = (i - 1) / b * b; - T& v_head = *mat_hh_local.ptr({i_v, j}); - const T tau = v_head; - v_head = 1; + T& v_head = *mat_hh_local.ptr({i_v, j}); + const T tau = v_head; + v_head = 1; - using blas::Side; - lapack::larf(Side::Left, size, n, &v_head, 1, tau, mat_e_local.ptr({i, 0}), mat_e_local.ld()); + using blas::Side; + lapack::larf(Side::Left, size, n, &v_head, 1, tau, mat_e_local.ptr({i, 0}), mat_e_local.ld()); + } } } @@ -212,24 +214,26 @@ void testBacktransformation(comm::CommunicatorGrid grid, SizeType m, SizeType n, if (m == 0 || n == 0) return; - dlaf::common::internal::SingleThreadedBlasScope single; + { + dlaf::common::internal::SingleThreadedBlasScope single; - using eigensolver::internal::nrStepsForSweep; - using eigensolver::internal::nrSweeps; - for (SizeType sweep = nrSweeps(m) - 1; sweep >= 0; --sweep) { - for (SizeType step = nrStepsForSweep(sweep, m, b) - 1; step >= 0; --step) { - const SizeType j = sweep; - const SizeType i = j + 1 + step * b; + using eigensolver::internal::nrStepsForSweep; + using eigensolver::internal::nrSweeps; + for (SizeType sweep = nrSweeps(m) - 1; sweep >= 0; --sweep) { + for (SizeType step = nrStepsForSweep(sweep, m, b) - 1; step >= 0; --step) { + const SizeType j = sweep; + const SizeType i = j + 1 + step * b; - const SizeType size = std::min(b, m - i); - const SizeType i_v = (i - 1) / b * b; + const SizeType size = std::min(b, m - i); + const SizeType i_v = (i - 1) / b * b; - T& v_head = *mat_hh_local.ptr({i_v, j}); - const T tau = v_head; - v_head = 1; + T& v_head = *mat_hh_local.ptr({i_v, j}); + const T tau = v_head; + v_head = 1; - using blas::Side; - lapack::larf(Side::Left, size, n, &v_head, 1, tau, mat_e_local.ptr({i, 0}), mat_e_local.ld()); + using blas::Side; + lapack::larf(Side::Left, size, n, &v_head, 1, tau, mat_e_local.ptr({i, 0}), mat_e_local.ld()); + } } } From 1e98e259d5b6ec0d9040f2689dba116d92c11c99 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 5 Jun 2023 12:07:58 +0200 Subject: [PATCH 3/3] Narrow scope of SingleThreadedBlasScope in test_tridiag_solver_rot to avoid keeping it alive while yielding --- test/unit/eigensolver/test_tridiag_solver_rot.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/unit/eigensolver/test_tridiag_solver_rot.cpp b/test/unit/eigensolver/test_tridiag_solver_rot.cpp index 42732c04f6..16a4c523e7 100644 --- a/test/unit/eigensolver/test_tridiag_solver_rot.cpp +++ b/test/unit/eigensolver/test_tridiag_solver_rot.cpp @@ -108,12 +108,14 @@ void testApplyGivenRotations(comm::CommunicatorGrid grid, const SizeType m, cons const SizeType n = std::min((idx_end) *mb, m) - idx_begin * mb; const GlobalElementSize offset(idx_begin * mb, idx_begin * mb); - dlaf::common::internal::SingleThreadedBlasScope single; + { + dlaf::common::internal::SingleThreadedBlasScope single; - for (auto rot : rots) { - T* x = mat_loc.ptr(GlobalElementIndex{0, rot.i} + offset); - T* y = mat_loc.ptr(GlobalElementIndex{0, rot.j} + offset); - blas::rot(n, x, 1, y, 1, rot.c, rot.s); + for (auto rot : rots) { + T* x = mat_loc.ptr(GlobalElementIndex{0, rot.i} + offset); + T* y = mat_loc.ptr(GlobalElementIndex{0, rot.j} + offset); + blas::rot(n, x, 1, y, 1, rot.c, rot.s); + } } auto result = [&dist = mat_h.distribution(), &mat_local = mat_loc](const GlobalElementIndex& element) {