DB::Exception: Cannot schedule a task: no free thread (timeout=0) (threads=60, jobs=120)

[JinheLin]

[2024/08/30 09:02:26.765 +00:00] [ERROR] [MPPTask.cpp:644] [\"task running meets error: Code: 11001, e.displayText() = DB::Exception: Cannot schedule a task: no free thread (timeout=0) (threads=60, jobs=120), e.what() = DB::Exception, Stack trace:\
\
\
  0x55bdc33f1175\\tDB::Exception::Exception<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, unsigned long, unsigned long&>(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, unsigned long&&, unsigned long&) [tiflash+34464117]\
                \\tdbms/src/Common/StackTrace.cpp:23\
  0x55bdc33f2302\\tvoid DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda'(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&)::operator()(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&) const [tiflash+34468610]\
                \\tdbms/src/Common/UniThreadPool.cpp:113\
  0x55bdc33eee27\\tvoid DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool) [tiflash+34455079]\
                \\tdbms/src/Common/UniThreadPool.cpp:134\
  0x55bdc33ef73c\\tDB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleWithFuture(std::__1::function<void ()>, unsigned long) [tiflash+34457404]\
                \\tdbms/src/Common/UniThreadPool.cpp:198\
  0x55bdca03e120\\tDB::StorageDisaggregated::buildReadTaskForWriteNode(DB::Context const&, std::__1::shared_ptr<DB::DM::ScanContext> const&, pingcap::coprocessor::BatchCopTask const&, std::__1::mutex&, std::__1::list<std::__1::shared_ptr<DB::DM::SegmentReadTask>, std::__1::allocator<std::__1::shared_ptr<DB::DM::SegmentReadTask>>>&) [tiflash+148025632]\
                \\tdbms/src/Storages/StorageDisaggregatedRemote.cpp:367\
  0x55bdc33f1ff5\\tstd::__1::packaged_task<void ()>::operator()() [tiflash+34467829]\
                \\t/usr/local/bin/../include/c++/v1/future:1891\
  0x55bdc33f02a9\\tDB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::worker(std::__1::__list_iterator<DB::ThreadFromGlobalPoolImpl<false>, void*>) [tiflash+34460329]\
                \\t/usr/local/bin/../include/c++/v1/__functional/function.h:517\
  0x55bdc33f2b03\\tstd::__1::__function::__func<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'(), std::__1::allocator<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'()>, void ()>::operator()() [tiflash+34470659]\
                \\tdbms/src/Common/UniThreadPool.cpp:160\
  0x55bdc33f1798\\tvoid* std::__1::__thread_proxy[abi:ue170006]<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, void DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>>(void*) [tiflash+34465688]\
                \\t/usr/local/bin/../include/c++/v1/__functional/function.h:517\
  0x7fcd70fbcc02\\tstart_thread [libc.so.6+564226]\
  0x7fcd71041c40\\tclone3 [libc.so.6+1109056]\"] [source=\"MPP<gather_id:1, query_ts:1725008545647700351, local_query_id:3077, server_id:1992, start_ts:452200640163872786,task_id:13>\"] [thread_id=131]