diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp index 23218e3eea..856bda6db5 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp @@ -10,6 +10,7 @@ desc: Provides a gdb interface to the guest state #include "GdbServer/Info.h" #include "LinuxSyscalls/NetStream.h" +#include "LinuxSyscalls/SignalDelegator.h" #include #include @@ -75,11 +76,6 @@ void GdbServer::Break(FEX::HLE::ThreadStateObject* ThreadObject, int signal) { SendPacket(*CommsStream, str); } -void GdbServer::WaitForThreadWakeup() { - // Wait for gdbserver to tell us to wake up - ThreadBreakEvent.Wait(); -} - GdbServer::~GdbServer() { CloseListenSocket(); CoreShuttingDown = true; @@ -120,7 +116,7 @@ GdbServer::GdbServer(FEXCore::Context::Context* ctx, FEX::HLE::SignalDelegator* // Let GDB know that we have a signal this->Break(ThreadObject, Signal); - WaitForThreadWakeup(); + this->SyscallHandler->TM.SleepThread(this->CTX, ThreadObject); return true; }, @@ -604,8 +600,11 @@ GdbServer::HandledPacketType GdbServer::handleProgramOffsets() { GdbServer::HandledPacketType GdbServer::ThreadAction(char action, uint32_t tid) { switch (action) { case 'c': { + auto Threads = SyscallHandler->TM.GetThreads(); + for (auto& Thread : *Threads) { + Thread->ThreadSleeping.NotifyOne(); + } SyscallHandler->TM.Run(); - ThreadBreakEvent.NotifyAll(); SyscallHandler->TM.WaitForThreadsToRun(); return {"", HandledPacketType::TYPE_ONLYACK}; } @@ -1431,6 +1430,13 @@ void GdbServer::StartThread() { FEXCore::Threads::SetSignalMask(OldMask); } +void GdbServer::CreateThreadCallback(FEX::HLE::ThreadStateObject* ThreadObject) { + if (SyscallHandler->TM.GetThreadCount() == 1) { + // Sleep the first thread created. This is because FEX only currently supports attaching at startup. + SyscallHandler->TM.SleepThread(CTX, ThreadObject); + } +} + void GdbServer::OpenListenSocket() { const auto GdbUnixPath = fextl::fmt::format("{}/FEX_gdbserver/", FEXServerClient::GetTempFolder()); if (FHU::Filesystem::CreateDirectory(GdbUnixPath) == FHU::Filesystem::CreateDirectoryResult::ERROR) { diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h index eb7ac11ac6..1cc2c142e4 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h @@ -35,6 +35,8 @@ class GdbServer { LibraryMapChanged = true; } + void CreateThreadCallback(FEX::HLE::ThreadStateObject* ThreadObject); + private: void Break(FEX::HLE::ThreadStateObject* ThreadObject, int signal); @@ -52,9 +54,6 @@ class GdbServer { void SendACK(std::ostream& stream, bool NACK); - Event ThreadBreakEvent {}; - void WaitForThreadWakeup(); - struct HandledPacketType { fextl::string Response {}; enum ResponseType { diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp index 7ca31e4cad..b49c00d383 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp @@ -475,7 +475,7 @@ bool SignalDelegator::HandleSignalPause(FEXCore::Core::InternalThreadState* Thre // We need to be a little bit careful here // If we were already paused (due to GDB) and we are immediately stopping (due to gdb kill) // Then we need to ensure we don't double decrement our idle thread counter - if (ThreadObject->ThreadSleeping) { + if (ThreadObject->ThreadSleeping.HasWaiter()) { // If the thread was sleeping then its idle counter was decremented // Reincrement it here to not break logic FEX::HLE::_SyscallHandler->TM.IncrementIdleRefCount(); diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h index 04b66c0239..f78e0e29b7 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h @@ -53,7 +53,7 @@ desc: Glue logic, STRACE magic namespace FEX { class CodeLoader; class GdbServer; -} +} // namespace FEX namespace FEXCore { namespace Context { diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp index 75cb1d87e5..7fc5e245f1 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +#include "LinuxSyscalls/GdbServer.h" #include "LinuxSyscalls/Syscalls.h" #include "LinuxSyscalls/SignalDelegator.h" @@ -29,6 +30,16 @@ FEX::HLE::ThreadStateObject* ThreadManager::CreateThread(uint64_t InitialRIP, ui return ThreadStateObject; } +void ThreadManager::TrackThread(FEX::HLE::ThreadStateObject* Thread) { + { + std::lock_guard lk(ThreadCreationMutex); + Threads.emplace_back(Thread); + } + + auto GdbServer = SyscallHandler->GetGdbServer(); + GdbServer->CreateThreadCallback(Thread); +} + void ThreadManager::DestroyThread(FEX::HLE::ThreadStateObject* Thread, bool NeedsTLSUninstall) { { std::lock_guard lk(ThreadCreationMutex); @@ -71,7 +82,10 @@ void ThreadManager::NotifyPause() { // Tell all the threads that they should pause std::lock_guard lk(ThreadCreationMutex); for (auto& Thread : Threads) { - SignalDelegation->SignalThread(Thread->Thread, SignalEvent::Pause); + if (Thread->ThreadSleeping.HasWaiter() == false) { + // Only signal if it isn't already sleeping. + SignalDelegation->SignalThread(Thread->Thread, SignalEvent::Pause); + } } } @@ -162,25 +176,20 @@ void ThreadManager::Stop(bool IgnoreCurrentThread) { } } -void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame) { - auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromCPUState(Frame); - +void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEX::HLE::ThreadStateObject* ThreadObject) { --IdleWaitRefCount; IdleWaitCV.notify_all(); - ThreadObject->ThreadSleeping = true; - - // Go to sleep - ThreadObject->ThreadPaused.Wait(); + // Go to sleep. + ThreadObject->ThreadSleeping.Wait(); ++IdleWaitRefCount; - ThreadObject->ThreadSleeping = false; IdleWaitCV.notify_all(); } void ThreadManager::UnpauseThread(FEX::HLE::ThreadStateObject* Thread) { - Thread->ThreadPaused.NotifyOne(); + Thread->ThreadSleeping.NotifyOne(); } void ThreadManager::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThread, bool Child) { diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h index 33b0de0064..d18668d290 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h @@ -22,6 +22,80 @@ namespace FEX::HLE { class SyscallHandler; class SignalDelegator; +// This is similar to FEXCore::InterruptableConditionVariable with the exception that notifying only occurs if there is a waiter and can check if +// there is a waiter. +// This allows us to remove the race condition between a thread trying to go asleep and something else telling it to go to sleep or wake up. +// +// Only one thread can ever wait on a latch, while another thread signals it. +class InspectableLatch final { +public: + bool Wait(struct timespec* Timeout = nullptr) { + while (true) { + uint32_t Expected = HAS_NO_WAITER; + const uint32_t Desired = HAS_WAITER; + + if (Mutex.compare_exchange_strong(Expected, Desired)) { + // We have latched, now futex. + constexpr int Op = FUTEX_WAIT | FUTEX_PRIVATE_FLAG; + // WAIT will keep sleeping on the futex word while it is `val` + int Result = ::syscall(SYS_futex, &Mutex, Op, + Desired, // val + Timeout, // Timeout/val2 + nullptr, // Addr2 + 0); // val3 + + if (Timeout && Result == -1 && errno == ETIMEDOUT) { + return false; + } + } else if (Expected == HAS_SIGNALED) { + // Reset the latch once signaled + Mutex.store(HAS_NO_WAITER); + return true; + } + } + } + + template + bool WaitFor(const std::chrono::duration& time) { + struct timespec Timeout {}; + auto SecondsDuration = std::chrono::duration_cast(time); + Timeout.tv_sec = SecondsDuration.count(); + Timeout.tv_nsec = std::chrono::duration_cast(time - SecondsDuration).count(); + return Wait(&Timeout); + } + + void NotifyOne() { + DoNotify(1); + } + + bool HasWaiter() const { + return Mutex.load() == HAS_WAITER; + } + +private: + std::atomic Mutex {}; + constexpr static uint32_t HAS_NO_WAITER = 0; + constexpr static uint32_t HAS_WAITER = 1; + constexpr static uint32_t HAS_SIGNALED = 2; + + void DoNotify(int Waiters) { + uint32_t Expected = HAS_WAITER; + const uint32_t Desired = HAS_SIGNALED; + + // If the mutex is in a waiting state and we have CAS exchanged it to HAS_SIGNALED, then futex. + // otherwise just leave since nothing was waiting. + if (Mutex.compare_exchange_strong(Expected, Desired)) { + constexpr int Op = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; + + ::syscall(SYS_futex, &Mutex, Op, + Waiters, // val - Number of waiters to wake + 0, // val2 + &Mutex, // Addr2 - Mutex to do the operation on + 0); // val3 + } + } +}; + enum class SignalEvent : uint32_t { Nothing, // If the guest uses our signal we need to know it was errant on our end Pause, @@ -81,8 +155,7 @@ struct ThreadStateObject : public FEXCore::Allocator::FEXAllocOperators { std::atomic SignalReason {SignalEvent::Nothing}; // Thread pause handling - std::atomic_bool ThreadSleeping {false}; - FEXCore::InterruptableConditionVariable ThreadPaused; + InspectableLatch ThreadSleeping; // GDB signal information struct GdbInfoStruct { @@ -116,10 +189,7 @@ class ThreadManager final { FEX::HLE::ThreadStateObject* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, const FEXCore::Core::CPUState* NewThreadState = nullptr, uint64_t ParentTID = 0, FEX::HLE::ThreadStateObject* InheritThread = nullptr); - void TrackThread(FEX::HLE::ThreadStateObject* Thread) { - std::lock_guard lk(ThreadCreationMutex); - Threads.emplace_back(Thread); - } + void TrackThread(FEX::HLE::ThreadStateObject* Thread); void DestroyThread(FEX::HLE::ThreadStateObject* Thread, bool NeedsTLSUninstall = false); void StopThread(FEX::HLE::ThreadStateObject* Thread); @@ -134,7 +204,11 @@ class ThreadManager final { void WaitForIdleWithTimeout(); void WaitForThreadsToRun(); - void SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame); + void SleepThread(FEXCore::Context::Context* CTX, FEX::HLE::ThreadStateObject* ThreadObject); + void SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame) { + auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromCPUState(Frame); + SleepThread(CTX, ThreadObject); + } void UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child); @@ -173,6 +247,11 @@ class ThreadManager final { return &Threads; } + size_t GetThreadCount() { + std::lock_guard lk(ThreadCreationMutex); + return Threads.size(); + } + private: FEXCore::Context::Context* CTX; FEX::HLE::SyscallHandler* SyscallHandler;