From 504b9546e5533422fb7d6cabba3366d2665fc99e Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Tue, 10 Dec 2024 14:05:20 -0800 Subject: [PATCH] GdbServer: Changes how thread addition and pausing works This is enough to get GdbServer working for my simple test case again. Some things of note: - There is now a callback which signals to GdbServer when a thread is created. - Thread sleeping is now a single uint32_t futex word for knowing when a thread is sleeping and if we need to signal it to wake up. - GdbServer thread sleeping versus ThreadManager thread sleeping distinction has been removed. GdbServer now uses ThreadManager to put a thread to sleep. This still only gets the attach at startup scheme of gdbserver working. I haven't yet dived back in to getting arbitrary attach working. Primarily the work necessary here is actually getting thread creation at the start to stop and wait for gdbserver to attach. Then actually sleeping correctly while gdbserver is communicating with gdb. --- .../LinuxSyscalls/GdbServer.cpp | 20 ++-- .../LinuxEmulation/LinuxSyscalls/GdbServer.h | 5 +- .../LinuxSyscalls/SignalDelegator.cpp | 2 +- .../LinuxEmulation/LinuxSyscalls/Syscalls.h | 2 +- .../LinuxSyscalls/ThreadManager.cpp | 29 ++++-- .../LinuxSyscalls/ThreadManager.h | 93 +++++++++++++++++-- 6 files changed, 122 insertions(+), 29 deletions(-) diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp index 23218e3eea..856bda6db5 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp @@ -10,6 +10,7 @@ desc: Provides a gdb interface to the guest state #include "GdbServer/Info.h" #include "LinuxSyscalls/NetStream.h" +#include "LinuxSyscalls/SignalDelegator.h" #include #include @@ -75,11 +76,6 @@ void GdbServer::Break(FEX::HLE::ThreadStateObject* ThreadObject, int signal) { SendPacket(*CommsStream, str); } -void GdbServer::WaitForThreadWakeup() { - // Wait for gdbserver to tell us to wake up - ThreadBreakEvent.Wait(); -} - GdbServer::~GdbServer() { CloseListenSocket(); CoreShuttingDown = true; @@ -120,7 +116,7 @@ GdbServer::GdbServer(FEXCore::Context::Context* ctx, FEX::HLE::SignalDelegator* // Let GDB know that we have a signal this->Break(ThreadObject, Signal); - WaitForThreadWakeup(); + this->SyscallHandler->TM.SleepThread(this->CTX, ThreadObject); return true; }, @@ -604,8 +600,11 @@ GdbServer::HandledPacketType GdbServer::handleProgramOffsets() { GdbServer::HandledPacketType GdbServer::ThreadAction(char action, uint32_t tid) { switch (action) { case 'c': { + auto Threads = SyscallHandler->TM.GetThreads(); + for (auto& Thread : *Threads) { + Thread->ThreadSleeping.NotifyOne(); + } SyscallHandler->TM.Run(); - ThreadBreakEvent.NotifyAll(); SyscallHandler->TM.WaitForThreadsToRun(); return {"", HandledPacketType::TYPE_ONLYACK}; } @@ -1431,6 +1430,13 @@ void GdbServer::StartThread() { FEXCore::Threads::SetSignalMask(OldMask); } +void GdbServer::CreateThreadCallback(FEX::HLE::ThreadStateObject* ThreadObject) { + if (SyscallHandler->TM.GetThreadCount() == 1) { + // Sleep the first thread created. This is because FEX only currently supports attaching at startup. + SyscallHandler->TM.SleepThread(CTX, ThreadObject); + } +} + void GdbServer::OpenListenSocket() { const auto GdbUnixPath = fextl::fmt::format("{}/FEX_gdbserver/", FEXServerClient::GetTempFolder()); if (FHU::Filesystem::CreateDirectory(GdbUnixPath) == FHU::Filesystem::CreateDirectoryResult::ERROR) { diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h index eb7ac11ac6..1cc2c142e4 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h @@ -35,6 +35,8 @@ class GdbServer { LibraryMapChanged = true; } + void CreateThreadCallback(FEX::HLE::ThreadStateObject* ThreadObject); + private: void Break(FEX::HLE::ThreadStateObject* ThreadObject, int signal); @@ -52,9 +54,6 @@ class GdbServer { void SendACK(std::ostream& stream, bool NACK); - Event ThreadBreakEvent {}; - void WaitForThreadWakeup(); - struct HandledPacketType { fextl::string Response {}; enum ResponseType { diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp index 7ca31e4cad..b49c00d383 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp @@ -475,7 +475,7 @@ bool SignalDelegator::HandleSignalPause(FEXCore::Core::InternalThreadState* Thre // We need to be a little bit careful here // If we were already paused (due to GDB) and we are immediately stopping (due to gdb kill) // Then we need to ensure we don't double decrement our idle thread counter - if (ThreadObject->ThreadSleeping) { + if (ThreadObject->ThreadSleeping.HasWaiter()) { // If the thread was sleeping then its idle counter was decremented // Reincrement it here to not break logic FEX::HLE::_SyscallHandler->TM.IncrementIdleRefCount(); diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h index 04b66c0239..f78e0e29b7 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h @@ -53,7 +53,7 @@ desc: Glue logic, STRACE magic namespace FEX { class CodeLoader; class GdbServer; -} +} // namespace FEX namespace FEXCore { namespace Context { diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp index 75cb1d87e5..7fc5e245f1 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +#include "LinuxSyscalls/GdbServer.h" #include "LinuxSyscalls/Syscalls.h" #include "LinuxSyscalls/SignalDelegator.h" @@ -29,6 +30,16 @@ FEX::HLE::ThreadStateObject* ThreadManager::CreateThread(uint64_t InitialRIP, ui return ThreadStateObject; } +void ThreadManager::TrackThread(FEX::HLE::ThreadStateObject* Thread) { + { + std::lock_guard lk(ThreadCreationMutex); + Threads.emplace_back(Thread); + } + + auto GdbServer = SyscallHandler->GetGdbServer(); + GdbServer->CreateThreadCallback(Thread); +} + void ThreadManager::DestroyThread(FEX::HLE::ThreadStateObject* Thread, bool NeedsTLSUninstall) { { std::lock_guard lk(ThreadCreationMutex); @@ -71,7 +82,10 @@ void ThreadManager::NotifyPause() { // Tell all the threads that they should pause std::lock_guard lk(ThreadCreationMutex); for (auto& Thread : Threads) { - SignalDelegation->SignalThread(Thread->Thread, SignalEvent::Pause); + if (Thread->ThreadSleeping.HasWaiter() == false) { + // Only signal if it isn't already sleeping. + SignalDelegation->SignalThread(Thread->Thread, SignalEvent::Pause); + } } } @@ -162,25 +176,20 @@ void ThreadManager::Stop(bool IgnoreCurrentThread) { } } -void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame) { - auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromCPUState(Frame); - +void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEX::HLE::ThreadStateObject* ThreadObject) { --IdleWaitRefCount; IdleWaitCV.notify_all(); - ThreadObject->ThreadSleeping = true; - - // Go to sleep - ThreadObject->ThreadPaused.Wait(); + // Go to sleep. + ThreadObject->ThreadSleeping.Wait(); ++IdleWaitRefCount; - ThreadObject->ThreadSleeping = false; IdleWaitCV.notify_all(); } void ThreadManager::UnpauseThread(FEX::HLE::ThreadStateObject* Thread) { - Thread->ThreadPaused.NotifyOne(); + Thread->ThreadSleeping.NotifyOne(); } void ThreadManager::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThread, bool Child) { diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h index 33b0de0064..d18668d290 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h @@ -22,6 +22,80 @@ namespace FEX::HLE { class SyscallHandler; class SignalDelegator; +// This is similar to FEXCore::InterruptableConditionVariable with the exception that notifying only occurs if there is a waiter and can check if +// there is a waiter. +// This allows us to remove the race condition between a thread trying to go asleep and something else telling it to go to sleep or wake up. +// +// Only one thread can ever wait on a latch, while another thread signals it. +class InspectableLatch final { +public: + bool Wait(struct timespec* Timeout = nullptr) { + while (true) { + uint32_t Expected = HAS_NO_WAITER; + const uint32_t Desired = HAS_WAITER; + + if (Mutex.compare_exchange_strong(Expected, Desired)) { + // We have latched, now futex. + constexpr int Op = FUTEX_WAIT | FUTEX_PRIVATE_FLAG; + // WAIT will keep sleeping on the futex word while it is `val` + int Result = ::syscall(SYS_futex, &Mutex, Op, + Desired, // val + Timeout, // Timeout/val2 + nullptr, // Addr2 + 0); // val3 + + if (Timeout && Result == -1 && errno == ETIMEDOUT) { + return false; + } + } else if (Expected == HAS_SIGNALED) { + // Reset the latch once signaled + Mutex.store(HAS_NO_WAITER); + return true; + } + } + } + + template + bool WaitFor(const std::chrono::duration& time) { + struct timespec Timeout {}; + auto SecondsDuration = std::chrono::duration_cast(time); + Timeout.tv_sec = SecondsDuration.count(); + Timeout.tv_nsec = std::chrono::duration_cast(time - SecondsDuration).count(); + return Wait(&Timeout); + } + + void NotifyOne() { + DoNotify(1); + } + + bool HasWaiter() const { + return Mutex.load() == HAS_WAITER; + } + +private: + std::atomic Mutex {}; + constexpr static uint32_t HAS_NO_WAITER = 0; + constexpr static uint32_t HAS_WAITER = 1; + constexpr static uint32_t HAS_SIGNALED = 2; + + void DoNotify(int Waiters) { + uint32_t Expected = HAS_WAITER; + const uint32_t Desired = HAS_SIGNALED; + + // If the mutex is in a waiting state and we have CAS exchanged it to HAS_SIGNALED, then futex. + // otherwise just leave since nothing was waiting. + if (Mutex.compare_exchange_strong(Expected, Desired)) { + constexpr int Op = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; + + ::syscall(SYS_futex, &Mutex, Op, + Waiters, // val - Number of waiters to wake + 0, // val2 + &Mutex, // Addr2 - Mutex to do the operation on + 0); // val3 + } + } +}; + enum class SignalEvent : uint32_t { Nothing, // If the guest uses our signal we need to know it was errant on our end Pause, @@ -81,8 +155,7 @@ struct ThreadStateObject : public FEXCore::Allocator::FEXAllocOperators { std::atomic SignalReason {SignalEvent::Nothing}; // Thread pause handling - std::atomic_bool ThreadSleeping {false}; - FEXCore::InterruptableConditionVariable ThreadPaused; + InspectableLatch ThreadSleeping; // GDB signal information struct GdbInfoStruct { @@ -116,10 +189,7 @@ class ThreadManager final { FEX::HLE::ThreadStateObject* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, const FEXCore::Core::CPUState* NewThreadState = nullptr, uint64_t ParentTID = 0, FEX::HLE::ThreadStateObject* InheritThread = nullptr); - void TrackThread(FEX::HLE::ThreadStateObject* Thread) { - std::lock_guard lk(ThreadCreationMutex); - Threads.emplace_back(Thread); - } + void TrackThread(FEX::HLE::ThreadStateObject* Thread); void DestroyThread(FEX::HLE::ThreadStateObject* Thread, bool NeedsTLSUninstall = false); void StopThread(FEX::HLE::ThreadStateObject* Thread); @@ -134,7 +204,11 @@ class ThreadManager final { void WaitForIdleWithTimeout(); void WaitForThreadsToRun(); - void SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame); + void SleepThread(FEXCore::Context::Context* CTX, FEX::HLE::ThreadStateObject* ThreadObject); + void SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame) { + auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromCPUState(Frame); + SleepThread(CTX, ThreadObject); + } void UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child); @@ -173,6 +247,11 @@ class ThreadManager final { return &Threads; } + size_t GetThreadCount() { + std::lock_guard lk(ThreadCreationMutex); + return Threads.size(); + } + private: FEXCore::Context::Context* CTX; FEX::HLE::SyscallHandler* SyscallHandler;