GdbServer: Changes how thread addition and pausing works

This is enough to get GdbServer working for my simple test case again.

Some things of note:
- There is now a callback which signals to GdbServer when a thread is
  created.
- Thread sleeping is now a single uint32_t futex word for knowing when a
  thread is sleeping and if we need to signal it to wake up.
- GdbServer thread sleeping versus ThreadManager thread sleeping
  distinction has been removed. GdbServer now uses ThreadManager to put
  a thread to sleep.

This still only gets the attach at startup scheme of gdbserver working.
I haven't yet dived back in to getting arbitrary attach working.
Primarily the work necessary here is actually getting thread creation at
the start to stop and wait for gdbserver to attach. Then actually
sleeping correctly while gdbserver is communicating with gdb.

Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp

@@ -10,6 +10,7 @@ desc: Provides a gdb interface to the guest state
 #include "GdbServer/Info.h"
 
 #include "LinuxSyscalls/NetStream.h"
+#include "LinuxSyscalls/SignalDelegator.h"
 
 #include <cstdlib>
 #include <cstdio>
@@ -75,11 +76,6 @@ void GdbServer::Break(FEX::HLE::ThreadStateObject* ThreadObject, int signal) {
   SendPacket(*CommsStream, str);
 }
 
-void GdbServer::WaitForThreadWakeup() {
-  // Wait for gdbserver to tell us to wake up
-  ThreadBreakEvent.Wait();
-}
-
 GdbServer::~GdbServer() {
   CloseListenSocket();
   CoreShuttingDown = true;
@@ -120,7 +116,7 @@ GdbServer::GdbServer(FEXCore::Context::Context* ctx, FEX::HLE::SignalDelegator*
       // Let GDB know that we have a signal
       this->Break(ThreadObject, Signal);
 
-      WaitForThreadWakeup();
+      this->SyscallHandler->TM.SleepThread(this->CTX, ThreadObject);
 
       return true;
       },
@@ -604,8 +600,11 @@ GdbServer::HandledPacketType GdbServer::handleProgramOffsets() {
 GdbServer::HandledPacketType GdbServer::ThreadAction(char action, uint32_t tid) {
   switch (action) {
   case 'c': {
+    auto Threads = SyscallHandler->TM.GetThreads();
+    for (auto& Thread : *Threads) {
+      Thread->ThreadSleeping.NotifyOne();
+    }
     SyscallHandler->TM.Run();
-    ThreadBreakEvent.NotifyAll();
     SyscallHandler->TM.WaitForThreadsToRun();
     return {"", HandledPacketType::TYPE_ONLYACK};
   }
@@ -1431,6 +1430,13 @@ void GdbServer::StartThread() {
   FEXCore::Threads::SetSignalMask(OldMask);
 }
 
+void GdbServer::CreateThreadCallback(FEX::HLE::ThreadStateObject* ThreadObject) {
+  if (SyscallHandler->TM.GetThreadCount() == 1) {
+    // Sleep the first thread created. This is because FEX only currently supports attaching at startup.
+    SyscallHandler->TM.SleepThread(CTX, ThreadObject);
+  }
+}
+
 void GdbServer::OpenListenSocket() {
   const auto GdbUnixPath = fextl::fmt::format("{}/FEX_gdbserver/", FEXServerClient::GetTempFolder());
   if (FHU::Filesystem::CreateDirectory(GdbUnixPath) == FHU::Filesystem::CreateDirectoryResult::ERROR) {

Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h

@@ -35,6 +35,8 @@ class GdbServer {
     LibraryMapChanged = true;
   }
 
+  void CreateThreadCallback(FEX::HLE::ThreadStateObject* ThreadObject);
+
 private:
   void Break(FEX::HLE::ThreadStateObject* ThreadObject, int signal);
 
@@ -52,9 +54,6 @@ class GdbServer {
 
   void SendACK(std::ostream& stream, bool NACK);
 
-  Event ThreadBreakEvent {};
-  void WaitForThreadWakeup();
-
   struct HandledPacketType {
     fextl::string Response {};
     enum ResponseType {

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp

@@ -475,7 +475,7 @@ bool SignalDelegator::HandleSignalPause(FEXCore::Core::InternalThreadState* Thre
     // We need to be a little bit careful here
     // If we were already paused (due to GDB) and we are immediately stopping (due to gdb kill)
     // Then we need to ensure we don't double decrement our idle thread counter
-    if (ThreadObject->ThreadSleeping) {
+    if (ThreadObject->ThreadSleeping.HasWaiter()) {
       // If the thread was sleeping then its idle counter was decremented
       // Reincrement it here to not break logic
       FEX::HLE::_SyscallHandler->TM.IncrementIdleRefCount();

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h

@@ -53,7 +53,7 @@ desc: Glue logic, STRACE magic
 namespace FEX {
 class CodeLoader;
 class GdbServer;
-}
+} // namespace FEX
 
 namespace FEXCore {
 namespace Context {

Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: MIT
 
+#include "LinuxSyscalls/GdbServer.h"
 #include "LinuxSyscalls/Syscalls.h"
 #include "LinuxSyscalls/SignalDelegator.h"
 
@@ -29,6 +30,16 @@ FEX::HLE::ThreadStateObject* ThreadManager::CreateThread(uint64_t InitialRIP, ui
   return ThreadStateObject;
 }
 
+void ThreadManager::TrackThread(FEX::HLE::ThreadStateObject* Thread) {
+  {
+    std::lock_guard lk(ThreadCreationMutex);
+    Threads.emplace_back(Thread);
+  }
+
+  auto GdbServer = SyscallHandler->GetGdbServer();
+  GdbServer->CreateThreadCallback(Thread);
+}
+
 void ThreadManager::DestroyThread(FEX::HLE::ThreadStateObject* Thread, bool NeedsTLSUninstall) {
   {
     std::lock_guard lk(ThreadCreationMutex);
@@ -71,7 +82,10 @@ void ThreadManager::NotifyPause() {
   // Tell all the threads that they should pause
   std::lock_guard lk(ThreadCreationMutex);
   for (auto& Thread : Threads) {
-    SignalDelegation->SignalThread(Thread->Thread, SignalEvent::Pause);
+    if (Thread->ThreadSleeping.HasWaiter() == false) {
+      // Only signal if it isn't already sleeping.
+      SignalDelegation->SignalThread(Thread->Thread, SignalEvent::Pause);
+    }
   }
 }
 
@@ -162,25 +176,20 @@ void ThreadManager::Stop(bool IgnoreCurrentThread) {
   }
 }
 
-void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame) {
-  auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromCPUState(Frame);
-
+void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEX::HLE::ThreadStateObject* ThreadObject) {
   --IdleWaitRefCount;
   IdleWaitCV.notify_all();
 
-  ThreadObject->ThreadSleeping = true;
-
-  // Go to sleep
-  ThreadObject->ThreadPaused.Wait();
+  // Go to sleep.
+  ThreadObject->ThreadSleeping.Wait();
 
   ++IdleWaitRefCount;
-  ThreadObject->ThreadSleeping = false;
 
   IdleWaitCV.notify_all();
 }
 
 void ThreadManager::UnpauseThread(FEX::HLE::ThreadStateObject* Thread) {
-  Thread->ThreadPaused.NotifyOne();
+  Thread->ThreadSleeping.NotifyOne();
 }
 
 void ThreadManager::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThread, bool Child) {

Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h

@@ -22,6 +22,80 @@ namespace FEX::HLE {
 class SyscallHandler;
 class SignalDelegator;
 
+// This is similar to FEXCore::InterruptableConditionVariable with the exception that notifying only occurs if there is a waiter and can check if
+// there is a waiter.
+// This allows us to remove the race condition between a thread trying to go asleep and something else telling it to go to sleep or wake up.
+//
+// Only one thread can ever wait on a latch, while another thread signals it.
+class InspectableLatch final {
+public:
+  bool Wait(struct timespec* Timeout = nullptr) {
+    while (true) {
+      uint32_t Expected = HAS_NO_WAITER;
+      const uint32_t Desired = HAS_WAITER;
+
+      if (Mutex.compare_exchange_strong(Expected, Desired)) {
+        // We have latched, now futex.
+        constexpr int Op = FUTEX_WAIT | FUTEX_PRIVATE_FLAG;
+        // WAIT will keep sleeping on the futex word while it is `val`
+        int Result = ::syscall(SYS_futex, &Mutex, Op,
+                               Desired, // val
+                               Timeout, // Timeout/val2
+                               nullptr, // Addr2
+                               0);      // val3
+
+        if (Timeout && Result == -1 && errno == ETIMEDOUT) {
+          return false;
+        }
+      } else if (Expected == HAS_SIGNALED) {
+        // Reset the latch once signaled
+        Mutex.store(HAS_NO_WAITER);
+        return true;
+      }
+    }
+  }
+
+  template<class Rep, class Period>
+  bool WaitFor(const std::chrono::duration<Rep, Period>& time) {
+    struct timespec Timeout {};
+    auto SecondsDuration = std::chrono::duration_cast<std::chrono::seconds>(time);
+    Timeout.tv_sec = SecondsDuration.count();
+    Timeout.tv_nsec = std::chrono::duration_cast<std::chrono::nanoseconds>(time - SecondsDuration).count();
+    return Wait(&Timeout);
+  }
+
+  void NotifyOne() {
+    DoNotify(1);
+  }
+
+  bool HasWaiter() const {
+    return Mutex.load() == HAS_WAITER;
+  }
+
+private:
+  std::atomic<uint32_t> Mutex {};
+  constexpr static uint32_t HAS_NO_WAITER = 0;
+  constexpr static uint32_t HAS_WAITER = 1;
+  constexpr static uint32_t HAS_SIGNALED = 2;
+
+  void DoNotify(int Waiters) {
+    uint32_t Expected = HAS_WAITER;
+    const uint32_t Desired = HAS_SIGNALED;
+
+    // If the mutex is in a waiting state and we have CAS exchanged it to HAS_SIGNALED, then futex.
+    // otherwise just leave since nothing was waiting.
+    if (Mutex.compare_exchange_strong(Expected, Desired)) {
+      constexpr int Op = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
+
+      ::syscall(SYS_futex, &Mutex, Op,
+                Waiters, // val - Number of waiters to wake
+                0,       // val2
+                &Mutex,  // Addr2 - Mutex to do the operation on
+                0);      // val3
+    }
+  }
+};
+
 enum class SignalEvent : uint32_t {
   Nothing, // If the guest uses our signal we need to know it was errant on our end
   Pause,
@@ -81,8 +155,7 @@ struct ThreadStateObject : public FEXCore::Allocator::FEXAllocOperators {
   std::atomic<SignalEvent> SignalReason {SignalEvent::Nothing};
 
   // Thread pause handling
-  std::atomic_bool ThreadSleeping {false};
-  FEXCore::InterruptableConditionVariable ThreadPaused;
+  InspectableLatch ThreadSleeping;
 
   // GDB signal information
   struct GdbInfoStruct {
@@ -116,10 +189,7 @@ class ThreadManager final {
 
   FEX::HLE::ThreadStateObject* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, const FEXCore::Core::CPUState* NewThreadState = nullptr,
                                             uint64_t ParentTID = 0, FEX::HLE::ThreadStateObject* InheritThread = nullptr);
-  void TrackThread(FEX::HLE::ThreadStateObject* Thread) {
-    std::lock_guard lk(ThreadCreationMutex);
-    Threads.emplace_back(Thread);
-  }
+  void TrackThread(FEX::HLE::ThreadStateObject* Thread);
 
   void DestroyThread(FEX::HLE::ThreadStateObject* Thread, bool NeedsTLSUninstall = false);
   void StopThread(FEX::HLE::ThreadStateObject* Thread);
@@ -134,7 +204,11 @@ class ThreadManager final {
   void WaitForIdleWithTimeout();
   void WaitForThreadsToRun();
 
-  void SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame);
+  void SleepThread(FEXCore::Context::Context* CTX, FEX::HLE::ThreadStateObject* ThreadObject);
+  void SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame) {
+    auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromCPUState(Frame);
+    SleepThread(CTX, ThreadObject);
+  }
 
   void UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child);
 
@@ -173,6 +247,11 @@ class ThreadManager final {
     return &Threads;
   }
 
+  size_t GetThreadCount() {
+    std::lock_guard lk(ThreadCreationMutex);
+    return Threads.size();
+  }
+
 private:
   FEXCore::Context::Context* CTX;
   FEX::HLE::SyscallHandler* SyscallHandler;