GDBServer fixes

FEXCore/Source/Interface/Context/Context.h

@@ -407,7 +407,6 @@ class ContextImpl final : public FEXCore::Context::Context {
   IR::AOTIRCaptureCache IRCaptureCache;
   fextl::unique_ptr<FEXCore::CodeSerialize::CodeObjectSerializeService> CodeObjectCacheService;
 
-  bool StartPaused = false;
   bool IsMemoryShared = false;
   bool SupportsHardwareTSO = false;
   bool AtomicTSOEmulationEnabled = true;

FEXCore/Source/Interface/Core/Core.cpp

@@ -311,8 +311,6 @@ bool ContextImpl::InitCore() {
   if (Config.GdbServer) {
     // If gdbserver is enabled then this needs to be enabled.
     Config.NeedsPendingInterruptFaultCheck = true;
-    // FEX needs to start paused when gdb is enabled.
-    StartPaused = true;
   }
 
   return true;
@@ -867,7 +865,7 @@ void ContextImpl::ExecutionThread(FEXCore::Core::InternalThreadState* Thread) {
   // Now notify the thread that we are initialized
   Thread->ThreadWaiting.NotifyAll();
 
-  if (StartPaused || Thread->StartPaused) {
+  if (Thread->StartPaused) {
     // Parent thread doesn't need to wait to run
     Thread->StartRunning.Wait();
   }

Source/Tools/FEXLoader/FEXLoader.cpp

@@ -543,7 +543,7 @@ int main(int argc, char** argv, char** const envp) {
       FEX::AOT::AOTGenSection(CTX.get(), Section);
     }
   } else {
-    CTX->RunUntilExit(ParentThread);
+    SyscallHandler->TM.RunPrimaryThread(CTX.get(), ParentThread);
   }
 
   if (AOTEnabled) {

Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp

@@ -17,6 +17,7 @@ desc: Provides a gdb interface to the guest state
 #include <optional>
 
 #include <Common/FEXServerClient.h>
+#include <Common/StringUtil.h>
 #include <FEXCore/Config/Config.h>
 #include <FEXCore/Core/Context.h>
 #include <FEXCore/Core/CoreState.h>
@@ -101,7 +102,21 @@ void GdbServer::Break(int signal) {
   SendPacket(*CommsStream, str);
 }
 
+void GdbServer::BreakThread(FEXCore::Core::InternalThreadState* Thread, int signal) {
+  std::lock_guard lk(sendMutex);
+  if (!CommsStream) {
+    return;
+  }
+
+  const fextl::string str = fextl::fmt::format("T{:02x}thread:{:x};", signal, Thread->ThreadManager.GetTID());
+  SendPacket(*CommsStream, str);
+  // Current debugging thread switches to the thread that is breaking.
+  CurrentDebuggingThread = Thread->ThreadManager.GetTID();
+}
+
 void GdbServer::WaitForThreadWakeup() {
+  ThreadBreakEventInfo.Thread = nullptr;
+
   // Wait for gdbserver to tell us to wake up
   ThreadBreakEvent.Wait();
 }
@@ -142,8 +157,12 @@ GdbServer::GdbServer(FEXCore::Context::Context* ctx, FEX::HLE::SignalDelegator*
         return false;
       }
 
+      this->SyscallHandler->TM.Pausing(Thread);
+      this->ThreadBreakEventInfo.HostPC = ArchHelpers::Context::GetPc(ucontext);
+      this->ThreadBreakEventInfo.Thread = Thread;
+
       // Let GDB know that we have a signal
-      this->Break(Signal);
+      this->BreakThread(Thread, Signal);
 
       WaitForThreadWakeup();
 
@@ -197,6 +216,8 @@ static fextl::string getThreadName(uint32_t ThreadID) {
   const auto ThreadFile = fextl::fmt::format("/proc/{}/task/{}/comm", getpid(), ThreadID);
   fextl::string ThreadName;
   FEXCore::FileLoading::LoadFile(ThreadName, ThreadFile);
+  // Trim out the potential newline, breaks GDB if it exists.
+  FEX::StringUtil::trim(ThreadName);
   return ThreadName;
 }
 
@@ -340,7 +361,11 @@ fextl::string GdbServer::readRegs() {
 
   // Encode the GDB context definition
   memcpy(&GDB.gregs[0], &state.gregs[0], sizeof(GDB.gregs));
-  memcpy(&GDB.rip, &state.rip, sizeof(GDB.rip));
+  if (ThreadBreakEventInfo.Thread == CurrentThread) {
+    GDB.rip = CTX->RestoreRIPFromHostPC(CurrentThread, ThreadBreakEventInfo.HostPC);
+  } else {
+    memcpy(&GDB.rip, &state.rip, sizeof(GDB.rip));
+  }
 
   GDB.eflags = CTX->ReconstructCompactedEFLAGS(CurrentThread, false, nullptr, 0);
 

Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h

@@ -37,6 +37,7 @@ class GdbServer {
 
 private:
   void Break(int signal);
+  void BreakThread(FEXCore::Core::InternalThreadState* Thread, int signal);
 
   void OpenListenSocket();
   void CloseListenSocket();
@@ -53,6 +54,11 @@ class GdbServer {
   void SendACK(std::ostream& stream, bool NACK);
 
   Event ThreadBreakEvent {};
+  struct ThreadBreakEventInfoStruct {
+    uint64_t HostPC {};
+    FEXCore::Core::InternalThreadState* Thread {};
+  };
+  ThreadBreakEventInfoStruct ThreadBreakEventInfo {};
   void WaitForThreadWakeup();
 
   struct HandledPacketType {

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h

@@ -111,13 +111,15 @@ class ThreadManager final {
   void StopThread(FEXCore::Core::InternalThreadState* Thread);
   void RunThread(FEXCore::Core::InternalThreadState* Thread);
 
+  void RunPrimaryThread(FEXCore::Context::Context* CTX, FEXCore::Core::InternalThreadState* Thread);
+
   void Pause();
   void Run();
   void Step();
   void Stop(bool IgnoreCurrentThread = false);
+  void Pausing(FEXCore::Core::InternalThreadState* Thread);
 
   void WaitForIdle();
-  void WaitForIdleWithTimeout();
   void WaitForThreadsToRun();
 
   void SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame);
@@ -160,6 +162,8 @@ class ThreadManager final {
   }
 
 private:
+  FEX_CONFIG_OPT(GdbServer, GDBSERVER);
+
   FEXCore::Context::Context* CTX;
   FEX::HLE::SignalDelegator* SignalDelegation;
 
@@ -169,7 +173,6 @@ class ThreadManager final {
   // Thread idling support.
   bool Running {};
   std::mutex IdleWaitMutex;
-  std::condition_variable IdleWaitCV;
   std::atomic<uint32_t> IdleWaitRefCount {};
 
   void HandleThreadDeletion(FEXCore::Core::InternalThreadState* Thread);

Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp

@@ -8,10 +8,7 @@
 namespace FEX::HLE {
 FEXCore::Core::InternalThreadState*
 ThreadManager::CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState* NewThreadState, uint64_t ParentTID) {
-  auto Thread = CTX->CreateThread(InitialRIP, StackPointer, NewThreadState, ParentTID);
-
-  ++IdleWaitRefCount;
-  return Thread;
+  return CTX->CreateThread(InitialRIP, StackPointer, NewThreadState, ParentTID);
 }
 
 void ThreadManager::DestroyThread(FEXCore::Core::InternalThreadState* Thread) {
@@ -34,6 +31,19 @@ void ThreadManager::StopThread(FEXCore::Core::InternalThreadState* Thread) {
 void ThreadManager::RunThread(FEXCore::Core::InternalThreadState* Thread) {
   // Tell the thread to start executing
   Thread->StartRunning.NotifyAll();
+  ++IdleWaitRefCount;
+}
+
+void ThreadManager::RunPrimaryThread(FEXCore::Context::Context* CTX, FEXCore::Core::InternalThreadState* Thread) {
+  Thread->ThreadManager.TID = FHU::Syscalls::gettid();
+  Thread->ThreadManager.PID = ::getpid();
+
+  if (GdbServer()) {
+    Thread->StartRunning.Wait();
+  }
+
+  ++IdleWaitRefCount;
+  CTX->RunUntilExit(Thread);
 }
 
 void ThreadManager::HandleThreadDeletion(FEXCore::Core::InternalThreadState* Thread) {
@@ -49,14 +59,15 @@ void ThreadManager::HandleThreadDeletion(FEXCore::Core::InternalThreadState* Thr
 
   CTX->DestroyThread(Thread);
   --IdleWaitRefCount;
-  IdleWaitCV.notify_all();
 }
 
 void ThreadManager::NotifyPause() {
   // Tell all the threads that they should pause
   std::lock_guard lk(ThreadCreationMutex);
   for (auto& Thread : Threads) {
-    SignalDelegation->SignalThread(Thread, FEXCore::Core::SignalEvent::Pause);
+    if (Thread->RunningEvents.Running.load()) {
+      SignalDelegation->SignalThread(Thread, FEXCore::Core::SignalEvent::Pause);
+    }
   }
 }
 
@@ -65,6 +76,11 @@ void ThreadManager::Pause() {
   WaitForIdle();
 }
 
+void ThreadManager::Pausing(FEXCore::Core::InternalThreadState* Thread) {
+  Thread->RunningEvents.Running.store(false);
+  --IdleWaitRefCount;
+}
+
 void ThreadManager::Run() {
   // Spin up all the threads
   std::lock_guard lk(ThreadCreationMutex);
@@ -77,21 +93,6 @@ void ThreadManager::Run() {
   }
 }
 
-void ThreadManager::WaitForIdleWithTimeout() {
-  std::unique_lock<std::mutex> lk(IdleWaitMutex);
-  bool WaitResult = IdleWaitCV.wait_for(lk, std::chrono::milliseconds(1500), [this] { return IdleWaitRefCount.load() == 0; });
-
-  if (!WaitResult) {
-    // The wait failed, this will occur if we stepped in to a syscall
-    // That's okay, we just need to pause the threads manually
-    NotifyPause();
-  }
-
-  // We have sent every thread a pause signal
-  // Now wait again because they /will/ be going to sleep
-  WaitForIdle();
-}
-
 void ThreadManager::WaitForThreadsToRun() {
   size_t NumThreads {};
   {
@@ -100,8 +101,8 @@ void ThreadManager::WaitForThreadsToRun() {
   }
 
   // Spin while waiting for the threads to start up
-  std::unique_lock<std::mutex> lk(IdleWaitMutex);
-  IdleWaitCV.wait(lk, [this, NumThreads] { return IdleWaitRefCount.load() >= NumThreads; });
+  while (IdleWaitRefCount.load() < NumThreads)
+    ;
 
   Running = true;
 }
@@ -164,18 +165,14 @@ void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::C
   auto Thread = Frame->Thread;
 
   --IdleWaitRefCount;
-  IdleWaitCV.notify_all();
-
-  Thread->RunningEvents.ThreadSleeping = true;
+  Thread->RunningEvents.Running = false;
 
   // Go to sleep
   Thread->StartRunning.Wait();
 
   Thread->RunningEvents.Running = true;
   ++IdleWaitRefCount;
   Thread->RunningEvents.ThreadSleeping = false;
-
-  IdleWaitCV.notify_all();
 }
 
 void ThreadManager::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThread, bool Child) {
@@ -223,8 +220,8 @@ void ThreadManager::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThre
 }
 
 void ThreadManager::WaitForIdle() {
-  std::unique_lock<std::mutex> lk(IdleWaitMutex);
-  IdleWaitCV.wait(lk, [this] { return IdleWaitRefCount.load() == 0; });
+  while (IdleWaitRefCount.load() != 0)
+    ;
 
   Running = false;
 }