FEXCore: Move InternalThreadState StartRunning to frontend

We were using this variable for two things, letting the frontend signal
to the backend that it wants to start executing once the thread is
created, and also for handling thread pausing. These two features are
conflated with one another and actually makes things more confusing.

- Move StartRunning/StartPaused to the frontend, because its a construct
  that only needs to exist in the frontend
- Adds a FEX::HLE::ThreadStateObject CV for handling pausing, which only
  needs to exist for gdbserver

Relies on FEX-Emu#4170 and FEX-Emu#4179 being merged first. Which is what the first
two commits are from.

FEXCore/Source/Interface/Core/Core.cpp

@@ -873,11 +873,6 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
 void ContextImpl::ExecutionThread(FEXCore::Core::InternalThreadState* Thread) {
   Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_WAITING;
 
-  if (Thread->StartPaused) {
-    // Parent thread doesn't need to wait to run
-    Thread->StartRunning.Wait();
-  }
-
   Thread->RunningEvents.WaitingToStart = false;
 
   Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_NONE;

FEXCore/include/FEXCore/Debug/InternalThreadState.h

@@ -89,8 +89,6 @@ struct InternalThreadState : public FEXCore::Allocator::FEXAllocOperators {
   FEXCore::Context::Context* const CTX;
 
   NonMovableUniquePtr<FEXCore::Threads::Thread> ExecutionThread;
-  bool StartPaused {false};
-  InterruptableConditionVariable StartRunning;
 
   NonMovableUniquePtr<FEXCore::IR::OpDispatchBuilder> OpDispatcher;
 

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.cpp

@@ -659,9 +659,6 @@ uint64_t CloneHandler(FEXCore::Core::CpuStateFrame* Frame, FEX::HLE::clone3_args
     // Return the new threads TID
     uint64_t Result = NewThread->ThreadInfo.TID;
 
-    // Actually start the thread
-    FEX::HLE::_SyscallHandler->TM.RunThread(NewThread);
-
     if (flags & CLONE_VFORK) {
       // If VFORK is set then the calling process is suspended until the thread exits with execve or exit
       NewThread->Thread->ExecutionThread->join(nullptr);

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp

@@ -48,13 +48,20 @@ struct ExecutionThreadHandler {
   FEXCore::Context::Context* CTX;
   FEX::HLE::ThreadStateObject* Thread;
   Event* ThreadWaiting;
+
+  // Pause on thread start handling.
+  FEXCore::InterruptableConditionVariable* StartRunningCV{};
+  FEXCore::InterruptableConditionVariable* StartRunningResponse{};
 };
 
 static void* ThreadHandler(void* Data) {
   ExecutionThreadHandler* Handler = reinterpret_cast<ExecutionThreadHandler*>(Data);
   auto CTX = Handler->CTX;
   auto Thread = Handler->Thread;
   auto ThreadWaiting = Handler->ThreadWaiting;
+  auto StartRunningCV = Handler->StartRunningCV;
+  auto StartRunningResponse = Handler->StartRunningResponse;
+
   FEXCore::Allocator::free(Handler);
 
   Thread->ThreadInfo.PID = ::getpid();
@@ -65,6 +72,9 @@ static void* ThreadHandler(void* Data) {
   // Now notify the thread that we are initialized
   ThreadWaiting->NotifyOne();
 
+  StartRunningCV->Wait();
+  StartRunningResponse->NotifyOne();
+
   CTX->ExecutionThread(Thread->Thread);
   FEX::HLE::_SyscallHandler->UninstallTLSState(Thread);
   FEX::HLE::_SyscallHandler->TM.DestroyThread(Thread);
@@ -98,15 +108,18 @@ FEX::HLE::ThreadStateObject* CreateNewThread(FEXCore::Context::Context* CTX, FEX
     x32::AdjustRipForNewThread(NewThread->Thread->CurrentFrame);
   }
 
-  // We need to do some post-thread creation setup.
-  NewThread->Thread->StartPaused = true;
-
   // Initialize a new thread for execution.
   Event ThreadWaitingEvent {};
+  FEXCore::InterruptableConditionVariable StartRunningCV{};
+  FEXCore::InterruptableConditionVariable StartRunningResponse{};
+
   ExecutionThreadHandler* Arg = reinterpret_cast<ExecutionThreadHandler*>(FEXCore::Allocator::malloc(sizeof(ExecutionThreadHandler)));
   Arg->CTX = CTX;
   Arg->Thread = NewThread;
   Arg->ThreadWaiting = &ThreadWaitingEvent;
+  Arg->StartRunningCV = &StartRunningCV;
+  Arg->StartRunningResponse = &StartRunningResponse;
+
   NewThread->Thread->ExecutionThread = FEXCore::Threads::Thread::Create(ThreadHandler, Arg);
 
   // Wait for the thread to have started.
@@ -155,6 +168,12 @@ FEX::HLE::ThreadStateObject* CreateNewThread(FEXCore::Context::Context* CTX, FEX
 
   FEX::HLE::_SyscallHandler->TM.TrackThread(NewThread);
 
+  // Start running the thread
+  StartRunningCV.NotifyOne();
+
+  // Wait for the thread to start running.
+  StartRunningResponse.Wait();
+
   return NewThread;
 }
 
@@ -179,7 +198,6 @@ uint64_t HandleNewClone(FEX::HLE::ThreadStateObject* Thread, FEXCore::Context::C
 
     // CLONE_PARENT_SETTID, CLONE_CHILD_SETTID, CLONE_CHILD_CLEARTID, CLONE_PIDFD will be handled by kernel
     // Call execution thread directly since we already are on the new thread
-    NewThread->Thread->StartRunning.NotifyAll(); // Clear the start running flag
     CreatedNewThreadObject = true;
   } else {
     // If we don't have CLONE_THREAD then we are effectively a fork

Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp

@@ -46,11 +46,6 @@ void ThreadManager::StopThread(FEX::HLE::ThreadStateObject* Thread) {
   }
 }
 
-void ThreadManager::RunThread(FEX::HLE::ThreadStateObject* Thread) {
-  // Tell the thread to start executing
-  Thread->Thread->StartRunning.NotifyAll();
-}
-
 void ThreadManager::HandleThreadDeletion(FEX::HLE::ThreadStateObject* Thread, bool NeedsTLSUninstall) {
   if (Thread->Thread->ExecutionThread) {
     if (Thread->Thread->ExecutionThread->joinable()) {
@@ -93,10 +88,6 @@ void ThreadManager::Run() {
   for (auto& Thread : Threads) {
     Thread->SignalReason.store(SignalEvent::Return);
   }
-
-  for (auto& Thread : Threads) {
-    Thread->Thread->StartRunning.NotifyAll();
-  }
 }
 
 void ThreadManager::WaitForIdleWithTimeout() {
@@ -166,12 +157,6 @@ void ThreadManager::Stop(bool IgnoreCurrentThread) {
       if (Thread->Thread->RunningEvents.Running.load()) {
         StopThread(Thread);
       }
-
-      // If the thread is waiting to start but immediately killed then there can be a hang
-      // This occurs in the case of gdb attach with immediate kill
-      if (Thread->Thread->RunningEvents.WaitingToStart.load()) {
-        Thread->Thread->StartRunning.NotifyAll();
-      }
     }
   }
 
@@ -182,6 +167,7 @@ void ThreadManager::Stop(bool IgnoreCurrentThread) {
 }
 
 void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame) {
+  auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromCPUState(Frame);
   auto Thread = Frame->Thread;
 
   --IdleWaitRefCount;
@@ -190,7 +176,7 @@ void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::C
   Thread->RunningEvents.ThreadSleeping = true;
 
   // Go to sleep
-  Thread->StartRunning.Wait();
+  ThreadObject->ThreadPaused.Wait();
 
   Thread->RunningEvents.Running = true;
   ++IdleWaitRefCount;
@@ -199,6 +185,10 @@ void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::C
   IdleWaitCV.notify_all();
 }
 
+void ThreadManager::UnpauseThread(FEX::HLE::ThreadStateObject* Thread) {
+  Thread->ThreadPaused.NotifyOne();
+}
+
 void ThreadManager::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThread, bool Child) {
   if (!Child) {
     return;

Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h

@@ -78,6 +78,9 @@ struct ThreadStateObject : public FEXCore::Allocator::FEXAllocOperators {
   // Thread signaling information
   std::atomic<SignalEvent> SignalReason {SignalEvent::Nothing};
 
+  // Thread pause handling
+  FEXCore::InterruptableConditionVariable ThreadPaused;
+
   int StatusCode {};
 };
 
@@ -108,7 +111,7 @@ class ThreadManager final {
 
   void DestroyThread(FEX::HLE::ThreadStateObject* Thread, bool NeedsTLSUninstall = false);
   void StopThread(FEX::HLE::ThreadStateObject* Thread);
-  void RunThread(FEX::HLE::ThreadStateObject* Thread);
+  void UnpauseThread(FEX::HLE::ThreadStateObject* Thread);
 
   void Pause();
   void Run();