Merge pull request #3598 from Sonicadvance1/half_barrier_delete_hack

Arm64: Adds another TSO hack to disable half-barrier TSO

FEXCore/Source/Interface/Config/Config.json.in

@@ -414,6 +414,14 @@
           "Only affects REP MOVS and REP STOS instructions"
         ]
       },
+      "HalfBarrierTSOEnabled": {
+        "Type": "bool",
+        "Default": "true",
+        "Desc": [
+          "When TSO emulation is enabled, controls if unaligned loads and stores should be backpatched to half-barrier atomics.",
+          "Can be dangerous due to aligned loadstores through the same code now become non-atomic."
+        ]
+      },
       "TSOAutoMigration": {
         "Type": "bool",
         "Default": "true",

FEXCore/Source/Utils/ArchHelpers/Arm64.cpp

@@ -1822,7 +1822,7 @@ static uint64_t HandleAtomicLoadstoreExclusive(uintptr_t ProgramCounter, uint64_
 
 [[nodiscard]]
 std::pair<bool, int32_t>
-HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, bool ParanoidTSO, uintptr_t ProgramCounter, uint64_t* GPRs) {
+HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandlerType HandleType, uintptr_t ProgramCounter, uint64_t* GPRs) {
 #ifdef _M_ARM_64
   constexpr bool is_arm64 = true;
 #else
@@ -1843,7 +1843,7 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, bool ParanoidT
     uint32_t DataReg = Instr & 0x1F;
 
     // ParanoidTSO path doesn't modify any code.
-    if (ParanoidTSO) [[unlikely]] {
+    if (HandleType == UnalignedHandlerType::Paranoid) [[unlikely]] {
       if ((Instr & LDAXR_MASK) == LDAR_INST ||  // LDAR*
           (Instr & LDAXR_MASK) == LDAPR_INST) { // LDAPR*
         if (ArchHelpers::Arm64::HandleAtomicLoad(Instr, GPRs, 0)) {
@@ -1900,16 +1900,20 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, bool ParanoidT
       LDR |= AddrReg << 5;
       LDR |= DataReg;
       PC[0] = LDR;
-      PC[1] = DMB_LD; // Back-patch the half-barrier.
-      ClearICache(&PC[-1], 16);
+      if (HandleType != UnalignedHandlerType::NonAtomic) {
+        PC[1] = DMB_LD; // Back-patch the half-barrier.
+      }
+      ClearICache(&PC[0], 16);
       // With the instruction modified, now execute again.
       return std::make_pair(true, 0);
     } else if ((Instr & LDAXR_MASK) == STLR_INST) { // STLR*
       uint32_t STR = 0b0011'1000'0011'1111'0110'1000'0000'0000;
       STR |= Size << 30;
       STR |= AddrReg << 5;
       STR |= DataReg;
-      PC[-1] = DMB; // Back-patch the half-barrier.
+      if (HandleType != UnalignedHandlerType::NonAtomic) {
+        PC[-1] = DMB; // Back-patch the half-barrier.
+      }
       PC[0] = STR;
       ClearICache(&PC[-1], 16);
       // Back up one instruction and have another go
@@ -1922,8 +1926,10 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, bool ParanoidT
       LDUR |= DataReg;
       LDUR |= Instr & (0b1'1111'1111 << 9);
       PC[0] = LDUR;
-      PC[1] = DMB_LD; // Back-patch the half-barrier.
-      ClearICache(&PC[-1], 16);
+      if (HandleType != UnalignedHandlerType::NonAtomic) {
+        PC[1] = DMB_LD; // Back-patch the half-barrier.
+      }
+      ClearICache(&PC[0], 16);
       // With the instruction modified, now execute again.
       return std::make_pair(true, 0);
     } else if ((Instr & RCPC2_MASK) == STLUR_INST) { // STLUR*
@@ -1932,7 +1938,9 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, bool ParanoidT
       STUR |= AddrReg << 5;
       STUR |= DataReg;
       STUR |= Instr & (0b1'1111'1111 << 9);
-      PC[-1] = DMB; // Back-patch the half-barrier.
+      if (HandleType != UnalignedHandlerType::NonAtomic) {
+        PC[-1] = DMB; // Back-patch the half-barrier.
+      }
       PC[0] = STUR;
       ClearICache(&PC[-1], 16);
       // Back up one instruction and have another go

FEXCore/Source/Utils/ArchHelpers/Arm64_stubs.cpp

@@ -24,7 +24,7 @@ bool HandleAtomicMemOp(void* _ucontext, void* _info, uint32_t Instr) {
 }
 
 std::pair<bool, int32_t>
-HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, bool ParanoidTSO, uintptr_t ProgramCounter, uint64_t* GPRs) {
+HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandlerType HandleType, uintptr_t ProgramCounter, uint64_t* GPRs) {
   ERROR_AND_DIE_FMT("HandleAtomicMemOp Not Implemented");
 }
 

FEXCore/include/FEXCore/Utils/ArchHelpers/Arm64.h

@@ -108,6 +108,15 @@ inline uint32_t GetRmReg(uint32_t Instr) {
   return (Instr >> RM_OFFSET) & REGISTER_MASK;
 }
 
+enum class UnalignedHandlerType {
+  ///< Don't backpatch code, instead handle inside SIGBUS handler.
+  Paranoid,
+  ///< Backpatch unaligned access to half-barrier based atomic.
+  HalfBarrier,
+  ///< Backpatch unaligned access to non-atomic.
+  NonAtomic,
+};
+
 /**
  * @brief On ARM64 handles an unaligned memory access that the JIT has done.
  *
@@ -123,5 +132,5 @@ inline uint32_t GetRmReg(uint32_t Instr) {
  */
 [[nodiscard]]
 FEX_DEFAULT_VISIBILITY std::pair<bool, int32_t>
-HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, bool ParanoidTSO, uintptr_t ProgramCounter, uint64_t* GPRs);
+HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandlerType HandleType, uintptr_t ProgramCounter, uint64_t* GPRs);
 } // namespace FEXCore::ArchHelpers::Arm64

Source/Tools/FEXConfig/Main.cpp

@@ -556,10 +556,12 @@ void FillHackConfig() {
     auto Value = LoadedConfig->Get(FEXCore::Config::ConfigOption::CONFIG_TSOENABLED);
     auto VectorTSO = LoadedConfig->Get(FEXCore::Config::ConfigOption::CONFIG_VECTORTSOENABLED);
     auto MemcpyTSO = LoadedConfig->Get(FEXCore::Config::ConfigOption::CONFIG_MEMCPYSETTSOENABLED);
+    auto HalfBarrierTSO = LoadedConfig->Get(FEXCore::Config::ConfigOption::CONFIG_HALFBARRIERTSOENABLED);
 
     bool TSOEnabled = Value.has_value() && **Value == "1";
     bool VectorTSOEnabled = VectorTSO.has_value() && **VectorTSO == "1";
     bool MemcpyTSOEnabled = MemcpyTSO.has_value() && **MemcpyTSO == "1";
+    bool HalfBarrierTSOEnabled = HalfBarrierTSO.has_value() && **HalfBarrierTSO == "1";
 
     if (ImGui::Checkbox("TSO Enabled", &TSOEnabled)) {
       LoadedConfig->EraseSet(FEXCore::Config::ConfigOption::CONFIG_TSOENABLED, TSOEnabled ? "1" : "0");
@@ -588,6 +590,16 @@ void FillHackConfig() {
           ImGui::EndTooltip();
         }
 
+        if (ImGui::Checkbox("Unaligned Half-Barrier TSO Enabled", &HalfBarrierTSOEnabled)) {
+          LoadedConfig->EraseSet(FEXCore::Config::ConfigOption::CONFIG_HALFBARRIERTSOENABLED, HalfBarrierTSOEnabled ? "1" : "0");
+          ConfigChanged = true;
+        }
+        if (ImGui::IsItemHovered()) {
+          ImGui::BeginTooltip();
+          ImGui::Text("Disables half-barrier TSO emulation on unaligned load/store instructions");
+          ImGui::EndTooltip();
+        }
+
         ImGui::TreePop();
       }
     }

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp

@@ -1627,6 +1627,14 @@ SignalDelegator::SignalDelegator(FEXCore::Context::Context* _CTX, const std::str
   HostHandlers[SIGKILL].Installed = true;
   HostHandlers[SIGSTOP].Installed = true;
 
+  if (ParanoidTSO()) {
+    UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::Paranoid;
+  } else if (HalfBarrierTSOEnabled()) {
+    UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::HalfBarrier;
+  } else {
+    UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::NonAtomic;
+  }
+
   // Most signals default to termination
   // These ones are slightly different
   static constexpr std::array<std::pair<int, SignalDelegator::DefaultBehaviour>, 14> SignalDefaultBehaviours = {{
@@ -1703,7 +1711,7 @@ SignalDelegator::SignalDelegator(FEXCore::Context::Context* _CTX, const std::str
       return false;
     }
 
-    const auto Result = FEXCore::ArchHelpers::Arm64::HandleUnalignedAccess(Thread, GlobalDelegator->ParanoidTSO(), PC,
+    const auto Result = FEXCore::ArchHelpers::Arm64::HandleUnalignedAccess(Thread, GlobalDelegator->GetUnalignedHandlerType(), PC,
                                                                            ArchHelpers::Context::GetArmGPRs(ucontext));
     ArchHelpers::Context::SetPc(ucontext, PC + Result.second);
     return Result.first;

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.h

@@ -23,6 +23,7 @@ tags: LinuxSyscalls|common
 #include <mutex>
 
 #include <FEXCore/Core/SignalDelegator.h>
+#include <FEXCore/Utils/ArchHelpers/Arm64.h>
 #include <FEXCore/Utils/Telemetry.h>
 
 namespace FEXCore {
@@ -127,7 +128,9 @@ class SignalDelegator final : public FEXCore::SignalDelegator, public FEXCore::A
 
   void SignalThread(FEXCore::Core::InternalThreadState* Thread, FEXCore::Core::SignalEvent Event) override;
 
-  FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
+  FEXCore::ArchHelpers::Arm64::UnalignedHandlerType GetUnalignedHandlerType() const {
+    return UnalignedHandlerType;
+  }
 
   void SaveTelemetry();
 private:
@@ -156,6 +159,10 @@ class SignalDelegator final : public FEXCore::SignalDelegator, public FEXCore::A
   fextl::string const ApplicationName;
   FEXCORE_TELEMETRY_INIT(CrashMask, TYPE_CRASH_MASK);
   FEXCORE_TELEMETRY_INIT(UnhandledNonCanonical, TYPE_UNHANDLED_NONCANONICAL_ADDRESS);
+  FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
+  FEX_CONFIG_OPT(HalfBarrierTSOEnabled, HALFBARRIERTSOENABLED);
+
+  FEXCore::ArchHelpers::Arm64::UnalignedHandlerType UnalignedHandlerType {FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::HalfBarrier};
 
   enum DefaultBehaviour {
     DEFAULT_TERM,

Source/Windows/WOW64/Module.cpp

@@ -263,14 +263,39 @@ WOW64_CONTEXT ReconstructWowContext(CONTEXT* Context) {
   return WowContext;
 }
 
+class TSOHandlerConfig final {
+public:
+  TSOHandlerConfig() {
+    if (ParanoidTSO()) {
+      UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::Paranoid;
+    } else if (HalfBarrierTSOEnabled()) {
+      UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::HalfBarrier;
+    } else {
+      UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::NonAtomic;
+    }
+  }
+
+  FEXCore::ArchHelpers::Arm64::UnalignedHandlerType GetUnalignedHandlerType() const {
+    return UnalignedHandlerType;
+  }
+
+private:
+  FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
+  FEX_CONFIG_OPT(HalfBarrierTSOEnabled, HALFBARRIERTSOENABLED);
+
+  FEXCore::ArchHelpers::Arm64::UnalignedHandlerType UnalignedHandlerType {FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::HalfBarrier};
+};
+
+static std::optional<TSOHandlerConfig> HandlerConfig;
+
 bool HandleUnalignedAccess(CONTEXT* Context) {
   auto Thread = GetTLS().ThreadState();
   if (!Thread->CTX->IsAddressInCodeBuffer(Thread, Context->Pc)) {
     return false;
   }
 
-  FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
-  const auto Result = FEXCore::ArchHelpers::Arm64::HandleUnalignedAccess(Thread, ParanoidTSO(), Context->Pc, &Context->X0);
+  const auto Result =
+    FEXCore::ArchHelpers::Arm64::HandleUnalignedAccess(Thread, HandlerConfig->GetUnalignedHandlerType(), Context->Pc, &Context->X0);
   if (!Result.first) {
     return false;
   }
@@ -418,6 +443,7 @@ void BTCpuProcessInit() {
 
   SignalDelegator = fextl::make_unique<FEX::DummyHandlers::DummySignalDelegator>();
   SyscallHandler = fextl::make_unique<WowSyscallHandler>();
+  Context::HandlerConfig.emplace();
 
   CTX = FEXCore::Context::Context::CreateNewContext();
   CTX->SetSignalDelegator(SignalDelegator.get());