Support debugger on OS X.

ArchDebugger...() calls are now supported on OS X.

(Internal change: 1734239)

LICENSE.txt

@@ -330,3 +330,50 @@ OpenEXR/IlmBase/Half
 //
 ///////////////////////////////////////////////////////////////////////////
 
+============================================================
+Apple Technical Q&A QA1361 - Detecting the Debugger
+https://developer.apple.com/library/content/qa/qa1361/_index.html
+============================================================
+
+Sample code project: Detecting the Debugger
+Version: 1.0
+
+Abstract: Shows how to determine if code is being run under the debugger.
+
+IMPORTANT:  This Apple software is supplied to you by Apple
+Inc. ("Apple") in consideration of your agreement to the following
+terms, and your use, installation, modification or redistribution of
+this Apple software constitutes acceptance of these terms.  If you do
+not agree with these terms, please do not use, install, modify or
+redistribute this Apple software.
+
+In consideration of your agreement to abide by the following terms, and
+subject to these terms, Apple grants you a personal, non-exclusive
+license, under Apple's copyrights in this original Apple software (the
+"Apple Software"), to use, reproduce, modify and redistribute the Apple
+Software, with or without modifications, in source and/or binary forms;
+provided that if you redistribute the Apple Software in its entirety and
+without modifications, you must retain this notice and the following
+text and disclaimers in all such redistributions of the Apple Software.
+Neither the name, trademarks, service marks or logos of Apple Inc. may
+be used to endorse or promote products derived from the Apple Software
+without specific prior written permission from Apple.  Except as
+expressly stated in this notice, no other rights or licenses, express or
+implied, are granted by Apple herein, including but not limited to any
+patent rights that may be infringed by your derivative works or by other
+works in which the Apple Software may be incorporated.
+
+The Apple Software is provided by Apple on an "AS IS" basis.  APPLE
+MAKES NO WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION
+THE IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND
+OPERATION ALONE OR IN COMBINATION WITH YOUR PRODUCTS.
+
+IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION,
+MODIFICATION AND/OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED
+AND WHETHER UNDER THEORY OF CONTRACT, TORT (INCLUDING NEGLIGENCE),
+STRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

pxr/base/lib/arch/debugger.cpp

@@ -30,11 +30,7 @@
 #include "pxr/base/arch/export.h"
 #include "pxr/base/arch/stackTrace.h"
 #include "pxr/base/arch/systemInfo.h"
-
-#include <atomic>
-#if defined(ARCH_OS_WINDOWS)
-#include <Windows.h>
-#else
+#if defined(ARCH_OS_LINUX) || defined(ARCH_OS_DARWIN)
 #include "pxr/base/arch/inttypes.h"
 #include <sys/types.h>
 #include <sys/ptrace.h>
@@ -49,6 +45,13 @@
 #include <unistd.h>
 #include <string>
 #endif
+#if defined(ARCH_OS_DARWIN)
+#include <sys/sysctl.h>
+#endif
+#if defined(ARCH_OS_WINDOWS)
+#include <Windows.h>
+#endif
+#include <atomic>
 
 PXR_NAMESPACE_OPEN_SCOPE
 
@@ -340,16 +343,14 @@ Arch_DebuggerAttachExecPosix(void* data)
     return false;
 }
 
+#endif // defined(ARCH_OS_LINUX) || defined(ARCH_OS_DARWIN)
+
+#if defined(ARCH_OS_LINUX)
+
 static
 bool
 Arch_DebuggerIsAttachedPosix()
 {
-#if defined(ARCH_OS_DARWIN)
-    const int PTRACE_ATTACH = PT_ATTACHEXC;
-    const int PTRACE_DETACH = PT_DETACH;
-    return false;
-#endif
-
     // Check for a ptrace based debugger by trying to ptrace.
     pid_t parent = getpid();
     pid_t pid = nonLockingFork();
@@ -392,7 +393,45 @@ Arch_DebuggerIsAttachedPosix()
 
 }
 
-#endif // defined(ARCH_OS_LINUX) || defined(ARCH_OS_DARWIN)
+#elif defined(ARCH_OS_DARWIN)
+
+// From Apple:
+//   https://developer.apple.com/library/content/qa/qa1361/_index.html
+//
+// Returns true if the current process is being debugged (either 
+// running under the debugger or has a debugger attached post facto).
+static bool
+AmIBeingDebugged()
+{
+    int                 junk;
+    int                 mib[4];
+    struct kinfo_proc   info;
+    size_t              size;
+
+    // Initialize the flags so that, if sysctl fails for some bizarre 
+    // reason, we get a predictable result.
+
+    info.kp_proc.p_flag = 0;
+
+    // Initialize mib, which tells sysctl the info we want, in this case
+    // we're looking for information about a specific process ID.
+
+    mib[0] = CTL_KERN;
+    mib[1] = KERN_PROC;
+    mib[2] = KERN_PROC_PID;
+    mib[3] = getpid();
+
+    // Call sysctl.
+
+    size = sizeof(info);
+    junk = sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &size, NULL, 0);
+
+    // We're being debugged if the P_TRACED flag is set.
+
+    return junk == 0 && ( (info.kp_proc.p_flag & P_TRACED) != 0 );
+}
+
+#endif // defined(ARCH_OS_LINUX)
 
 static
 bool
@@ -401,6 +440,11 @@ Arch_DebuggerAttach()
     // Be very careful here to avoid using the heap.  We're not even sure
     // the stack is available but there's only so much we can do about that.
 
+    // We assume Arch_DebuggerInit() has been called.
+    if (!_archDebuggerEnabled) {
+        return false;
+    }
+
 #if defined(ARCH_OS_LINUX) || defined(ARCH_OS_DARWIN)
 
     // To attach to gdb under Unix/Linux and Gnome:
@@ -420,6 +464,15 @@ Arch_DebuggerAttach()
     // debugged program, however, does stop so users simply need to click
     // TotalView's pause button to see the program state.  Unfortunately,
     // there's no obvious indication that the program has stopped.
+    //
+    // To attach to lldb on Darwin:
+    //   ARCH_DEBUGGER='osascript -e "tell application \"Terminal\"" -e "activate" -e "set newTab to do script(\"lldb -p %p\")" -e "end tell"'
+    // This will bring up lldb in a (new) terminal window.  If your system
+    // has System Integrity Protection then this won't work but there's a
+    // workaround:  make a copy of Terminal (in /Applications/Utilities);
+    // put it in /Applications with a new name, say, TerminalCopy;  then
+    // use that new name in the environment variable above instead of
+    // "Terminal".
 
     if (_archDebuggerAttachArgs) {
         // We need to start a process unrelated to this process so the
@@ -524,28 +577,33 @@ Arch_InitDebuggerAttach()
         // Terminate the command string.
         *a = '\0';
     }
+#elif defined(ARCH_OS_WINDOWS)
+    // If ARCH_DEBUGGER is in the environment then enable attaching.
+    size_t requiredSize;
+    getenv_s(&requiredSize, nullptr, 0, "ARCH_DEBUGGER");
+    if (requiredSize) {
+        _archDebuggerAttachArgs = (char**)&_archDebuggerAttachArgs;
+    }
 #endif
 }
 
 void
 ArchDebuggerTrap()
 {
-    Arch_DebuggerInit();
+    // Trap if a debugger is attached or we try and fail to attach one.
+    // If we attach one we assume it will automatically stop this process.
+    if (ArchDebuggerIsAttached() || !Arch_DebuggerAttach()) {
     if (_archDebuggerEnabled) {
 #if defined(ARCH_OS_WINDOWS)
-        if (IsDebuggerPresent()) {
             DebugBreak();
-        }
-#elif defined(ARCH_CPU_INTEL) && \
-      (defined(ARCH_COMPILER_GCC) || defined(ARCH_COMPILER_CLANG))
-        // Send a trap if a debugger is attached or we fail to start one.
-        // If we start one we assume it will automatically stop this process.
-        if (Arch_DebuggerIsAttachedPosix() || !Arch_DebuggerAttach()) {
+#elif defined(ARCH_CPU_INTEL)
             asm("int $3");
-        }
+#else
+            raise(SIGTRAP);
 #endif
     }
 }
+}
 
 void
 ArchDebuggerWait(bool wait)
@@ -580,7 +638,9 @@ ArchDebuggerIsAttached()
     Arch_DebuggerInit();
 #if defined(ARCH_OS_WINDOWS)
     return IsDebuggerPresent() == TRUE;
-#elif defined(ARCH_OS_LINUX) || defined(ARCH_OS_DARWIN)
+#elif defined(ARCH_OS_DARWIN)
+    return AmIBeingDebugged();
+#elif defined(ARCH_OS_LINUX)
     return Arch_DebuggerIsAttachedPosix();
 #endif
     return false;