GH-103082: Implementation of PEP 669: Low Impact Monitoring for CPython

Include/cpython/code.h

@@ -3,10 +3,22 @@
 #ifndef Py_LIMITED_API
 #ifndef Py_CODE_H
 #define Py_CODE_H
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+
+/* Count of all "real" monitoring events (not derived from other events) */
+#define PY_MONITORING_UNGROUPED_EVENTS 14
+/* Count of all  monitoring events */
+#define PY_MONITORING_EVENTS 16
+
+/* Table of which tools are active for each monitored event. */
+typedef struct _Py_Monitors {
+    uint8_t tools[PY_MONITORING_UNGROUPED_EVENTS];
+} _Py_Monitors;
+
 /* Each instruction in a code object is a fixed-width value,
  * currently 2 bytes: 1-byte opcode + 1-byte oparg.  The EXTENDED_ARG
  * opcode allows for larger values but the current limit is 3 uses
@@ -56,6 +68,35 @@ typedef struct {
     PyObject *_co_freevars;
 } _PyCoCached;
 
+/* Ancilliary data structure used for instrumentation.
+   Line instrumentation creates an array of
+   these. One entry per code unit.*/
+typedef struct {
+    uint8_t original_opcode;
+    int8_t line_delta;
+} _PyCoLineInstrumentationData;
+
+/* Main data structure used for instrumentation.
+ * This is allocated when needed for instrumentation
+ */
+typedef struct {
+    /* Monitoring specific to this code object */
+    _Py_Monitors local_monitors;
+    /* Monitoring that is active on this code object */
+    _Py_Monitors active_monitors;
+    /* The tools that are to be notified for events for the matching code unit */
+    uint8_t *tools;
+    /* Information to support line events */
+    _PyCoLineInstrumentationData *lines;
+    /* The tools that are to be notified for line events for the matching code unit */
+    uint8_t *line_tools;
+    /* Information to support instruction events */
+    /* The underlying instructions, which can themselves be instrumented */
+    uint8_t *per_instruction_opcodes;
+    /* The tools that are to be notified for instruction events for the matching code unit */
+    uint8_t *per_instruction_tools;
+} _PyCoMonitoringData;
+
 // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are
 // defined in this macro:
 #define _PyCode_DEF(SIZE) {                                                    \
@@ -87,7 +128,6 @@ typedef struct {
     PyObject *co_exceptiontable;   /* Byte string encoding exception handling  \
                                       table */                                 \
     int co_flags;                  /* CO_..., see below */                     \
-    short _co_linearray_entry_size;  /* Size of each entry in _co_linearray */ \
                                                                                \
     /* The rest are not so impactful on performance. */                        \
     int co_argcount;              /* #arguments, except *args */               \
@@ -114,8 +154,9 @@ typedef struct {
     PyObject *co_linetable;       /* bytes object that holds location info */  \
     PyObject *co_weakreflist;     /* to support weakrefs to code objects */    \
     _PyCoCached *_co_cached;      /* cached co_* attributes */                 \
+    uint64_t _co_instrumentation_version; /* current instrumentation version */  \
+    _PyCoMonitoringData *_co_monitoring; /* Monitoring data */                 \
     int _co_firsttraceable;       /* index of first traceable instruction */   \
-    char *_co_linearray;          /* array of line offsets */                  \
     /* Scratch space for extra data relating to the code object.               \
        Type is a void* to keep the format private in codeobject.c to force     \
        people to go through the proper APIs. */                                \

Include/cpython/pystate.h

@@ -58,12 +58,6 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *);
 #define PyTrace_C_RETURN 6
 #define PyTrace_OPCODE 7
 
-
-typedef struct {
-    PyCodeObject *code; // The code object for the bounds. May be NULL.
-    PyCodeAddressRange bounds; // Only valid if code != NULL.
-} PyTraceInfo;
-
 // Internal structure: you should not use it directly, but use public functions
 // like PyThreadState_EnterTracing() and PyThreadState_LeaveTracing().
 typedef struct _PyCFrame {
@@ -77,7 +71,6 @@ typedef struct _PyCFrame {
      * discipline and make sure that instances of this struct cannot
      * accessed outside of their lifetime.
      */
-    uint8_t use_tracing;  // 0 or 255 (or'ed into opcode, hence 8-bit type)
     /* Pointer to the currently executing frame (it can be NULL) */
     struct _PyInterpreterFrame *current_frame;
     struct _PyCFrame *previous;
@@ -157,7 +150,7 @@ struct _ts {
        This is to prevent the actual trace/profile code from being recorded in
        the trace/profile. */
     int tracing;
-    int tracing_what; /* The event currently being traced, if any. */
+    int what_event; /* The event currently being monitored, if any. */
 
     /* Pointer to current _PyCFrame in the C stack frame of the currently,
      * or most recently, executing _PyEval_EvalFrameDefault. */
@@ -228,8 +221,6 @@ struct _ts {
     /* Unique thread state id. */
     uint64_t id;
 
-    PyTraceInfo trace_info;
-
     _PyStackChunk *datastack_chunk;
     PyObject **datastack_top;
     PyObject **datastack_limit;

Include/internal/pycore_code.h

@@ -441,32 +441,6 @@ adaptive_counter_backoff(uint16_t counter) {
 
 /* Line array cache for tracing */
 
-extern int _PyCode_CreateLineArray(PyCodeObject *co);
-
-static inline int
-_PyCode_InitLineArray(PyCodeObject *co)
-{
-    if (co->_co_linearray) {
-        return 0;
-    }
-    return _PyCode_CreateLineArray(co);
-}
-
-static inline int
-_PyCode_LineNumberFromArray(PyCodeObject *co, int index)
-{
-    assert(co->_co_linearray != NULL);
-    assert(index >= 0);
-    assert(index < Py_SIZE(co));
-    if (co->_co_linearray_entry_size == 2) {
-        return ((int16_t *)co->_co_linearray)[index];
-    }
-    else {
-        assert(co->_co_linearray_entry_size == 4);
-        return ((int32_t *)co->_co_linearray)[index];
-    }
-}
-
 typedef struct _PyShimCodeDef {
     const uint8_t *code;
     int codelen;
@@ -500,6 +474,10 @@ extern uint32_t _Py_next_func_version;
 
 #define COMPARISON_NOT_EQUALS (COMPARISON_UNORDERED | COMPARISON_LESS_THAN | COMPARISON_GREATER_THAN)
 
+extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp);
+
+extern int _Py_GetBaseOpcode(PyCodeObject *code, int offset);
+
 
 #ifdef __cplusplus
 }

Include/internal/pycore_frame.h

@@ -19,6 +19,7 @@ struct _frame {
     struct _PyInterpreterFrame *f_frame; /* points to the frame data */
     PyObject *f_trace;          /* Trace function */
     int f_lineno;               /* Current line number. Only valid if non-zero */
+    int f_last_traced_line;     /* The last line traced for this frame */
     char f_trace_lines;         /* Emit per-line trace events? */
     char f_trace_opcodes;       /* Emit per-opcode trace events? */
     char f_fast_as_locals;      /* Have the fast locals of this frame been converted to a dict? */
@@ -137,10 +138,16 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
     return frame->localsplus;
 }
 
+/* Fetches the stack pointer, and sets stacktop to -1.
+    Having stacktop <= 0 ensures that invalid
+    values are not visible to the cycle GC.
+    We choose -1 rather than 0 to assist debugging. */
 static inline PyObject**
 _PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
 {
-    return frame->localsplus+frame->stacktop;
+    PyObject **sp = frame->localsplus + frame->stacktop;
+    frame->stacktop = -1;
+    return sp;
 }
 
 static inline void

Include/internal/pycore_instruments.h

@@ -0,0 +1,107 @@
+
+#ifndef Py_INTERNAL_INSTRUMENT_H
+#define Py_INTERNAL_INSTRUMENT_H
+
+
+#include "pycore_bitutils.h"      // _Py_popcount32
+#include "pycore_frame.h"
+
+#include "cpython/code.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define PY_MONITORING_TOOL_IDS 8
+
+/* Local events.
+ * These require bytecode instrumentation */
+
+#define PY_MONITORING_EVENT_PY_START 0
+#define PY_MONITORING_EVENT_PY_RESUME 1
+#define PY_MONITORING_EVENT_PY_RETURN 2
+#define PY_MONITORING_EVENT_PY_YIELD 3
+#define PY_MONITORING_EVENT_CALL 4
+#define PY_MONITORING_EVENT_LINE 5
+#define PY_MONITORING_EVENT_INSTRUCTION 6
+#define PY_MONITORING_EVENT_JUMP 7
+#define PY_MONITORING_EVENT_BRANCH 8
+#define PY_MONITORING_EVENT_STOP_ITERATION 9
+
+#define PY_MONITORING_INSTRUMENTED_EVENTS 10
+
+/* Other events, mainly exceptions */
+
+#define PY_MONITORING_EVENT_RAISE 10
+#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11
+#define PY_MONITORING_EVENT_PY_UNWIND 12
+#define PY_MONITORING_EVENT_PY_THROW 13
+
+
+/* Ancilliary events */
+
+#define PY_MONITORING_EVENT_C_RETURN 14
+#define PY_MONITORING_EVENT_C_RAISE 15
+
+
+typedef uint32_t _PyMonitoringEventSet;
+
+/* Tool IDs */
+
+/* These are defined in PEP 669 for convenience to avoid clashes */
+#define PY_MONITORING_DEBUGGER_ID 0
+#define PY_MONITORING_COVERAGE_ID 1
+#define PY_MONITORING_PROFILER_ID 2
+#define PY_MONITORING_OPTIMIZER_ID 5
+
+/* Internal IDs used to suuport sys.setprofile() and sys.settrace() */
+#define PY_MONITORING_SYS_PROFILE_ID 6
+#define PY_MONITORING_SYS_TRACE_ID 7
+
+
+PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj);
+
+int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events);
+
+extern int
+_Py_call_instrumentation(PyThreadState *tstate, int event,
+    _PyInterpreterFrame *frame, _Py_CODEUNIT *instr);
+
+extern int
+_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame,
+                              _Py_CODEUNIT *instr);
+
+extern int
+_Py_call_instrumentation_instruction(
+    PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr);
+
+int
+_Py_call_instrumentation_jump(
+    PyThreadState *tstate, int event,
+    _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target);
+
+extern int
+_Py_call_instrumentation_arg(PyThreadState *tstate, int event,
+    _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg);
+
+extern int
+_Py_call_instrumentation_2args(PyThreadState *tstate, int event,
+    _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1);
+
+extern void
+_Py_call_instrumentation_exc0(PyThreadState *tstate, int event,
+    _PyInterpreterFrame *frame, _Py_CODEUNIT *instr);
+
+extern void
+_Py_call_instrumentation_exc2(PyThreadState *tstate, int event,
+    _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1);
+
+extern int
+_Py_Instrumentation_GetLine(PyCodeObject *code, int index);
+
+extern PyObject _PyInstrumentation_MISSING;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_INSTRUMENT_H */

Include/internal/pycore_interp.h

@@ -24,6 +24,7 @@ extern "C" {
 #include "pycore_genobject.h"     // struct _Py_async_gen_state
 #include "pycore_gc.h"            // struct _gc_runtime_state
 #include "pycore_import.h"        // struct _import_state
+#include "pycore_instruments.h"   // PY_MONITORING_EVENTS
 #include "pycore_list.h"          // struct _Py_list_state
 #include "pycore_global_objects.h"  // struct _Py_interp_static_objects
 #include "pycore_object_state.h"   // struct _py_object_state
@@ -37,7 +38,6 @@ struct _Py_long_state {
     int max_str_digits;
 };
 
-
 /* interpreter state */
 
 /* PyInterpreterState holds the global state for one of the runtime's
@@ -49,6 +49,9 @@ struct _is {
 
     PyInterpreterState *next;
 
+    uint64_t monitoring_version;
+    uint64_t last_restart_version;
+
     struct pythreads {
         uint64_t next_unique_id;
         /* The linked list of threads, newest first. */
@@ -148,6 +151,15 @@ struct _is {
     struct callable_cache callable_cache;
     PyCodeObject *interpreter_trampoline;
 
+    _Py_Monitors monitors;
+    bool f_opcode_trace_set;
+    bool sys_profile_initialized;
+    bool sys_trace_initialized;
+    Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */
+    Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */
+    PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][PY_MONITORING_EVENTS];
+    PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS];
+
     struct _Py_interp_cached_objects cached_objects;
     struct _Py_interp_static_objects static_objects;