attach: Add breakpoints for more safe entry points

`memray attach` needs to inject some custom code into the process it has
attached to. It does this by using `dlopen` and then calling a function
we control, but that can't be done just anywhere: we need to ensure
we're not currently in the middle of a call to `dlopen`, or in the
middle of a call to `malloc`, etc.

To work around this, we set a handful of breakpoints for known safe
(probably) places to call into our custom code, and load and call it
only when the breakpoint is hit.

Add 2 new breakpoints, `PyCallable_Check` and `PyError_CheckSignals`.
Also, use `Py_AddPendingCall` to schedule a call to `PyCallable_Check`
on the main thread, in the hopes of triggering one of our breakpoints to
happen sooner.

Signed-off-by: Matt Wozniski <[email protected]>

src/memray/commands/_attach.gdb

@@ -14,7 +14,10 @@ p PyMem_Calloc
 p PyMem_Realloc
 p PyMem_Free
 
+
 p "MEMRAY: Process is Python 3.7+."
+set scheduler-locking on
+call (int)Py_AddPendingCall(&PyCallable_Check, (void*)0)
 
 # When updating this list, also update the "commands" call below,
 # and the breakpoints hardcoded for lldb in attach.py
@@ -26,12 +29,17 @@ b PyMem_Malloc
 b PyMem_Calloc
 b PyMem_Realloc
 b PyMem_Free
-# Apply commands to all 8 breakpoints above
-commands 1-8
+b PyErr_CheckSignals
+b PyCallable_Check
+# Apply commands to all 10 breakpoints above
+commands 1-10
+    bt
     disable breakpoints
+    delete breakpoints
     call (void*)dlopen($libpath, $rtld_now)
     p (char*)dlerror()
     eval "sharedlibrary %s", $libpath
     p (int)memray_spawn_client($port) ? "FAILURE" : "SUCCESS"
 end
+set scheduler-locking off
 continue

src/memray/commands/_attach.lldb

@@ -10,11 +10,13 @@ p ((void(*)(void*))PyMem_Free)
 p "MEMRAY: Process is Python 3.7+."
 
 # When adding new breakpoints, also update _attach.gdb
-breakpoint set -b malloc -b calloc -b realloc -b free -b PyMem_Malloc -b PyMem_Calloc -b PyMem_Realloc -b PyMem_Free
+breakpoint set -b malloc -b calloc -b realloc -b free -b PyMem_Malloc -b PyMem_Calloc -b PyMem_Realloc -b PyMem_Free -b PyErr_CheckSignals -b PyCallable_Check
 
 # Set commands to execute when breakpoint is reached
 breakpoint command add -e true
+bt
 breakpoint disable
+breakpoint delete
 expr auto $dlsym = (void* (*)(void*, const char*))&::dlsym
 expr auto $dlopen = $dlsym($rtld_default, "dlopen")
 expr auto $dlerror = $dlsym($rtld_default, "dlerror")
@@ -24,4 +26,8 @@ expr auto $spawn = $dlsym($dll, "memray_spawn_client")
 p ((int(*)(int))$spawn)($port)?"FAILURE":"SUCCESS"
 DONE
 
+expr auto $Py_AddPendingCall = (int(*)(int(*)(void*), void*))&::Py_AddPendingCall
+expr auto $PyCallable_Check = (int(*)(void*))&::PyCallable_Check
+p $Py_AddPendingCall($PyCallable_Check, (void*)0)
+
 continue