Add features from upstream rebase, enable python 3.12 support

include/CPyCppyy/API.h

@@ -93,7 +93,7 @@ class Dimensions {      // Windows note: NOT exported/imported
     dim_t* fDims;
 
 public:
-    Dimensions(dim_t ndim = 0, dim_t* dims = nullptr) : fDims(nullptr) {}
+    Dimensions(dim_t /*ndim*/ = 0, dim_t* /*dims*/ = nullptr) : fDims(nullptr) {}
     ~Dimensions() { delete [] fDims; }
 
 public:
@@ -133,6 +133,9 @@ CPYCPPYY_EXTERN void DestroyConverter(Converter* p);
 typedef Converter* (*ConverterFactory_t)(cdims_t);
 CPYCPPYY_EXTERN bool RegisterConverter(const std::string& name, ConverterFactory_t);
 
+// register a custom converter that is a reference to an existing converter
+CPYCPPYY_EXTERN bool RegisterConverterAlias(const std::string& name, const std::string& target);
+
 // remove a custom converter
 CPYCPPYY_EXTERN bool UnregisterConverter(const std::string& name);
 
@@ -161,6 +164,9 @@ CPYCPPYY_EXTERN void DestroyConverter(Converter* p);
 typedef Executor* (*ExecutorFactory_t)(cdims_t);
 CPYCPPYY_EXTERN bool RegisterExecutor(const std::string& name, ExecutorFactory_t);
 
+// register a custom executor that is a reference to an existing converter
+CPYCPPYY_EXTERN bool RegisterExecutorAlias(const std::string& name, const std::string& target);
+
 // remove a custom executor
 CPYCPPYY_EXTERN bool UnregisterExecutor(const std::string& name);
 
@@ -186,6 +192,9 @@ CPYCPPYY_EXTERN bool Scope_CheckExact(PyObject* pyobject);
 CPYCPPYY_EXTERN bool Instance_Check(PyObject* pyobject);
 CPYCPPYY_EXTERN bool Instance_CheckExact(PyObject* pyobject);
 
+// type verifier for sequences
+CPYCPPYY_EXTERN bool Sequence_Check(PyObject* pyobject);
+
 // helper to verify expected safety of moving an instance into C++
 CPYCPPYY_EXTERN bool Instance_IsLively(PyObject* pyobject);
 

src/API.cxx

@@ -66,7 +66,7 @@ static bool Initialize()
     // try again to see if the interpreter is initialized
         if (!Py_IsInitialized()) {
         // give up ...
-            std::cerr << "Error: python has not been intialized; returning." << std::endl;
+            std::cerr << "Error: python has not been initialized; returning." << std::endl;
             return false;
         }
 
@@ -88,7 +88,10 @@ static bool Initialize()
     // retrieve the main dictionary
         gMainDict = PyModule_GetDict(
             PyImport_AddModule(const_cast<char*>("__main__")));
-        Py_INCREF(gMainDict);
+    // The gMainDict is borrowed, i.e. we are not calling Py_INCREF(gMainDict).
+    // Like this, we avoid unexpectedly affecting how long __main__ is kept
+    // alive. The gMainDict is only used in Exec(), ExecScript(), and Eval(),
+    // which should not be called after __main__ is garbage collected anyway.
     }
 
 // declare success ...
@@ -198,6 +201,39 @@ bool CPyCppyy::Instance_CheckExact(PyObject* pyobject)
     return CPPInstance_CheckExact(pyobject);
 }
 
+//-----------------------------------------------------------------------------
+bool CPyCppyy::Sequence_Check(PyObject* pyobject)
+{
+// Extends on PySequence_Check() to determine whether an object can be iterated
+// over (technically, all objects can b/c of C++ pointer arithmetic, hence this
+// check isn't 100% accurate, but neither is PySequence_Check()).
+
+// Note: simply having the iterator protocol does not constitute a sequence, bc
+// PySequence_GetItem() would fail.
+
+// default to PySequence_Check() if called with a non-C++ object
+    if (!CPPInstance_Check(pyobject))
+        return (bool)PySequence_Check(pyobject);
+
+// all C++ objects should have sq_item defined, but a user-derived class may
+// have deleted it, in which case this is not a sequence
+    PyTypeObject* t = Py_TYPE(pyobject);
+    if (!t->tp_as_sequence || !t->tp_as_sequence->sq_item)
+        return false;
+
+// if this is the default getitem, it is only a sequence if it's an array type
+    if (t->tp_as_sequence->sq_item == CPPInstance_Type.tp_as_sequence->sq_item) {
+        if (((CPPInstance*)pyobject)->fFlags & CPPInstance::kIsArray)
+            return true;
+        return false;
+    }
+
+// TODO: could additionally verify whether __len__ is supported and/or whether
+// operator()[] takes an int argument type
+
+    return true;
+}
+
 //-----------------------------------------------------------------------------
 bool CPyCppyy::Instance_IsLively(PyObject* pyobject)
 {
@@ -332,18 +368,80 @@ void CPyCppyy::ExecScript(const std::string& name, const std::vector<std::string
         oldargv = l;
     }
 
-// create and set (add progam name) the new command line
-#if PY_VERSION_HEX < 0x03000000
+// create and set (add program name) the new command line
     int argc = args.size() + 1;
+#if PY_VERSION_HEX < 0x03000000
+// This is a legacy implementation for Python 2
     const char** argv = new const char*[argc];
     for (int i = 1; i < argc; ++i) argv[i] = args[i-1].c_str();
     argv[0] = Py_GetProgramName();
     PySys_SetArgv(argc, const_cast<char**>(argv));
     delete [] argv;
 #else
-// TODO: fix this to work like above ...
-    (void)args;
-#endif
+// This is a common code block for Python 3. We prefer using objects to
+// automatize memory management and not introduce even more preprocessor
+// branching for deletion at the end of the method.
+//
+// FUTURE IMPROVEMENT ONCE OLD PYTHON VERSIONS ARE NOT SUPPORTED BY CPPYY:
+// Right now we use C++ objects to automatize memory management. One could use
+// RAAI and the Python memory allocation API (PEP 445) once some old Python
+// version is deprecated in CPPYY. That new feature is available since version
+// 3.4 and the preprocessor branching to also support that would be so
+// complicated to make the code unreadable.
+   std::vector<std::wstring> argv2;
+   argv2.reserve(argc);
+   argv2.emplace_back(name.c_str(), &name[name.size()]);
+
+   for (int i = 1; i < argc; ++i) {
+      auto iarg = args[i - 1].c_str();
+      argv2.emplace_back(iarg, &iarg[strlen(iarg)]);
+   }
+
+#if PY_VERSION_HEX < 0x03080000
+// Before version 3.8, the code is one simple line
+   wchar_t *argv2_arr[argc];
+   for (int i = 0; i < argc; ++i) {
+      argv2_arr[i] = const_cast<wchar_t *>(argv2[i].c_str());
+   }
+   PySys_SetArgv(argc, argv2_arr);
+
+#else
+// Here we comply to "PEP 587 – Python Initialization Configuration" to avoid
+// deprecation warnings at compile time.
+   class PyConfigHelperRAAI {
+   public:
+      PyConfigHelperRAAI(const std::vector<std::wstring> &argv2)
+      {
+         PyConfig_InitPythonConfig(&fConfig);
+         fConfig.parse_argv = 1;
+         UpdateArgv(argv2);
+         InitFromConfig();
+      }
+      ~PyConfigHelperRAAI() { PyConfig_Clear(&fConfig); }
+
+   private:
+      void InitFromConfig() { Py_InitializeFromConfig(&fConfig); };
+      void UpdateArgv(const std::vector<std::wstring> &argv2)
+      {
+         auto WideStringListAppendHelper = [](PyWideStringList *wslist, const wchar_t *wcstr) {
+            PyStatus append_status = PyWideStringList_Append(wslist, wcstr);
+            if (PyStatus_IsError(append_status)) {
+               std::wcerr << "Error: could not append element " << wcstr << " to arglist - " << append_status.err_msg
+                          << std::endl;
+            }
+         };
+         WideStringListAppendHelper(&fConfig.argv, Py_GetProgramName());
+         for (const auto &iarg : argv2) {
+            WideStringListAppendHelper(&fConfig.argv, iarg.c_str());
+         }
+      }
+      PyConfig fConfig;
+   };
+
+   PyConfigHelperRAAI pych(argv2);
+
+#endif // of the else branch of PY_VERSION_HEX < 0x03080000
+#endif // of the else branch of PY_VERSION_HEX < 0x03000000
 
 // actual script execution
     PyObject* gbl = PyDict_Copy(gMainDict);
@@ -403,7 +501,7 @@ const CPyCppyy::PyResult CPyCppyy::Eval(const std::string& expr)
         return PyResult();
     }
 
-// results that require no convserion
+// results that require no conversion
     if (result == Py_None || CPPInstance_Check(result) ||
             PyBytes_Check(result) ||
             PyFloat_Check(result) || PyLong_Check(result) || PyInt_Check(result))

src/CPPClassMethod.cxx

@@ -24,7 +24,7 @@ PyObject* CPyCppyy::CPPClassMethod::Call(CPPInstance*&
 
 // translate the arguments
 #if PY_VERSION_HEX >= 0x03080000
-// TODO: The following is not robust and should be revisited e.g. by makeing CPPOverloads
+// TODO: The following is not robust and should be revisited e.g. by making CPPOverloads
 // that have only CPPClassMethods be true Python classmethods? Note that the original
 // implementation wasn't 100% correct either (e.g. static size() mapped to len()).
 //
@@ -35,7 +35,7 @@ PyObject* CPyCppyy::CPPClassMethod::Call(CPPInstance*&
     if ((!self || (PyObject*)self == Py_None) && nargs) {
         PyObject* arg0 = CPyCppyy_PyArgs_GET_ITEM(args, 0);
         if ((CPPInstance_Check(arg0) && ((CPPInstance*)arg0)->ObjectIsA() == GetScope()) && \
-                (fArgsRequired <= nargs-1) && (GetMaxArgs() < nargs)) {
+                (fArgsRequired <= nargs-1)) {
             args   += 1;     // drops first argument
             nargsf -= 1;
         }

src/CPPDataMember.cxx

@@ -18,7 +18,6 @@
 #include <limits.h>
 #include <structmember.h>
 
-
 namespace CPyCppyy {
 
 enum ETypeDetails {
@@ -35,7 +34,7 @@ enum ETypeDetails {
 static PyObject* dm_get(CPPDataMember* dm, CPPInstance* pyobj, PyObject* /* kls */)
 {
 // cache lookup for low level views
-    if (dm->fFlags & kIsCachable) {
+    if (pyobj && dm->fFlags & kIsCachable) {
         CPyCppyy::CI_DatamemberCache_t& cache = pyobj->GetDatamemberCache();
         for (auto it = cache.begin(); it != cache.end(); ++it) {
             if (it->first == dm->fOffset) {
@@ -49,6 +48,7 @@ static PyObject* dm_get(CPPDataMember* dm, CPPInstance* pyobj, PyObject* /* kls
         }
     }
 
+
     if (dm->fFlags & (kIsEnumPrep | kIsEnumType)) {
         if (dm->fFlags & kIsEnumPrep) {
         // still need to do lookup; only ever try this once, then fallback on converter
@@ -88,7 +88,6 @@ static PyObject* dm_get(CPPDataMember* dm, CPPInstance* pyobj, PyObject* /* kls
             return pyval_from_enum(Cppyy::ResolveEnum(dm->fScope), nullptr, nullptr, dm->fScope);
         }
     }
-
 // non-initialized or public data accesses through class (e.g. by help())
     void* address = dm->GetAddress(pyobj);
     if (!address || (intptr_t)address == -1 /* Cling error */)
@@ -220,7 +219,7 @@ static void dm_dealloc(CPPDataMember* dm)
 static PyMemberDef dm_members[] = {
         {(char*)"__doc__", T_OBJECT, offsetof(CPPDataMember, fDoc), 0,
                 (char*)"writable documentation"},
-        {NULL}  /* Sentinel */
+        {NULL, 0, 0, 0, nullptr}  /* Sentinel */
 };
 
 //= CPyCppyy datamember proxy access to internals ============================
@@ -308,10 +307,17 @@ PyTypeObject CPPDataMember_Type = {
 #if PY_VERSION_HEX >= 0x03040000
     , 0                            // tp_finalize
 #endif
+#if PY_VERSION_HEX >= 0x03080000
+    , 0                           // tp_vectorcall
+#endif
+#if PY_VERSION_HEX >= 0x030c0000
+    , 0                           // tp_watched
+#endif
 };
 
 } // namespace CPyCppyy
 
+
 //- public members -----------------------------------------------------------
 void CPyCppyy::CPPDataMember::Set(Cppyy::TCppScope_t scope, Cppyy::TCppScope_t data)
 {

src/CPPEnum.cxx

@@ -2,6 +2,7 @@
 #include "CPyCppyy.h"
 #include "CPPEnum.h"
 #include "PyStrings.h"
+#include "TypeManip.h"
 #include "Utility.h"
 
 
@@ -46,7 +47,6 @@ PyObject* CPyCppyy::pyval_from_enum(const std::string& enum_type, PyObject* pyty
     return bval;
 }
 
-
 //- enum methods -------------------------------------------------------------
 static int enum_setattro(PyObject* /* pyclass */, PyObject* /* pyname */, PyObject* /* pyval */)
 {
@@ -172,6 +172,14 @@ CPyCppyy::CPPEnum* CPyCppyy::CPPEnum_New(const std::string& name, Cppyy::TCppSco
         PyObject* pyresolved = CPyCppyy_PyText_FromString(resolved.c_str());
         PyDict_SetItem(dct, PyStrings::gUnderlying, pyresolved);
         Py_DECREF(pyresolved);
+
+    // add the __module__ to allow pickling
+        std::string modname = TypeManip::extract_namespace(ename);
+        TypeManip::cppscope_to_pyscope(modname);      // :: -> .
+        if (!modname.empty()) modname = "."+modname;
+        PyObject* pymodname = CPyCppyy_PyText_FromString(("cppyy.gbl"+modname).c_str());
+        PyDict_SetItem(dct, PyStrings::gModule, pymodname);
+        Py_DECREF(pymodname);
 
     // create the actual enum class
         args = Py_BuildValue((char*)"sOO", name.c_str(), pybases, dct);
@@ -210,4 +218,4 @@ CPyCppyy::CPPEnum* CPyCppyy::CPPEnum_New(const std::string& name, Cppyy::TCppSco
     }
 
     return pyenum;
-}
+}

src/CPPExcInstance.cxx

@@ -29,7 +29,7 @@ static PyObject* ep_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds)
         PyObject* ulc = PyObject_GetAttr((PyObject*)subtype, PyStrings::gUnderlying);
         excobj->fCppInstance = PyType_Type.tp_call(ulc, args, kwds);
         if (!excobj->fCppInstance) {
-        // if this fails, then the contruction may have been attempted from a string
+        // if this fails, then the construction may have been attempted from a string
         // (e.g. from PyErr_Format); if so, drop the proxy and use fTopMessage instead
             PyErr_Clear();
             if (PyTuple_GET_SIZE(args) == 1) {
@@ -282,6 +282,12 @@ PyTypeObject CPPExcInstance_Type = {
 #if PY_VERSION_HEX >= 0x03040000
     , 0                            // tp_finalize
 #endif
+#if PY_VERSION_HEX >= 0x03080000
+    , 0                            // tp_vectorcall
+#endif
+#if PY_VERSION_HEX >= 0x030c0000
+    , 0                            // tp_watched
+#endif
 };
 
 } // namespace CPyCppyy