From 5becfb6fb3d299845e2b380c36b5cb4cb8c7bf93 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Thu, 24 Oct 2024 17:00:59 +0200 Subject: [PATCH 01/28] Sync API.cxx --- src/API.cxx | 114 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 8 deletions(-) diff --git a/src/API.cxx b/src/API.cxx index 2d4f9e1..193ffad 100644 --- a/src/API.cxx +++ b/src/API.cxx @@ -66,7 +66,7 @@ static bool Initialize() // try again to see if the interpreter is initialized if (!Py_IsInitialized()) { // give up ... - std::cerr << "Error: python has not been intialized; returning." << std::endl; + std::cerr << "Error: python has not been initialized; returning." << std::endl; return false; } @@ -88,7 +88,10 @@ static bool Initialize() // retrieve the main dictionary gMainDict = PyModule_GetDict( PyImport_AddModule(const_cast("__main__"))); - Py_INCREF(gMainDict); + // The gMainDict is borrowed, i.e. we are not calling Py_INCREF(gMainDict). + // Like this, we avoid unexpectedly affecting how long __main__ is kept + // alive. The gMainDict is only used in Exec(), ExecScript(), and Eval(), + // which should not be called after __main__ is garbage collected anyway. } // declare success ... @@ -198,6 +201,39 @@ bool CPyCppyy::Instance_CheckExact(PyObject* pyobject) return CPPInstance_CheckExact(pyobject); } +//----------------------------------------------------------------------------- +bool CPyCppyy::Sequence_Check(PyObject* pyobject) +{ +// Extends on PySequence_Check() to determine whether an object can be iterated +// over (technically, all objects can b/c of C++ pointer arithmetic, hence this +// check isn't 100% accurate, but neither is PySequence_Check()). + +// Note: simply having the iterator protocol does not constitute a sequence, bc +// PySequence_GetItem() would fail. + +// default to PySequence_Check() if called with a non-C++ object + if (!CPPInstance_Check(pyobject)) + return (bool)PySequence_Check(pyobject); + +// all C++ objects should have sq_item defined, but a user-derived class may +// have deleted it, in which case this is not a sequence + PyTypeObject* t = Py_TYPE(pyobject); + if (!t->tp_as_sequence || !t->tp_as_sequence->sq_item) + return false; + +// if this is the default getitem, it is only a sequence if it's an array type + if (t->tp_as_sequence->sq_item == CPPInstance_Type.tp_as_sequence->sq_item) { + if (((CPPInstance*)pyobject)->fFlags & CPPInstance::kIsArray) + return true; + return false; + } + +// TODO: could additionally verify whether __len__ is supported and/or whether +// operator()[] takes an int argument type + + return true; +} + //----------------------------------------------------------------------------- bool CPyCppyy::Instance_IsLively(PyObject* pyobject) { @@ -332,18 +368,80 @@ void CPyCppyy::ExecScript(const std::string& name, const std::vector(argv)); delete [] argv; #else -// TODO: fix this to work like above ... - (void)args; -#endif +// This is a common code block for Python 3. We prefer using objects to +// automatize memory management and not introduce even more preprocessor +// branching for deletion at the end of the method. +// +// FUTURE IMPROVEMENT ONCE OLD PYTHON VERSIONS ARE NOT SUPPORTED BY CPPYY: +// Right now we use C++ objects to automatize memory management. One could use +// RAAI and the Python memory allocation API (PEP 445) once some old Python +// version is deprecated in CPPYY. That new feature is available since version +// 3.4 and the preprocessor branching to also support that would be so +// complicated to make the code unreadable. + std::vector argv2; + argv2.reserve(argc); + argv2.emplace_back(name.c_str(), &name[name.size()]); + + for (int i = 1; i < argc; ++i) { + auto iarg = args[i - 1].c_str(); + argv2.emplace_back(iarg, &iarg[strlen(iarg)]); + } + +#if PY_VERSION_HEX < 0x03080000 +// Before version 3.8, the code is one simple line + wchar_t *argv2_arr[argc]; + for (int i = 0; i < argc; ++i) { + argv2_arr[i] = const_cast(argv2[i].c_str()); + } + PySys_SetArgv(argc, argv2_arr); + +#else +// Here we comply to "PEP 587 – Python Initialization Configuration" to avoid +// deprecation warnings at compile time. + class PyConfigHelperRAAI { + public: + PyConfigHelperRAAI(const std::vector &argv2) + { + PyConfig_InitPythonConfig(&fConfig); + fConfig.parse_argv = 1; + UpdateArgv(argv2); + InitFromConfig(); + } + ~PyConfigHelperRAAI() { PyConfig_Clear(&fConfig); } + + private: + void InitFromConfig() { Py_InitializeFromConfig(&fConfig); }; + void UpdateArgv(const std::vector &argv2) + { + auto WideStringListAppendHelper = [](PyWideStringList *wslist, const wchar_t *wcstr) { + PyStatus append_status = PyWideStringList_Append(wslist, wcstr); + if (PyStatus_IsError(append_status)) { + std::wcerr << "Error: could not append element " << wcstr << " to arglist - " << append_status.err_msg + << std::endl; + } + }; + WideStringListAppendHelper(&fConfig.argv, Py_GetProgramName()); + for (const auto &iarg : argv2) { + WideStringListAppendHelper(&fConfig.argv, iarg.c_str()); + } + } + PyConfig fConfig; + }; + + PyConfigHelperRAAI pych(argv2); + +#endif // of the else branch of PY_VERSION_HEX < 0x03080000 +#endif // of the else branch of PY_VERSION_HEX < 0x03000000 // actual script execution PyObject* gbl = PyDict_Copy(gMainDict); @@ -403,7 +501,7 @@ const CPyCppyy::PyResult CPyCppyy::Eval(const std::string& expr) return PyResult(); } -// results that require no convserion +// results that require no conversion if (result == Py_None || CPPInstance_Check(result) || PyBytes_Check(result) || PyFloat_Check(result) || PyLong_Check(result) || PyInt_Check(result)) From a787ba1ae3668d468a28afdd559b3d89698bf213 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Thu, 24 Oct 2024 17:02:17 +0200 Subject: [PATCH 02/28] Sync Callcontext --- src/CallContext.h | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/CallContext.h b/src/CallContext.h index 597970c..fc6a43b 100644 --- a/src/CallContext.h +++ b/src/CallContext.h @@ -58,7 +58,7 @@ struct CallContext { kIsCreator = 0x000002, // if method creates python-owned objects kIsConstructor = 0x000004, // if method is a C++ constructor kHaveImplicit = 0x000008, // indicate that implicit converters are available - kAllowImplicit = 0x000010, // indicate that implicit coversions are allowed + kAllowImplicit = 0x000010, // indicate that implicit conversions are allowed kNoImplicit = 0x000020, // disable implicit to prevent recursion kCallDirect = 0x000040, // call wrapped method directly, no inheritance kFromDescr = 0x000080, // initiated from a descriptor @@ -154,6 +154,24 @@ inline bool UseStrictOwnership(CallContext* ctxt) { return CallContext::sMemoryPolicy == CallContext::kUseStrict; } +template +class CallContextRAII { +public: + CallContextRAII(CallContext* ctxt) : fCtxt(ctxt) { + fPrior = fCtxt->fFlags & F; + fCtxt->fFlags |= F; + } + CallContextRAII(const CallContextRAII&) = delete; + CallContextRAII& operator=(const CallContextRAII&) = delete; + ~CallContextRAII() { + if (!fPrior) fCtxt->fFlags &= ~F; + } + +private: + CallContext* fCtxt; + bool fPrior; +}; + } // namespace CPyCppyy #endif // !CPYCPPYY_CALLCONTEXT_H From b93b7e67c1424f5379d862973988f9c9b6aaff13 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Thu, 24 Oct 2024 17:15:31 +0200 Subject: [PATCH 03/28] Sync Converters --- src/Converters.cxx | 290 +++++++++++++++++++++++++++++++-------------- src/Converters.h | 8 ++ 2 files changed, 210 insertions(+), 88 deletions(-) diff --git a/src/Converters.cxx b/src/Converters.cxx index f2c3495..7117114 100644 --- a/src/Converters.cxx +++ b/src/Converters.cxx @@ -256,7 +256,8 @@ static bool HasLifeLine(PyObject* holder, intptr_t ref) //- helper to work with both CPPInstance and CPPExcInstance ------------------ -static inline CPyCppyy::CPPInstance* GetCppInstance(PyObject* pyobject) +static inline CPyCppyy::CPPInstance* GetCppInstance( + PyObject* pyobject, Cppyy::TCppType_t klass = (Cppyy::TCppType_t)0, bool accept_rvalue = false) { using namespace CPyCppyy; if (CPPInstance_Check(pyobject)) @@ -269,6 +270,22 @@ static inline CPyCppyy::CPPInstance* GetCppInstance(PyObject* pyobject) if (castobj) { if (CPPInstance_Check(castobj)) return (CPPInstance*)castobj; + else if (klass && PyTuple_CheckExact(castobj)) { + // allow implicit conversion from a tuple of arguments + PyObject* pyclass = GetScopeProxy(klass); + if (pyclass) { + CPPInstance* pytmp = (CPPInstance*)PyObject_Call(pyclass, castobj, NULL); + Py_DECREF(pyclass); + if (CPPInstance_Check(pytmp)) { + if (accept_rvalue) + pytmp->fFlags |= CPPInstance::kIsRValue; + Py_DECREF(castobj); + return pytmp; + } + Py_XDECREF(pytmp); + } + } + Py_DECREF(castobj); return nullptr; } @@ -541,7 +558,7 @@ CPPYY_IMPL_BASIC_CONVERTER_METHODS(name, type, stype, ctype, F1, F2) #define CPPYY_IMPL_BASIC_CONVERTER_NB(name, type, stype, ctype, F1, F2, tc) \ bool CPyCppyy::name##Converter::SetArg( \ - PyObject* pyobject, Parameter& para, CallContext* ctxt) \ + PyObject* pyobject, Parameter& para, CallContext* /*ctxt*/) \ { \ if (PyBool_Check(pyobject)) \ return false; \ @@ -813,7 +830,7 @@ CPPYY_IMPL_REFCONVERTER(LLong, c_longlong, long long, 'q'); CPPYY_IMPL_REFCONVERTER(ULLong, c_ulonglong, unsigned long long, 'Q'); CPPYY_IMPL_REFCONVERTER(Float, c_float, float, 'f'); CPPYY_IMPL_REFCONVERTER_FROM_MEMORY(Double, c_double); -CPPYY_IMPL_REFCONVERTER(LDouble, c_longdouble, PY_LONG_DOUBLE, 'D'); +CPPYY_IMPL_REFCONVERTER(LDouble, c_longdouble, PY_LONG_DOUBLE, 'g'); //---------------------------------------------------------------------------- @@ -1217,7 +1234,7 @@ bool CPyCppyy::CStringConverter::SetArg( // use internal buffer as workaround fBuffer = std::string(cstr, len); if (fMaxSize != std::string::npos) - fBuffer.resize(fMaxSize, '\0'); // padd remainder of buffer as needed + fBuffer.resize(fMaxSize, '\0'); // pad remainder of buffer as needed cstr = fBuffer.c_str(); } else SetLifeLine(ctxt->fPyContext, pyobject, (intptr_t)this); @@ -1280,7 +1297,7 @@ bool CPyCppyy::CStringConverter::ToMemory(PyObject* value, void* address, PyObje // the pointer value is non-zero and not ours: assume byte copy if (fMaxSize != std::string::npos) - strncpy(*(char**)address, cstr, fMaxSize); // padds remainder + strncpy(*(char**)address, cstr, fMaxSize); // pads remainder else // coverity[secure_coding] - can't help it, it's intentional. strcpy(*(char**)address, cstr); @@ -1558,7 +1575,7 @@ bool CPyCppyy::VoidArrayConverter::ToMemory(PyObject* value, void* address, PyOb //---------------------------------------------------------------------------- -#define CPPYY_IMPL_ARRAY_CONVERTER(name, ctype, type, code) \ +#define CPPYY_IMPL_ARRAY_CONVERTER(name, ctype, type, code, suffix) \ CPyCppyy::name##ArrayConverter::name##ArrayConverter(cdims_t dims) : \ fShape(dims) { \ fIsFixed = dims ? fShape[0] != UNKNOWN_SIZE : false; \ @@ -1626,8 +1643,8 @@ bool CPyCppyy::name##ArrayConverter::SetArg( \ PyObject* CPyCppyy::name##ArrayConverter::FromMemory(void* address) \ { \ if (!fIsFixed) \ - return CreateLowLevelView((type**)address, fShape); \ - return CreateLowLevelView(*(type**)address, fShape); \ + return CreateLowLevelView##suffix((type**)address, fShape); \ + return CreateLowLevelView##suffix(*(type**)address, fShape); \ } \ \ bool CPyCppyy::name##ArrayConverter::ToMemory( \ @@ -1673,25 +1690,27 @@ bool CPyCppyy::name##ArrayConverter::ToMemory( \ //---------------------------------------------------------------------------- -CPPYY_IMPL_ARRAY_CONVERTER(Bool, c_bool, bool, '?') -CPPYY_IMPL_ARRAY_CONVERTER(SChar, c_char, signed char, 'b') -CPPYY_IMPL_ARRAY_CONVERTER(UChar, c_ubyte, unsigned char, 'B') +CPPYY_IMPL_ARRAY_CONVERTER(Bool, c_bool, bool, '?', ) +CPPYY_IMPL_ARRAY_CONVERTER(SChar, c_char, signed char, 'b', ) +CPPYY_IMPL_ARRAY_CONVERTER(UChar, c_ubyte, unsigned char, 'B', ) #if __cplusplus > 201402L -CPPYY_IMPL_ARRAY_CONVERTER(Byte, c_ubyte, std::byte, 'B') +CPPYY_IMPL_ARRAY_CONVERTER(Byte, c_ubyte, std::byte, 'B', ) #endif -CPPYY_IMPL_ARRAY_CONVERTER(Short, c_short, short, 'h') -CPPYY_IMPL_ARRAY_CONVERTER(UShort, c_ushort, unsigned short, 'H') -CPPYY_IMPL_ARRAY_CONVERTER(Int, c_int, int, 'i') -CPPYY_IMPL_ARRAY_CONVERTER(UInt, c_uint, unsigned int, 'I') -CPPYY_IMPL_ARRAY_CONVERTER(Long, c_long, long, 'l') -CPPYY_IMPL_ARRAY_CONVERTER(ULong, c_ulong, unsigned long, 'L') -CPPYY_IMPL_ARRAY_CONVERTER(LLong, c_longlong, long long, 'q') -CPPYY_IMPL_ARRAY_CONVERTER(ULLong, c_ulonglong, unsigned long long, 'Q') -CPPYY_IMPL_ARRAY_CONVERTER(Float, c_float, float, 'f') -CPPYY_IMPL_ARRAY_CONVERTER(Double, c_double, double, 'd') -CPPYY_IMPL_ARRAY_CONVERTER(LDouble, c_longdouble, long double, 'D') -CPPYY_IMPL_ARRAY_CONVERTER(ComplexF, c_fcomplex, std::complex, 'z') -CPPYY_IMPL_ARRAY_CONVERTER(ComplexD, c_complex, std::complex, 'Z') +CPPYY_IMPL_ARRAY_CONVERTER(Int8, c_byte, int8_t, 'b', _i8) +CPPYY_IMPL_ARRAY_CONVERTER(UInt8, c_ubyte, uint8_t, 'B', _i8) +CPPYY_IMPL_ARRAY_CONVERTER(Short, c_short, short, 'h', ) +CPPYY_IMPL_ARRAY_CONVERTER(UShort, c_ushort, unsigned short, 'H', ) +CPPYY_IMPL_ARRAY_CONVERTER(Int, c_int, int, 'i', ) +CPPYY_IMPL_ARRAY_CONVERTER(UInt, c_uint, unsigned int, 'I', ) +CPPYY_IMPL_ARRAY_CONVERTER(Long, c_long, long, 'l', ) +CPPYY_IMPL_ARRAY_CONVERTER(ULong, c_ulong, unsigned long, 'L', ) +CPPYY_IMPL_ARRAY_CONVERTER(LLong, c_longlong, long long, 'q', ) +CPPYY_IMPL_ARRAY_CONVERTER(ULLong, c_ulonglong, unsigned long long, 'Q', ) +CPPYY_IMPL_ARRAY_CONVERTER(Float, c_float, float, 'f', ) +CPPYY_IMPL_ARRAY_CONVERTER(Double, c_double, double, 'd', ) +CPPYY_IMPL_ARRAY_CONVERTER(LDouble, c_longdouble, long double, 'g', ) +CPPYY_IMPL_ARRAY_CONVERTER(ComplexF, c_fcomplex, std::complex, 'z', ) +CPPYY_IMPL_ARRAY_CONVERTER(ComplexD, c_complex, std::complex, 'Z', ) //---------------------------------------------------------------------------- @@ -1750,17 +1769,21 @@ bool CPyCppyy::CStringArrayConverter::SetArg( //---------------------------------------------------------------------------- PyObject* CPyCppyy::CStringArrayConverter::FromMemory(void* address) { - if (fShape[0] == UNKNOWN_SIZE) - return CreateLowLevelView((const char**)address, fShape); - return CreateLowLevelView(*(const char***)address, fShape); + if (fIsFixed) + return CreateLowLevelView(*(char**)address, fShape); + else if (fShape[0] == UNKNOWN_SIZE) + return CreateLowLevelViewString((const char**)address, fShape); + return CreateLowLevelViewString(*(const char***)address, fShape); } //---------------------------------------------------------------------------- PyObject* CPyCppyy::NonConstCStringArrayConverter::FromMemory(void* address) { - if (fShape[0] == UNKNOWN_SIZE) - return CreateLowLevelView((char**)address, fShape); - return CreateLowLevelView(*(char***)address, fShape); + if (fIsFixed) + return CreateLowLevelView(*(char**)address, fShape); + else if (fShape[0] == UNKNOWN_SIZE) + return CreateLowLevelViewString((char**)address, fShape); + return CreateLowLevelViewString(*(char***)address, fShape); } //- converters for special cases --------------------------------------------- @@ -1844,33 +1867,7 @@ bool CPyCppyy::name##Converter::ToMemory( \ } CPPYY_IMPL_STRING_AS_PRIMITIVE_CONVERTER(STLString, std::string, c_str, size) -#if __cplusplus > 201402L -CPPYY_IMPL_STRING_AS_PRIMITIVE_CONVERTER(STLStringViewBase, std::string_view, data, size) -bool CPyCppyy::STLStringViewConverter::SetArg( - PyObject* pyobject, Parameter& para, CallContext* ctxt) -{ - if (this->STLStringViewBaseConverter::SetArg(pyobject, para, ctxt)) - return true; - if (!CPPInstance_Check(pyobject)) - return false; - - static Cppyy::TCppScope_t sStringID = Cppyy::GetFullScope("std::string"); - CPPInstance* pyobj = (CPPInstance*)pyobject; - if (pyobj->ObjectIsA() == sStringID) { - void* ptr = pyobj->GetObject(); - if (!ptr) - return false; - - fBuffer = *((std::string*)ptr); - para.fValue.fVoidp = &fBuffer; - para.fTypeCode = 'V'; - return true; - } - - return false; -} -#endif CPyCppyy::STLWStringConverter::STLWStringConverter(bool keepControl) : InstanceConverter(Cppyy::GetFullScope("std::wstring"), keepControl) {} @@ -1935,6 +1932,89 @@ bool CPyCppyy::STLWStringConverter::ToMemory(PyObject* value, void* address, PyO } +#if __cplusplus > 201402L +CPyCppyy::STLStringViewConverter::STLStringViewConverter(bool keepControl) : + InstanceConverter(Cppyy::GetFullScope("std::string_view"), keepControl) {} + +bool CPyCppyy::STLStringViewConverter::SetArg( + PyObject* pyobject, Parameter& para, CallContext* ctxt) +{ +// normal instance convertion (eg. string_view object passed) + if (!PyInt_Check(pyobject) && !PyLong_Check(pyobject)) { + CallContextRAII noimp(ctxt); + if (InstanceConverter::SetArg(pyobject, para, ctxt)) { + para.fTypeCode = 'V'; + return true; + } else + PyErr_Clear(); + } + +// passing of a Python string; buffering done Python-side b/c str is immutable + Py_ssize_t len; + const char* cstr = CPyCppyy_PyText_AsStringAndSize(pyobject, &len); + if (cstr) { + SetLifeLine(ctxt->fPyContext, pyobject, (intptr_t)this); + fBuffer = std::string_view(cstr, (std::string_view::size_type)len); + para.fValue.fVoidp = &fBuffer; + para.fTypeCode = 'V'; + return true; + } + + if (!CPPInstance_Check(pyobject)) + return false; + +// special case of a C++ std::string object; life-time management is left to +// the caller to ensure any external changes propagate correctly + if (CPPInstance_Check(pyobject)) { + static Cppyy::TCppScope_t sStringID = Cppyy::GetFullScope("std::string"); + CPPInstance* pyobj = (CPPInstance*)pyobject; + if (pyobj->ObjectIsA() == sStringID) { + void* ptr = pyobj->GetObject(); + if (!ptr) + return false; // leaves prior conversion error for report + + PyErr_Clear(); + + fBuffer = *((std::string*)ptr); + para.fValue.fVoidp = &fBuffer; + para.fTypeCode = 'V'; + return true; + } + } + + return false; +} + +PyObject* CPyCppyy::STLStringViewConverter::FromMemory(void* address) +{ + if (address) + return InstanceConverter::FromMemory(address); + auto* empty = new std::string_view(); + return BindCppObjectNoCast(empty, fClass, CPPInstance::kIsOwner); +} + +bool CPyCppyy::STLStringViewConverter::ToMemory( + PyObject* value, void* address, PyObject* ctxt) +{ +// common case of simple object assignment + if (InstanceConverter::ToMemory(value, address, ctxt)) + return true; + +// assignment of a Python string; buffering done Python-side b/c str is immutable + Py_ssize_t len; + const char* cstr = CPyCppyy_PyText_AsStringAndSize(value, &len); + if (cstr) { + SetLifeLine(ctxt, value, (intptr_t)this); + *reinterpret_cast(address) = \ + std::string_view(cstr, (std::string_view::size_type)len); + return true; + } + + return false; +} +#endif + + bool CPyCppyy::STLStringMoveConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* ctxt) { @@ -1969,7 +2049,7 @@ bool CPyCppyy::InstancePtrConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* ctxt) { // convert to C++ instance*, set arg for call - CPPInstance* pyobj = GetCppInstance(pyobject); + CPPInstance* pyobj = GetCppInstance(pyobject, ISCONST ? fClass : (Cppyy::TCppType_t)0); if (!pyobj) { if (GetAddressSpecialCase(pyobject, para.fValue.fVoidp)) { para.fTypeCode = 'p'; // allow special cases such as nullptr @@ -2021,7 +2101,7 @@ template bool CPyCppyy::InstancePtrConverter::ToMemory(PyObject* value, void* address, PyObject* /* ctxt */) { // convert to C++ instance, write it at
- CPPInstance* pyobj = GetCppInstance(value); + CPPInstance* pyobj = GetCppInstance(value, ISCONST ? fClass : (Cppyy::TCppType_t)0); if (!pyobj) { void* ptr = nullptr; if (GetAddressSpecialCase(value, ptr)) { @@ -2052,7 +2132,7 @@ bool CPyCppyy::InstanceConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* ctxt) { // convert to C++ instance, set arg for call - CPPInstance* pyobj = GetCppInstance(pyobject); + CPPInstance* pyobj = GetCppInstance(pyobject, fClass); if (pyobj) { auto oisa = pyobj->ObjectIsA(); if (oisa && ((oisa == (Cppyy::IsTypedefed(fClass) @@ -2108,7 +2188,7 @@ bool CPyCppyy::InstanceRefConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* ctxt) { // convert to C++ instance&, set arg for call - CPPInstance* pyobj = GetCppInstance(pyobject); + CPPInstance* pyobj = GetCppInstance(pyobject, fIsConst ? fClass : (Cppyy::TCppType_t)0); if (pyobj) { // reject moves @@ -2170,7 +2250,7 @@ bool CPyCppyy::InstanceMoveConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* ctxt) { // convert to C++ instance&&, set arg for call - CPPInstance* pyobj = GetCppInstance(pyobject); + CPPInstance* pyobj = GetCppInstance(pyobject, fClass, true /* accept_rvalue */); if (!pyobj || (pyobj->fFlags & CPPInstance::kIsLValue)) { // implicit conversion is fine as the temporary by definition is moveable return (bool)ConvertImplicit(fClass, pyobject, para, ctxt); @@ -2473,7 +2553,7 @@ static PyObject* WrapperCacheEraser(PyObject*, PyObject* pyref) Py_RETURN_NONE; } static PyMethodDef gWrapperCacheEraserMethodDef = { - const_cast("interal_WrapperCacheEraser"), + const_cast("internal_WrapperCacheEraser"), (PyCFunction)WrapperCacheEraser, METH_O, nullptr }; @@ -2573,7 +2653,7 @@ static void* PyFunction_AsCPointer(PyObject* pyobject, // start function body Utility::ConstructCallbackPreamble(rettype, argtypes, code); - // create a referencable pointer + // create a referenceable pointer PyObject** ref = new PyObject*{pyobject}; // function call itself and cleanup @@ -2674,12 +2754,9 @@ bool CPyCppyy::StdFunctionConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* ctxt) { // prefer normal "object" conversion - bool rf = ctxt->fFlags & CallContext::kNoImplicit; - ctxt->fFlags |= CallContext::kNoImplicit; - if (fConverter->SetArg(pyobject, para, ctxt)) { - if (!rf) ctxt->fFlags &= ~CallContext::kNoImplicit; + CallContextRAII noimp(ctxt); + if (fConverter->SetArg(pyobject, para, ctxt)) return true; - } PyErr_Clear(); @@ -2693,12 +2770,10 @@ bool CPyCppyy::StdFunctionConverter::SetArg( bool result = fConverter->SetArg(func, para, ctxt); if (result) ctxt->AddTemporary(func); else Py_DECREF(func); - if (!rf) ctxt->fFlags &= ~CallContext::kNoImplicit; return result; } } - if (!rf) ctxt->fFlags &= ~CallContext::kNoImplicit; return false; } @@ -2833,9 +2908,20 @@ struct faux_initlist } // unnamed namespace +CPyCppyy::InitializerListConverter::InitializerListConverter(Cppyy::TCppType_t klass, std::string const &value_type) + + : InstanceConverter{klass}, + fValueTypeName{value_type}, + fValueType{Cppyy::GetScope(value_type)}, + fValueSize{Cppyy::SizeOf(value_type)} +{ +} + CPyCppyy::InitializerListConverter::~InitializerListConverter() { - if (fConverter && fConverter->HasState()) delete fConverter; + for (Converter *converter : fConverters) { + if (converter && converter->HasState()) delete converter; + } if (fBuffer) Clear(); } @@ -2894,6 +2980,12 @@ bool CPyCppyy::InitializerListConverter::SetArg( fake->_Last = fake->_M_array+buflen*fValueSize; #endif } else if (fValueSize) { + // Remove any errors set by GetBuffer(); note that if the argument was an array + // that failed to extract because of a type mismatch, the following will perform + // a (rather inefficient) copy. No warning is issued b/c e.g. numpy doesn't do + // so either. + PyErr_Clear(); + // can only construct empty lists, so use a fake initializer list size_t len = (size_t)PySequence_Size(pyobject); fake = (faux_initlist*)malloc(sizeof(faux_initlist)+fValueSize*len); @@ -2909,7 +3001,8 @@ bool CPyCppyy::InitializerListConverter::SetArg( PyObject* item = PySequence_GetItem(pyobject, i); bool convert_ok = false; if (item) { - if (!fConverter) { + Converter *converter = CreateConverter(fValueTypeName); + if (!converter) { if (CPPInstance_Check(item)) { // by convention, use byte copy memcpy((char*)fake->_M_array + i*fValueSize, @@ -2924,8 +3017,15 @@ bool CPyCppyy::InitializerListConverter::SetArg( // need not be a C++ object memloc = (void*)Cppyy::Construct(fValueType, memloc); if (memloc) entries += 1; + else { + PyErr_SetString(PyExc_TypeError, + "default ctor needed for initializer list of objects"); + } + } + if (memloc) { + convert_ok = converter->ToMemory(item, memloc); } - if (memloc) convert_ok = fConverter->ToMemory(item, memloc); + fConverters.emplace_back(converter); } @@ -2975,7 +3075,7 @@ static inline CPyCppyy::Converter* selectInstanceCnv(Cppyy::TCppScope_t klass, else if (cpd == "*&") result = new InstancePtrPtrConverter(klass, control); else if (cpd == "*" && dims.ndim() == UNKNOWN_SIZE) { - if (isConst) result = new InstancePtrConverter(klass); + if (isConst) result = new InstancePtrConverter(klass, control); else result = new InstancePtrConverter(klass, control); } else if (cpd == "&") @@ -3079,8 +3179,7 @@ CPyCppyy::Converter* CPyCppyy::CreateConverter(const std::string& fullType, cdim // get the type of the list and create a converter (TODO: get hold of value_type?) auto pos = realType.find('<'); std::string value_type = realType.substr(pos+1, realType.size()-pos-2); - return new InitializerListConverter(Cppyy::GetScope(realType), - CreateConverter(value_type), Cppyy::GetScope(value_type), Cppyy::SizeOf(value_type)); + return new InitializerListConverter(Cppyy::GetScope(realType), value_type); } //-- still nothing? use a generalized converter @@ -3125,12 +3224,7 @@ CPyCppyy::Converter* CPyCppyy::CreateConverter(const std::string& fullType, cdim if (!result) { // CLING WORKAROUND -- special case for STL iterators - if (realType.rfind("__gnu_cxx::__normal_iterator", 0) /* vector */ == 0 -#ifdef __APPLE__ - || realType.rfind("__wrap_iter", 0) == 0 -#endif - // TODO: Windows? - ) { + if (Utility::IsSTLIterator(realType)) { static STLIteratorConverter c; result = &c; } else @@ -3403,6 +3497,23 @@ bool CPyCppyy::RegisterConverter(const std::string& name, cf_t fac) return true; } +//---------------------------------------------------------------------------- +CPYCPPYY_EXPORT +bool CPyCppyy::RegisterConverterAlias(const std::string& name, const std::string& target) +{ +// register a custom converter that is a reference to an existing converter + auto f = gConvFactories.find(name); + if (f != gConvFactories.end()) + return false; + + auto t = gConvFactories.find(target); + if (t == gConvFactories.end()) + return false; + + gConvFactories[name] = t->second; + return true; +} + //---------------------------------------------------------------------------- CPYCPPYY_EXPORT bool CPyCppyy::UnregisterConverter(const std::string& name) @@ -3516,9 +3627,12 @@ static struct InitConvFactories_t { gf["const unsigned char*"] = (cf_t)+[](cdims_t d) { return new UCharArrayConverter{d}; }; gf["unsigned char ptr"] = (cf_t)+[](cdims_t d) { return new UCharArrayConverter{d}; }; gf["UCharAsInt*"] = gf["unsigned char ptr"]; + gf["UCharAsInt[]"] = gf["unsigned char ptr"]; #if __cplusplus > 201402L gf["std::byte ptr"] = (cf_t)+[](cdims_t d) { return new ByteArrayConverter{d}; }; #endif + gf["int8_t ptr"] = (cf_t)+[](cdims_t d) { return new Int8ArrayConverter{d}; }; + gf["uint8_t ptr"] = (cf_t)+[](cdims_t d) { return new UInt8ArrayConverter{d}; }; gf["short ptr"] = (cf_t)+[](cdims_t d) { return new ShortArrayConverter{d}; }; gf["unsigned short ptr"] = (cf_t)+[](cdims_t d) { return new UShortArrayConverter{d}; }; gf["int ptr"] = (cf_t)+[](cdims_t d) { return new IntArrayConverter{d}; }; @@ -3540,7 +3654,7 @@ static struct InitConvFactories_t { #if __cplusplus > 201402L gf["std::byte"] = gf["uint8_t"]; gf["const std::byte&"] = gf["const uint8_t&"]; - gf["std::byte&"] = gf["uint8&"]; + gf["std::byte&"] = gf["uint8_t&"]; #endif gf["std::int8_t"] = gf["int8_t"]; gf["const std::int8_t&"] = gf["const int8_t&"]; @@ -3574,7 +3688,7 @@ static struct InitConvFactories_t { gf["const char*&&"] = gf["const char*"]; gf["const char[]"] = (cf_t)+[](cdims_t) { return new CStringConverter{}; }; gf["char*"] = (cf_t)+[](cdims_t d) { return new NonConstCStringConverter{dims2stringsz(d)}; }; - gf["char[]"] = (cf_t)+[](cdims_t d) { return new NonConstCStringArrayConverter{d}; }; + gf["char[]"] = (cf_t)+[](cdims_t d) { return new NonConstCStringArrayConverter{d, true}; }; gf["signed char*"] = gf["char*"]; gf["wchar_t*"] = (cf_t)+[](cdims_t) { return new WCStringConverter{}; }; gf["char16_t*"] = (cf_t)+[](cdims_t) { return new CString16Converter{}; }; @@ -3584,10 +3698,10 @@ static struct InitConvFactories_t { // TODO: the following are handled incorrectly upstream (char16_t** where char16_t* intended)?! gf["char16_t**"] = gf["char16_t*"]; gf["char32_t**"] = gf["char32_t*"]; - gf["const char**"] = (cf_t)+[](cdims_t) { return new CStringArrayConverter{{UNKNOWN_SIZE, UNKNOWN_SIZE}}; }; + gf["const char**"] = (cf_t)+[](cdims_t) { return new CStringArrayConverter{{UNKNOWN_SIZE, UNKNOWN_SIZE}, false}; }; gf["char**"] = gf["const char**"]; - gf["const char*[]"] = (cf_t)+[](cdims_t d) { return new CStringArrayConverter{d}; }; - gf["char*[]"] = (cf_t)+[](cdims_t d) { return new NonConstCStringArrayConverter{d}; }; + gf["const char*[]"] = (cf_t)+[](cdims_t d) { return new CStringArrayConverter{d, false}; }; + gf["char*[]"] = (cf_t)+[](cdims_t d) { return new NonConstCStringArrayConverter{d, false}; }; gf["char ptr"] = gf["char*[]"]; gf["std::basic_string"] = (cf_t)+[](cdims_t) { return new STLStringConverter{}; }; gf["const std::basic_string&"] = gf["std::basic_string"]; diff --git a/src/Converters.h b/src/Converters.h index e7cdd05..9f2f714 100644 --- a/src/Converters.h +++ b/src/Converters.h @@ -17,6 +17,13 @@ class CPYCPPYY_CLASS_EXPORT Converter { public: virtual ~Converter(); + Converter() = default; + + Converter(Converter const& other) = delete; + Converter(Converter && other) = delete; + Converter& operator=(Converter const& other) = delete; + Converter& operator=(Converter && other) = delete; + public: virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr) = 0; virtual PyObject* FromMemory(void* address); @@ -31,6 +38,7 @@ CPYCPPYY_EXPORT Converter* CreateConverter(Cppyy::TCppType_t type, cdims_t dims CPYCPPYY_EXPORT void DestroyConverter(Converter* p); typedef Converter* (*cf_t)(cdims_t d); CPYCPPYY_EXPORT bool RegisterConverter(const std::string& name, cf_t fac); +CPYCPPYY_EXPORT bool RegisterConverterAlias(const std::string& name, const std::string& target); CPYCPPYY_EXPORT bool UnregisterConverter(const std::string& name); From eacb942b69e967e0819ef6868715fa68c67dfcdb Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Thu, 24 Oct 2024 17:16:42 +0200 Subject: [PATCH 04/28] Sync CPPClassMethod --- src/CPPClassMethod.cxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CPPClassMethod.cxx b/src/CPPClassMethod.cxx index cde26a3..c4cdf2f 100644 --- a/src/CPPClassMethod.cxx +++ b/src/CPPClassMethod.cxx @@ -24,7 +24,7 @@ PyObject* CPyCppyy::CPPClassMethod::Call(CPPInstance*& // translate the arguments #if PY_VERSION_HEX >= 0x03080000 -// TODO: The following is not robust and should be revisited e.g. by makeing CPPOverloads +// TODO: The following is not robust and should be revisited e.g. by making CPPOverloads // that have only CPPClassMethods be true Python classmethods? Note that the original // implementation wasn't 100% correct either (e.g. static size() mapped to len()). // @@ -35,7 +35,7 @@ PyObject* CPyCppyy::CPPClassMethod::Call(CPPInstance*& if ((!self || (PyObject*)self == Py_None) && nargs) { PyObject* arg0 = CPyCppyy_PyArgs_GET_ITEM(args, 0); if ((CPPInstance_Check(arg0) && ((CPPInstance*)arg0)->ObjectIsA() == GetScope()) && \ - (fArgsRequired <= nargs-1) && (GetMaxArgs() < nargs)) { + (fArgsRequired <= nargs-1)) { args += 1; // drops first argument nargsf -= 1; } From aec611acf73f150ef6bd28694790376ab22ee012 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Thu, 24 Oct 2024 17:25:46 +0200 Subject: [PATCH 05/28] Sync CPPDataMember --- src/CPPDataMember.cxx | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/CPPDataMember.cxx b/src/CPPDataMember.cxx index 5258e1a..8fc6971 100644 --- a/src/CPPDataMember.cxx +++ b/src/CPPDataMember.cxx @@ -18,7 +18,6 @@ #include #include - namespace CPyCppyy { enum ETypeDetails { @@ -35,7 +34,7 @@ enum ETypeDetails { static PyObject* dm_get(CPPDataMember* dm, CPPInstance* pyobj, PyObject* /* kls */) { // cache lookup for low level views - if (dm->fFlags & kIsCachable) { + if (pyobj && dm->fFlags & kIsCachable) { CPyCppyy::CI_DatamemberCache_t& cache = pyobj->GetDatamemberCache(); for (auto it = cache.begin(); it != cache.end(); ++it) { if (it->first == dm->fOffset) { @@ -49,6 +48,11 @@ static PyObject* dm_get(CPPDataMember* dm, CPPInstance* pyobj, PyObject* /* kls } } +// non-initialized or public data accesses through class (e.g. by help()) + void* address = dm->GetAddress(pyobj); + if (!address || (intptr_t)address == -1 /* Cling error */) + return nullptr; + if (dm->fFlags & (kIsEnumPrep | kIsEnumType)) { if (dm->fFlags & kIsEnumPrep) { // still need to do lookup; only ever try this once, then fallback on converter @@ -220,7 +224,7 @@ static void dm_dealloc(CPPDataMember* dm) static PyMemberDef dm_members[] = { {(char*)"__doc__", T_OBJECT, offsetof(CPPDataMember, fDoc), 0, (char*)"writable documentation"}, - {NULL} /* Sentinel */ + {NULL, 0, 0, 0, nullptr} /* Sentinel */ }; //= CPyCppyy datamember proxy access to internals ============================ @@ -308,10 +312,17 @@ PyTypeObject CPPDataMember_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; } // namespace CPyCppyy + //- public members ----------------------------------------------------------- void CPyCppyy::CPPDataMember::Set(Cppyy::TCppScope_t scope, Cppyy::TCppScope_t data) { From b2856d0d4920836afedb7a1fd16e452f8c5d5f15 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 12:06:18 +0100 Subject: [PATCH 06/28] Sync CPPEnum --- src/CPPEnum.cxx | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/CPPEnum.cxx b/src/CPPEnum.cxx index 6dca11c..45a453a 100644 --- a/src/CPPEnum.cxx +++ b/src/CPPEnum.cxx @@ -2,6 +2,7 @@ #include "CPyCppyy.h" #include "CPPEnum.h" #include "PyStrings.h" +#include "TypeManip.h" #include "Utility.h" @@ -18,9 +19,9 @@ static PyObject* pytype_from_enum_type(const std::string& enum_type) } //---------------------------------------------------------------------------- -PyObject* CPyCppyy::pyval_from_enum(const std::string& enum_type, PyObject* pytype, - PyObject* btype, Cppyy::TCppScope_t enum_constant) { - long long llval = Cppyy::GetEnumDataValue(enum_constant); +static PyObject* pyval_from_enum(const std::string& enum_type, PyObject* pytype, + PyObject* btype, Cppyy::TCppEnum_t etype, Cppyy::TCppIndex_t idata) { + long long llval = Cppyy::GetEnumDataValue(etype, idata); if (enum_type == "bool") { PyObject* result = (bool)llval ? Py_True : Py_False; @@ -37,13 +38,11 @@ PyObject* CPyCppyy::pyval_from_enum(const std::string& enum_type, PyObject* pyty else bval = PyLong_FromLongLong(llval); - if (pytype && btype) { PyObject* args = PyTuple_New(1); PyTuple_SET_ITEM(args, 0, bval); - bval = ((PyTypeObject*)btype)->tp_new((PyTypeObject*)pytype, args, nullptr); + PyObject* result = ((PyTypeObject*)btype)->tp_new((PyTypeObject*)pytype, args, nullptr); Py_DECREF(args); - } - return bval; + return result; } @@ -60,8 +59,6 @@ static PyObject* enum_repr(PyObject* self) { using namespace CPyCppyy; - PyObject* kls_scope = PyObject_GetAttr((PyObject*)Py_TYPE(self), PyStrings::gThisModule); - if (!kls_scope) PyErr_Clear(); PyObject* kls_cppname = PyObject_GetAttr((PyObject*)Py_TYPE(self), PyStrings::gCppName); if (!kls_cppname) PyErr_Clear(); PyObject* obj_cppname = PyObject_GetAttr(self, PyStrings::gCppName); @@ -70,7 +67,7 @@ static PyObject* enum_repr(PyObject* self) PyObject* repr = nullptr; if (kls_cppname && obj_cppname && obj_str) { - const std::string resolved = Cppyy::ResolveEnum(PyLong_AsVoidPtr(kls_scope)); + const std::string resolved = Cppyy::ResolveEnum(CPyCppyy_PyText_AsString(kls_cppname)); repr = CPyCppyy_PyText_FromFormat("(%s::%s) : (%s) %s", CPyCppyy_PyText_AsString(kls_cppname), CPyCppyy_PyText_AsString(obj_cppname), resolved.c_str(), CPyCppyy_PyText_AsString(obj_str)); @@ -140,12 +137,12 @@ CPyCppyy::CPPEnum* CPyCppyy::CPPEnum_New(const std::string& name, Cppyy::TCppSco CPPEnum* pyenum = nullptr; - Cppyy::TCppScope_t etype = scope; - const std::string& ename = Cppyy::GetScopedFinalName(scope); + const std::string& ename = scope == Cppyy::gGlobalScope ? name : Cppyy::GetScopedFinalName(scope)+"::"+name; + Cppyy::TCppEnum_t etype = Cppyy::GetEnum(scope, name); if (etype) { // create new enum type with labeled values in place, with a meta-class // to make sure the enum values are read-only - const std::string& resolved = Cppyy::ResolveEnum(etype); + const std::string& resolved = Cppyy::ResolveEnum(ename); PyObject* pyside_type = pytype_from_enum_type(resolved); PyObject* pymetabases = PyTuple_New(1); PyObject* btype = (PyObject*)Py_TYPE(pyside_type); @@ -173,6 +170,14 @@ CPyCppyy::CPPEnum* CPyCppyy::CPPEnum_New(const std::string& name, Cppyy::TCppSco PyDict_SetItem(dct, PyStrings::gUnderlying, pyresolved); Py_DECREF(pyresolved); + // add the __module__ to allow pickling + std::string modname = TypeManip::extract_namespace(ename); + TypeManip::cppscope_to_pyscope(modname); // :: -> . + if (!modname.empty()) modname = "."+modname; + PyObject* pymodname = CPyCppyy_PyText_FromString(("cppyy.gbl"+modname).c_str()); + PyDict_SetItem(dct, PyStrings::gModule, pymodname); + Py_DECREF(pymodname); + // create the actual enum class args = Py_BuildValue((char*)"sOO", name.c_str(), pybases, dct); Py_DECREF(pybases); @@ -186,10 +191,10 @@ CPyCppyy::CPPEnum* CPyCppyy::CPPEnum_New(const std::string& name, Cppyy::TCppSco ((PyTypeObject*)pyenum)->tp_str = ((PyTypeObject*)pyside_type)->tp_repr; // collect the enum values - std::vector econstants = Cppyy::GetEnumConstants(etype); - for (auto *econstant : econstants) { - PyObject* val = pyval_from_enum(resolved, pyenum, pyside_type, econstant); - PyObject* pydname = CPyCppyy_PyText_FromString(Cppyy::GetFinalName(econstant).c_str()); + Cppyy::TCppIndex_t ndata = Cppyy::GetNumEnumData(etype); + for (Cppyy::TCppIndex_t idata = 0; idata < ndata; ++idata) { + PyObject* val = pyval_from_enum(resolved, pyenum, pyside_type, etype, idata); + PyObject* pydname = CPyCppyy_PyText_FromString(Cppyy::GetEnumDataName(etype, idata).c_str()); PyObject_SetAttr(pyenum, pydname, val); PyObject_SetAttr(val, PyStrings::gCppName, pydname); Py_DECREF(pydname); From 8db92a680d6da7e3b6a886e6e5faf822192bc2c2 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 12:07:22 +0100 Subject: [PATCH 07/28] Sync CPPExcInstance --- src/CPPExcInstance.cxx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/CPPExcInstance.cxx b/src/CPPExcInstance.cxx index 9660938..4ed4c3f 100644 --- a/src/CPPExcInstance.cxx +++ b/src/CPPExcInstance.cxx @@ -29,7 +29,7 @@ static PyObject* ep_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) PyObject* ulc = PyObject_GetAttr((PyObject*)subtype, PyStrings::gUnderlying); excobj->fCppInstance = PyType_Type.tp_call(ulc, args, kwds); if (!excobj->fCppInstance) { - // if this fails, then the contruction may have been attempted from a string + // if this fails, then the construction may have been attempted from a string // (e.g. from PyErr_Format); if so, drop the proxy and use fTopMessage instead PyErr_Clear(); if (PyTuple_GET_SIZE(args) == 1) { @@ -282,6 +282,12 @@ PyTypeObject CPPExcInstance_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; } // namespace CPyCppyy From 47751b23fcf87433f645a387cbec5d9c7a93528a Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 12:10:22 +0100 Subject: [PATCH 08/28] Sync CPPInstance --- src/CPPInstance.cxx | 48 ++++++++++++++++++++++++++++++++++++++++----- src/CPPInstance.h | 3 ++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/CPPInstance.cxx b/src/CPPInstance.cxx index 4626d33..f990661 100644 --- a/src/CPPInstance.cxx +++ b/src/CPPInstance.cxx @@ -230,6 +230,13 @@ void CPyCppyy::op_dealloc_nofree(CPPInstance* pyobj) { namespace CPyCppyy { +//---------------------------------------------------------------------------- +static int op_traverse(CPPInstance* /*pyobj*/, visitproc /*visit*/, void* /*arg*/) +{ + return 0; +} + + //= CPyCppyy object proxy null-ness checking ================================= static int op_nonzero(CPPInstance* self) { @@ -314,6 +321,12 @@ void CPyCppyy::CPPInstance::CastToArray(Py_ssize_t sz) ARRAY_SIZE(this) = sz; } +Py_ssize_t CPyCppyy::CPPInstance::ArrayLength() { + if (!(fFlags & kIsArray)) + return -1; + return (Py_ssize_t)ARRAY_SIZE(this); +} + static PyObject* op_reshape(CPPInstance* self, PyObject* shape) { // Allow the user to fix up the actual (type-strided) size of the buffer. @@ -323,14 +336,17 @@ static PyObject* op_reshape(CPPInstance* self, PyObject* shape) } long sz = PyLong_AsLong(PyTuple_GET_ITEM(shape, 0)); - if (sz == -1) return nullptr; + if (sz <= 0) { + PyErr_SetString(PyExc_ValueError, "array length must be positive"); + return nullptr; + } self->CastToArray(sz); Py_RETURN_NONE; } -static PyObject* op_getitem(CPPInstance* self, PyObject* pyidx) +static PyObject* op_item(CPPInstance* self, Py_ssize_t idx) { // In C, it is common to represent an array of structs as a pointer to the first // object in the array. If the caller indexes a pointer to an object that does not @@ -374,6 +390,21 @@ static PyObject* op_getitem(CPPInstance* self, PyObject* pyidx) return BindCppObjectNoCast(indexed_obj, ((CPPClass*)Py_TYPE(self))->fCppType, flags); } +//- sequence methods -------------------------------------------------------- +static PySequenceMethods op_as_sequence = { + 0, // sq_length + 0, // sq_concat + 0, // sq_repeat + (ssizeargfunc)op_item, // sq_item + 0, // sq_slice + 0, // sq_ass_item + 0, // sq_ass_slice + 0, // sq_contains + 0, // sq_inplace_concat + 0, // sq_inplace_repeat +}; + + //---------------------------------------------------------------------------- static PyMethodDef op_methods[] = { {(char*)"__destruct__", (PyCFunction)op_destruct, METH_NOARGS, @@ -1025,7 +1056,7 @@ PyTypeObject CPPInstance_Type = { 0, // tp_as_async / tp_compare (reprfunc)op_repr, // tp_repr &op_as_number, // tp_as_number - 0, // tp_as_sequence + &op_as_sequence, // tp_as_sequence 0, // tp_as_mapping (hashfunc)op_hash, // tp_hash 0, // tp_call @@ -1035,9 +1066,10 @@ PyTypeObject CPPInstance_Type = { 0, // tp_as_buffer Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_CHECKTYPES, // tp_flags + Py_TPFLAGS_CHECKTYPES | + Py_TPFLAGS_HAVE_GC, // tp_flags (char*)"cppyy object proxy (internal)", // tp_doc - 0, // tp_traverse + (traverseproc)op_traverse, // tp_traverse (inquiry)op_clear, // tp_clear (richcmpfunc)op_richcompare, // tp_richcompare 0, // tp_weaklistoffset @@ -1070,6 +1102,12 @@ PyTypeObject CPPInstance_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; } // namespace CPyCppyy diff --git a/src/CPPInstance.h b/src/CPPInstance.h index 810ac18..bbde1ac 100644 --- a/src/CPPInstance.h +++ b/src/CPPInstance.h @@ -75,11 +75,12 @@ class CPPInstance { void* GetSmartObject() { return GetObjectRaw(); } Cppyy::TCppType_t GetSmartIsA() const; -// cross-inheritence dispatch +// cross-inheritance dispatch void SetDispatchPtr(void*); // redefine pointer to object as fixed-size array void CastToArray(Py_ssize_t sz); + Py_ssize_t ArrayLength(); private: void CreateExtension(); From 1d86915edc67316d741cfd91f25006eea6b081cc Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 12:20:45 +0100 Subject: [PATCH 09/28] Sync CPPMethod --- src/CPPMethod.cxx | 67 ++++++++++++++++++++++++++++++++++++++++++++--- src/CPPMethod.h | 2 ++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/src/CPPMethod.cxx b/src/CPPMethod.cxx index bb3bc5b..c050342 100644 --- a/src/CPPMethod.cxx +++ b/src/CPPMethod.cxx @@ -370,12 +370,40 @@ CPyCppyy::CPPMethod::~CPPMethod() //- public members ----------------------------------------------------------- +/** + * @brief Construct a Python string from the method's prototype + * + * @param fa Show formal arguments of the method + * @return PyObject* A Python string with the full method prototype, namespaces included. + * + * For example, given: + * + * int foo(int x); + * + * namespace a { + * namespace b { + * namespace c { + * int foo(int x); + * }}} + * + * This function returns: + * + * 'int foo(int x)' + * 'int a::b::c::foo(int x)' + */ PyObject* CPyCppyy::CPPMethod::GetPrototype(bool fa) { -// construct python string from the method's prototype - return CPyCppyy_PyText_FromFormat("%s%s %s%s", + // Gather the fully qualified final scope of the method. This includes + // all namespaces up to the one where the method is declared, for example: + // namespace a { namespace b { void foo(); }} + // gives + // a::b + std::string finalscope = Cppyy::GetScopedFinalName(fScope); + return CPyCppyy_PyText_FromFormat("%s%s %s%s%s%s", (Cppyy::IsStaticMethod(fMethod) ? "static " : ""), Cppyy::GetMethodReturnTypeAsString(fMethod).c_str(), + finalscope.c_str(), + (finalscope.empty() ? "" : "::"), // Add final set of '::' if the method is scoped in namespace(s) Cppyy::GetScopedFinalName(fMethod).c_str(), GetSignatureString(fa).c_str()); } @@ -488,7 +516,7 @@ int CPyCppyy::CPPMethod::GetPriority() const std::string& clean_name = TypeManip::clean_type(aname, false); Cppyy::TCppScope_t scope = Cppyy::GetScope(clean_name); if (scope) - priority += (int)Cppyy::GetNumBases(scope); + priority += static_cast(Cppyy::GetNumBasesLongestBranch(scope)); if (Cppyy::IsEnumScope(scope)) priority -= 100; @@ -656,6 +684,39 @@ Cppyy::TCppFuncAddr_t CPyCppyy::CPPMethod::GetFunctionAddress() return Cppyy::GetFunctionAddress(fMethod, false /* don't check fast path envar */); } +//---------------------------------------------------------------------------- +int CPyCppyy::CPPMethod::GetArgMatchScore(PyObject* args_tuple) +{ + Py_ssize_t n = PyTuple_Size(args_tuple); + + int req_args = Cppyy::GetMethodReqArgs(fMethod); + + // Not enough arguments supplied: no match + if (req_args > n) + return INT_MAX; + + size_t score = 0; + for (int i = 0; i < n; i++) { + PyObject *pItem = PyTuple_GetItem(args_tuple, i); + if(!CPyCppyy_PyText_Check(pItem)) { + PyErr_SetString(PyExc_TypeError, "argument types should be in string format"); + return INT_MAX; + } + std::string req_type(CPyCppyy_PyText_AsString(pItem)); + + size_t arg_score = Cppyy::CompareMethodArgType(fMethod, i, req_type); + + // Method is not compatible if even one argument does not match + if (arg_score >= 10) { + score = INT_MAX; + break; + } + + score += arg_score; + } + + return score; +} //---------------------------------------------------------------------------- bool CPyCppyy::CPPMethod::Initialize(CallContext* ctxt) diff --git a/src/CPPMethod.h b/src/CPPMethod.h index 4795431..62ba6e1 100644 --- a/src/CPPMethod.h +++ b/src/CPPMethod.h @@ -69,6 +69,8 @@ class CPPMethod : public PyCallable { virtual PyCallable* Clone() { return new CPPMethod(*this); } + virtual int GetArgMatchScore(PyObject* args_tuple); + public: virtual PyObject* Call(CPPInstance*& self, CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr); From 65643a50a37d9b233ab4e645ee8ccd3ad7ab605d Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 12:23:18 +0100 Subject: [PATCH 10/28] Sync CPPOverload --- src/CPPOverload.cxx | 99 ++++++++++++++++++++++++++++++++++++++++----- src/CPPOverload.h | 1 + 2 files changed, 91 insertions(+), 9 deletions(-) diff --git a/src/CPPOverload.cxx b/src/CPPOverload.cxx index 1909614..1a7fe7f 100644 --- a/src/CPPOverload.cxx +++ b/src/CPPOverload.cxx @@ -342,8 +342,15 @@ static PyObject* mp_func_closure(CPPOverload* /* pymeth */, void*) Py_RETURN_NONE; } +// To declare a variable as unused only when compiling for Python 3. +#if PY_VERSION_HEX < 0x03000000 +#define CPyCppyy_Py3_UNUSED(name) name +#else +#define CPyCppyy_Py3_UNUSED(name) +#endif + //---------------------------------------------------------------------------- -static PyObject* mp_func_code(CPPOverload* pymeth, void*) +static PyObject* mp_func_code(CPPOverload* CPyCppyy_Py3_UNUSED(pymeth), void*) { // Code details are used in module inspect to fill out interactive help() #if PY_VERSION_HEX < 0x03000000 @@ -410,7 +417,6 @@ static PyObject* mp_func_code(CPPOverload* pymeth, void*) return code; #else // not important for functioning of most code, so not implemented for p3 for now (TODO) - pymeth = 0; Py_RETURN_NONE; #endif } @@ -821,7 +827,7 @@ static CPPOverload* mp_descr_get(CPPOverload* pymeth, CPPInstance* pyobj, PyObje } // vector calls don't get here, unless a method is looked up on an instance, for -// e.g. class mathods (C++ static); notify downstream to expect a 'self' +// e.g. class methods (C++ static); notify downstream to expect a 'self' newPyMeth->fFlags |= CallContext::kFromDescr; #else @@ -925,13 +931,22 @@ static PyObject* mp_overload(CPPOverload* pymeth, PyObject* args) { // Select and call a specific C++ overload, based on its signature. const char* sigarg = nullptr; + PyObject* sigarg_tuple = nullptr; int want_const = -1; - if (PyTuple_GET_SIZE(args) && \ - !PyArg_ParseTuple(args, const_cast("s|i:__overload__"), &sigarg, &want_const)) + Py_ssize_t args_size = PyTuple_GET_SIZE(args); + if (args_size && + PyArg_ParseTuple(args, const_cast("s|i:__overload__"), &sigarg, &want_const)) { + want_const = args_size == 1 ? -1 : want_const; + return pymeth->FindOverload(sigarg ? sigarg : "", want_const); + } else if (args_size && + PyArg_ParseTuple(args, const_cast("O|i:__overload__"), &sigarg_tuple, &want_const)) { + PyErr_Clear(); + want_const = args_size == 1 ? -1 : want_const; + return pymeth->FindOverload(sigarg_tuple, want_const); + } else { + PyErr_Format(PyExc_TypeError, "Unexpected arguments to __overload__"); return nullptr; - want_const = PyTuple_GET_SIZE(args) == 1 ? -1 : want_const; - - return pymeth->FindOverload(sigarg ? sigarg : "", want_const); + } } static PyObject* mp_add_overload(CPPOverload* pymeth, PyObject* new_overload) @@ -1034,6 +1049,12 @@ PyTypeObject CPPOverload_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; } // namespace CPyCppyy @@ -1095,7 +1116,6 @@ PyObject* CPyCppyy::CPPOverload::FindOverload(const std::string& signature, int CPPOverload::Methods_t& methods = fMethodInfo->fMethods; for (auto& meth : methods) { - bool found = accept_any; if (!found) { PyObject* pysig2 = meth->GetSignature(false); @@ -1144,6 +1164,67 @@ PyObject* CPyCppyy::CPPOverload::FindOverload(const std::string& signature, int return (PyObject*)newmeth; } +PyObject* CPyCppyy::CPPOverload::FindOverload(PyObject *args_tuple, int want_const) +{ + Py_ssize_t n = PyTuple_Size(args_tuple); + + CPPOverload::Methods_t& methods = fMethodInfo->fMethods; + + // This value is set based on the maximum penalty in Cppyy::CompareMethodArgType + Py_ssize_t min_score = INT_MAX; + bool found = false; + size_t best_method = 0, method_index = 0; + + for (auto& meth : methods) { + if (0 <= want_const) { + bool isconst = meth->IsConst(); + if (!((want_const && isconst) || (!want_const && !isconst))) + continue; + } + + int score = meth->GetArgMatchScore(args_tuple); + + if (score < min_score) { + found = true; + min_score = score; + best_method = method_index; + } + + method_index++; + } + + if (!found) { + std::string sigargs("("); + + for (int i = 0; i < n; i++) { + PyObject *pItem = PyTuple_GetItem(args_tuple, i); + if(!CPyCppyy_PyText_Check(pItem)) { + PyErr_Format(PyExc_LookupError, "argument types should be in string format"); + return (PyObject*) nullptr; + } + std::string arg_type(CPyCppyy_PyText_AsString(pItem)); + sigargs += arg_type + ", "; + } + sigargs += ")"; + + PyErr_Format(PyExc_LookupError, "signature with arguments \"%s\" not found", sigargs.c_str()); + return (PyObject*) nullptr; + } + + CPPOverload* newmeth = mp_new(nullptr, nullptr, nullptr); + CPPOverload::Methods_t vec; + vec.push_back(methods[best_method]->Clone()); + newmeth->Set(fMethodInfo->fName, vec); + + if (fSelf) { + Py_INCREF(fSelf); + newmeth->fSelf = fSelf; + } + newmeth->fMethodInfo->fFlags = fMethodInfo->fFlags; + + return (PyObject*) newmeth; +} + //---------------------------------------------------------------------------- CPyCppyy::CPPOverload::MethodInfo_t::~MethodInfo_t() { diff --git a/src/CPPOverload.h b/src/CPPOverload.h index a6e708a..b392bf4 100644 --- a/src/CPPOverload.h +++ b/src/CPPOverload.h @@ -67,6 +67,7 @@ class CPPOverload { // find a method based on the provided signature PyObject* FindOverload(const std::string& signature, int want_const = -1); + PyObject* FindOverload(PyObject *args_tuple, int want_const = -1); public: // public, as the python C-API works with C structs PyObject_HEAD From 1d01ca6ef3ed91bafb699c39f005ec163ca65b97 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 12:26:00 +0100 Subject: [PATCH 11/28] Sync CPPScope --- src/CPPScope.cxx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/CPPScope.cxx b/src/CPPScope.cxx index 26bbe86..b72b3f6 100644 --- a/src/CPPScope.cxx +++ b/src/CPPScope.cxx @@ -701,6 +701,12 @@ PyTypeObject CPPScope_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; } // namespace CPyCppyy From a05406602d85616abf9987701acf4bd740aba8e5 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 12:32:13 +0100 Subject: [PATCH 12/28] Sync Cppyy.h and CPyCppyyModule.cxx --- src/CPyCppyyModule.cxx | 72 ++++++++++++++++++++++++++++++++++++------ src/Cppyy.h | 4 +++ 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/CPyCppyyModule.cxx b/src/CPyCppyyModule.cxx index 7f40105..f4d4992 100644 --- a/src/CPyCppyyModule.cxx +++ b/src/CPyCppyyModule.cxx @@ -158,6 +158,12 @@ static PyTypeObject PyNullPtr_t_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; @@ -192,6 +198,12 @@ static PyTypeObject PyDefault_t_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; namespace { @@ -206,7 +218,7 @@ PyObject _CPyCppyy_DefaultStruct = { 1, &PyDefault_t_Type }; -// TOOD: refactor with Converters.cxx +// TODO: refactor with Converters.cxx struct CPyCppyy_tagCDataObject { // non-public (but stable) PyObject_HEAD char* b_ptr; @@ -409,8 +421,9 @@ static PyObject* SetCppLazyLookup(PyObject*, PyObject* args) CPYCPPYY_GET_DICT_LOOKUP(dict) = CPyCppyyLookDictString; #else // As of py3.11, there is no longer a lookup function pointer in the dict object -// to replace. Since this feature is not widely advertised, it's simply droped +// to replace. Since this feature is not widely advertised, it's simply dropped PyErr_Warn(PyExc_RuntimeWarning, (char*)"lazy lookup is no longer supported"); + (void)args; // avoid warning about unused parameter #endif Py_RETURN_NONE; @@ -489,7 +502,7 @@ static void* GetCPPInstanceAddress(const char* fname, PyObject* args, PyObject* return &((CPPInstance*)pyobj)->GetObjectRaw(); } else if (CPyCppyy_PyText_Check(pyobj)) { - // special cases for acces to the CPyCppyy API + // special cases for access to the CPyCppyy API std::string req = CPyCppyy_PyText_AsString((PyObject*)pyobj); if (req == "Instance_AsVoidPtr") return (void*)&Instance_AsVoidPtr; @@ -528,8 +541,8 @@ static PyObject* addressof(PyObject* /* dummy */, PyObject* args, PyObject* kwds return nullptr; } - Cppyy::TCppFuncAddr_t addr = methods[0]->GetFunctionAddress(); - return PyLong_FromLongLong((intptr_t)addr); + Cppyy::TCppFuncAddr_t caddr = methods[0]->GetFunctionAddress(); + return PyLong_FromLongLong((intptr_t)caddr); } // C functions (incl. ourselves) @@ -568,7 +581,7 @@ static PyObject* AsCObject(PyObject* /* unused */, PyObject* args, PyObject* kwd } //---------------------------------------------------------------------------- -static PyObject* AsCapsule(PyObject* /* dummy */, PyObject* args, PyObject* kwds) +static PyObject* AsCapsule(PyObject* /* unused */, PyObject* args, PyObject* kwds) { // Return object proxy as an opaque PyCapsule. void* addr = GetCPPInstanceAddress("as_capsule", args, kwds); @@ -582,7 +595,7 @@ static PyObject* AsCapsule(PyObject* /* dummy */, PyObject* args, PyObject* kwds } //---------------------------------------------------------------------------- -static PyObject* AsCTypes(PyObject* /* dummy */, PyObject* args, PyObject* kwds) +static PyObject* AsCTypes(PyObject* /* unused */, PyObject* args, PyObject* kwds) { // Return object proxy as a ctypes c_void_p void* addr = GetCPPInstanceAddress("as_ctypes", args, kwds); @@ -607,6 +620,43 @@ static PyObject* AsCTypes(PyObject* /* dummy */, PyObject* args, PyObject* kwds) return ref; } +//---------------------------------------------------------------------------- +static PyObject* AsMemoryView(PyObject* /* unused */, PyObject* pyobject) +{ +// Return a raw memory view on arrays of PODs. + if (!CPPInstance_Check(pyobject)) { + PyErr_SetString(PyExc_TypeError, "C++ object proxy expected"); + return nullptr; + } + + CPPInstance* pyobj = (CPPInstance*)pyobject; + Cppyy::TCppType_t klass = ((CPPClass*)Py_TYPE(pyobject))->fCppType; + + Py_ssize_t array_len = pyobj->ArrayLength(); + + if (array_len < 0 || !Cppyy::IsAggregate(klass)) { + PyErr_SetString( + PyExc_TypeError, "object is not a proxy to an array of PODs of known size"); + return nullptr; + } + + Py_buffer view; + + view.obj = pyobject; + view.buf = pyobj->GetObject(); + view.itemsize = Cppyy::SizeOf(klass); + view.len = view.itemsize * array_len; + view.readonly = 0; + view.format = NULL; // i.e. "B" assumed + view.ndim = 1; + view.shape = NULL; + view.strides = NULL; + view.suboffsets = NULL; + view.internal = NULL; + + return PyMemoryView_FromBuffer(&view); +} + //---------------------------------------------------------------------------- static PyObject* BindObject(PyObject*, PyObject* args, PyObject* kwds) { @@ -850,9 +900,11 @@ static PyObject* SetMemoryPolicy(PyObject*, PyObject* args) if (!PyArg_ParseTuple(args, const_cast("O!"), &PyInt_Type, &policy)) return nullptr; + long old = (long)CallContext::sMemoryPolicy; + long l = PyInt_AS_LONG(policy); if (CallContext::SetMemoryPolicy((CallContext::ECallFlags)l)) { - Py_RETURN_NONE; + return PyInt_FromLong(old); } PyErr_Format(PyExc_ValueError, "Unknown policy %ld", l); @@ -946,6 +998,8 @@ static PyMethodDef gCPyCppyyMethods[] = { METH_VARARGS | METH_KEYWORDS, (char*)"Retrieve address of proxied object or field in a PyCapsule."}, {(char*) "as_ctypes", (PyCFunction)AsCTypes, METH_VARARGS | METH_KEYWORDS, (char*)"Retrieve address of proxied object or field in a ctypes c_void_p."}, + {(char*) "as_memoryview", (PyCFunction)AsMemoryView, + METH_O, (char*)"Represent an array of objects as raw memory."}, {(char*)"bind_object", (PyCFunction)BindObject, METH_VARARGS | METH_KEYWORDS, (char*) "Create an object of given type, from given address."}, {(char*) "move", (PyCFunction)Move, @@ -1027,7 +1081,7 @@ extern "C" void initlibcppyy() #endif #if PY_VERSION_HEX < 0x030b0000 -// prepare for lazyness (the insert is needed to capture the most generic lookup +// prepare for laziness (the insert is needed to capture the most generic lookup // function, just in case ...) PyObject* dict = PyDict_New(); PyObject* notstring = PyInt_FromLong(5); diff --git a/src/Cppyy.h b/src/Cppyy.h index 698f16d..0ab2f78 100644 --- a/src/Cppyy.h +++ b/src/Cppyy.h @@ -207,6 +207,8 @@ namespace Cppyy { CPPYY_IMPORT TCppIndex_t GetNumBases(TCppType_t type); CPPYY_IMPORT + TCppIndex_t GetNumBasesLongestBranch(TCppType_t type); + CPPYY_IMPORT std::string GetBaseName(TCppType_t type, TCppIndex_t ibase); CPPYY_IMPORT TCppScope_t GetBaseScope(TCppType_t type, TCppIndex_t ibase); @@ -259,6 +261,8 @@ namespace Cppyy { CPPYY_IMPORT std::string GetMethodArgCanonTypeAsString(TCppMethod_t, TCppIndex_t iarg); CPPYY_IMPORT + TCppIndex_t CompareMethodArgType(TCppMethod_t, TCppIndex_t iarg, const std::string &req_type); + CPPYY_IMPORT std::string GetMethodArgDefault(TCppMethod_t, TCppIndex_t iarg); CPPYY_IMPORT std::string GetMethodSignature(TCppMethod_t, bool show_formal_args, TCppIndex_t max_args = (TCppIndex_t)-1); From 26e9bd93da5734bdf5f63d310308874f344b0e1b Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 12:34:41 +0100 Subject: [PATCH 13/28] Sync CustomPyTypes --- src/CustomPyTypes.cxx | 55 +++++++++++++++++++++++++++++++++++-------- src/CustomPyTypes.h | 1 + 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/CustomPyTypes.cxx b/src/CustomPyTypes.cxx index 3e6a427..7b2b4a1 100644 --- a/src/CustomPyTypes.cxx +++ b/src/CustomPyTypes.cxx @@ -6,12 +6,20 @@ #include "ProxyWrappers.h" #include "PyStrings.h" +// As of Python 3.12, we can't use the PyMethod_GET_FUNCTION and +// PyMethod_GET_SELF macros anymore, as the contain asserts that check if the +// Python type is actually PyMethod_Type. If the Python type is +// CustomInstanceMethod_Type, we need our own macros. Technically they do they +// same, because the actual C++ type of the PyObject is PyMethodObject anyway. +#define CustomInstanceMethod_GET_SELF(meth) reinterpret_cast(meth)->im_self +#define CustomInstanceMethod_GET_FUNCTION(meth) reinterpret_cast(meth)->im_func #if PY_VERSION_HEX >= 0x03000000 // TODO: this will break functionality -#define PyMethod_GET_CLASS(meth) Py_None +#define CustomInstanceMethod_GET_CLASS(meth) Py_None +#else +#define CustomInstanceMethod_GET_CLASS(meth) PyMethod_GET_CLASS(meth) #endif - namespace CPyCppyy { #if PY_VERSION_HEX < 0x03000000 @@ -156,6 +164,12 @@ PyTypeObject TypedefPointerToClass_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; //= instancemethod object with a more efficient call function ================ @@ -237,13 +251,13 @@ static PyObject* im_call(PyObject* meth, PyObject* args, PyObject* kw) // into the list of arguments. However, the pythonized methods will then have // to undo that shuffling, which is inefficient. This method is the same as // the one for the instancemethod object, except for the shuffling. - PyObject* self = PyMethod_GET_SELF(meth); + PyObject* self = CustomInstanceMethod_GET_SELF(meth); if (!self) { // unbound methods must be called with an instance of the class (or a // derived class) as first argument Py_ssize_t argc = PyTuple_GET_SIZE(args); - PyObject* pyclass = PyMethod_GET_CLASS(meth); + PyObject* pyclass = CustomInstanceMethod_GET_CLASS(meth); if (1 <= argc && PyObject_IsInstance(PyTuple_GET_ITEM(args, 0), pyclass) == 1) { self = PyTuple_GET_ITEM(args, 0); @@ -262,7 +276,7 @@ static PyObject* im_call(PyObject* meth, PyObject* args, PyObject* kw) } else Py_INCREF(args); - PyCFunctionObject* func = (PyCFunctionObject*)PyMethod_GET_FUNCTION(meth); + PyCFunctionObject* func = (PyCFunctionObject*)CustomInstanceMethod_GET_FUNCTION(meth); // the function is globally shared, so set and reset its "self" (ok, b/c of GIL) Py_INCREF(self); @@ -279,10 +293,10 @@ static PyObject* im_descr_get(PyObject* meth, PyObject* obj, PyObject* pyclass) { // from instancemethod: don't rebind an already bound method, or an unbound method // of a class that's not a base class of pyclass - if (PyMethod_GET_SELF(meth) + if (CustomInstanceMethod_GET_SELF(meth) #if PY_VERSION_HEX < 0x03000000 - || (PyMethod_GET_CLASS(meth) && - !PyObject_IsSubclass(pyclass, PyMethod_GET_CLASS(meth))) + || (CustomInstanceMethod_GET_CLASS(meth) && + !PyObject_IsSubclass(pyclass, CustomInstanceMethod_GET_CLASS(meth))) #endif ) { Py_INCREF(meth); @@ -292,7 +306,7 @@ static PyObject* im_descr_get(PyObject* meth, PyObject* obj, PyObject* pyclass) if (obj == Py_None) obj = nullptr; - return CustomInstanceMethod_New(PyMethod_GET_FUNCTION(meth), obj, pyclass); + return CustomInstanceMethod_New(CustomInstanceMethod_GET_FUNCTION(meth), obj, pyclass); } //= CPyCppyy custom instance method type ===================================== @@ -321,6 +335,12 @@ PyTypeObject CustomInstanceMethod_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; @@ -372,6 +392,12 @@ PyTypeObject IndexIter_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; @@ -394,7 +420,10 @@ static PyObject* vectoriter_iternext(vectoriterobject* vi) { // that objects in vectors are simple and thus do not need to maintain object identity // (or at least not during the loop anyway). This gains 2x in performance. Cppyy::TCppObject_t cppobj = (Cppyy::TCppObject_t)((ptrdiff_t)vi->vi_data + vi->vi_stride * vi->ii_pos); - result = CPyCppyy::BindCppObjectNoCast(cppobj, vi->vi_klass, CPyCppyy::CPPInstance::kNoMemReg); + if (vi->vi_flags & vectoriterobject::kIsPolymorphic) + result = CPyCppyy::BindCppObject(*(void**)cppobj, vi->vi_klass, CPyCppyy::CPPInstance::kNoMemReg); + else + result = CPyCppyy::BindCppObjectNoCast(cppobj, vi->vi_klass, CPyCppyy::CPPInstance::kNoMemReg); if ((vi->vi_flags & vectoriterobject::kNeedLifeLine) && result) PyObject_SetAttr(result, PyStrings::gLifeLine, vi->ii_container); } else { @@ -431,6 +460,12 @@ PyTypeObject VectorIter_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; } // namespace CPyCppyy diff --git a/src/CustomPyTypes.h b/src/CustomPyTypes.h index a67e882..caa0fca 100644 --- a/src/CustomPyTypes.h +++ b/src/CustomPyTypes.h @@ -102,6 +102,7 @@ struct vectoriterobject : public indexiterobject { enum EFlags { kDefault = 0x0000, kNeedLifeLine = 0x0001, + kIsPolymorphic = 0x0002, }; }; From 9ea198b3bde545da4e042d46b850048bdc446058 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 12:37:48 +0100 Subject: [PATCH 14/28] Sync DeclareConverters and DeclareExecutors --- src/DeclareConverters.h | 23 ++++++++++++----------- src/DeclareExecutors.h | 2 ++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/DeclareConverters.h b/src/DeclareConverters.h index e049b35..18400a4 100644 --- a/src/DeclareConverters.h +++ b/src/DeclareConverters.h @@ -216,6 +216,8 @@ CPPYY_DECLARE_ARRAY_CONVERTER(UChar); #if __cplusplus > 201402L CPPYY_DECLARE_ARRAY_CONVERTER(Byte); #endif +CPPYY_DECLARE_ARRAY_CONVERTER(Int8); +CPPYY_DECLARE_ARRAY_CONVERTER(UInt8); CPPYY_DECLARE_ARRAY_CONVERTER(Short); CPPYY_DECLARE_ARRAY_CONVERTER(UShort); CPPYY_DECLARE_ARRAY_CONVERTER(Int); @@ -232,6 +234,9 @@ CPPYY_DECLARE_ARRAY_CONVERTER(ComplexD); class CStringArrayConverter : public SCharArrayConverter { public: + CStringArrayConverter(cdims_t dims, bool fixed) : SCharArrayConverter(dims) { + fIsFixed = fixed; // overrides SCharArrayConverter decision + } using SCharArrayConverter::SCharArrayConverter; virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); virtual PyObject* FromMemory(void* address); @@ -369,6 +374,7 @@ CPPYY_DECLARE_BASIC_CONVERTER(PyObject); class name##Converter : public InstanceConverter { \ public: \ name##Converter(bool keepControl = true); \ + \ public: \ virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); \ virtual PyObject* FromMemory(void* address); \ @@ -380,14 +386,10 @@ protected: \ } CPPYY_DECLARE_STRING_CONVERTER(STLString, std::string); +CPPYY_DECLARE_STRING_CONVERTER(STLWString, std::wstring); #if __cplusplus > 201402L -CPPYY_DECLARE_STRING_CONVERTER(STLStringViewBase, std::string_view); -class STLStringViewConverter : public STLStringViewBaseConverter { -public: - virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); -}; +CPPYY_DECLARE_STRING_CONVERTER(STLStringView, std::string_view); #endif -CPPYY_DECLARE_STRING_CONVERTER(STLWString, std::wstring); class STLStringMoveConverter : public STLStringConverter { public: @@ -467,9 +469,7 @@ class SmartPtrConverter : public Converter { // initializer lists class InitializerListConverter : public InstanceConverter { public: - InitializerListConverter(Cppyy::TCppType_t klass, - Converter* cnv, Cppyy::TCppType_t valuetype, size_t sz) : InstanceConverter(klass), - fBuffer(nullptr), fConverter(cnv), fValueType(valuetype), fValueSize(sz) {} + InitializerListConverter(Cppyy::TCppType_t klass, std::string const& value_type); InitializerListConverter(const InitializerListConverter&) = delete; InitializerListConverter& operator=(const InitializerListConverter&) = delete; virtual ~InitializerListConverter(); @@ -483,8 +483,9 @@ class InitializerListConverter : public InstanceConverter { void Clear(); protected: - void* fBuffer; - Converter* fConverter; + void* fBuffer = nullptr; + std::vector fConverters; + std::string fValueTypeName; Cppyy::TCppType_t fValueType; size_t fValueSize; }; diff --git a/src/DeclareExecutors.h b/src/DeclareExecutors.h index ad05ca1..3e0d957 100644 --- a/src/DeclareExecutors.h +++ b/src/DeclareExecutors.h @@ -115,6 +115,8 @@ class InstanceExecutor : public Executor { class IteratorExecutor : public InstanceExecutor { public: IteratorExecutor(Cppyy::TCppScope_t klass); + virtual PyObject* Execute( + Cppyy::TCppMethod_t, Cppyy::TCppObject_t, CallContext*); }; CPPYY_DECL_EXEC(Constructor); From 182779a9d1d01f69d3f802b84e1653b12fe7e524 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 12:43:04 +0100 Subject: [PATCH 15/28] Sync Dispatcher --- src/DispatchPtr.cxx | 9 ++++++-- src/Dispatcher.cxx | 56 ++++++++++++++++++++++++++++++++------------- 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/src/DispatchPtr.cxx b/src/DispatchPtr.cxx index 21dcdcd..43b73fb 100644 --- a/src/DispatchPtr.cxx +++ b/src/DispatchPtr.cxx @@ -11,7 +11,11 @@ PyObject* CPyCppyy::DispatchPtr::Get() const { if (fPyHardRef) return fPyHardRef; - if (fPyWeakRef) return PyWeakref_GetObject(fPyWeakRef); + if (fPyWeakRef) { + PyObject* disp = PyWeakref_GetObject(fPyWeakRef); + if (disp != Py_None) // dispatcher object disappeared? + return disp; + } return nullptr; } @@ -45,7 +49,7 @@ CPyCppyy::DispatchPtr::~DispatchPtr() { // continued access if (fPyWeakRef) { PyObject* pyobj = PyWeakref_GetObject(fPyWeakRef); - if (pyobj && ((CPPScope*)Py_TYPE(pyobj))->fFlags & CPPScope::kIsPython) + if (pyobj && pyobj != Py_None && ((CPPScope*)Py_TYPE(pyobj))->fFlags & CPPScope::kIsPython) ((CPPInstance*)pyobj)->GetObjectRaw() = nullptr; Py_DECREF(fPyWeakRef); } else if (fPyHardRef) { @@ -83,6 +87,7 @@ void CPyCppyy::DispatchPtr::CppOwns() // C++ maintains the hardref, keeping the PyObject alive w/o outstanding ref if (fPyWeakRef) { fPyHardRef = PyWeakref_GetObject(fPyWeakRef); + if (fPyHardRef == Py_None) fPyHardRef = nullptr; Py_XINCREF(fPyHardRef); Py_DECREF(fPyWeakRef); fPyWeakRef = nullptr; } diff --git a/src/Dispatcher.cxx b/src/Dispatcher.cxx index f053be4..ad627e3 100644 --- a/src/Dispatcher.cxx +++ b/src/Dispatcher.cxx @@ -34,7 +34,26 @@ static inline void InjectMethod(Cppyy::TCppMethod_t method, const std::string& m if (Cppyy::IsConstMethod(method)) code << "const "; code << "{\n"; -// start function body +// on destruction, the Python object may go first, in which case provide a diagnostic +// warning (raising a PyException may not be possible as this could happen during +// program shutdown); note that this means that the actual result will be the default +// and the caller may need to act on that, but that's still an improvement over a +// possible crash + code << " PyObject* iself = (PyObject*)_internal_self;\n" + " if (!iself || iself == Py_None) {\n" + " PyErr_Warn(PyExc_RuntimeWarning, (char*)\"Call attempted on deleted python-side proxy\");\n" + " return"; + if (retType != "void") { + if (retType.back() != '*') + code << " " << CPyCppyy::TypeManip::remove_const(retType) << "{}"; + else + code << " nullptr"; + } + code << ";\n" + " }\n" + " Py_INCREF(iself);\n"; + +// start actual function body Utility::ConstructCallbackPreamble(retType, argtypes, code); // perform actual method call @@ -43,13 +62,13 @@ static inline void InjectMethod(Cppyy::TCppMethod_t method, const std::string& m #else code << " PyObject* mtPyName = PyUnicode_FromString(\"" << mtCppName << "\");\n" #endif - " PyObject* pyresult = PyObject_CallMethodObjArgs((PyObject*)_internal_self, mtPyName"; + " PyObject* pyresult = PyObject_CallMethodObjArgs(iself, mtPyName"; for (Cppyy::TCppIndex_t i = 0; i < nArgs; ++i) code << ", pyargs[" << i << "]"; - code << ", NULL);\n Py_DECREF(mtPyName);\n"; + code << ", NULL);\n Py_DECREF(mtPyName);\n Py_DECREF(iself);\n"; // close - Utility::ConstructCallbackReturn(retType, nArgs, code); + Utility::ConstructCallbackReturn(retType, (int)nArgs, code); } //---------------------------------------------------------------------------- @@ -127,11 +146,11 @@ namespace { using namespace Cppyy; static inline -std::vector FindBaseMethod(TCppScope_t tbase, const std::string mtCppName) +std::vector FindBaseMethod(TCppScope_t tbase, const std::string mtCppName) { // Recursively walk the inheritance tree to find the overloads of the named method - std::vector result; - result = GetMethodsFromName(tbase, mtCppName); + std::vector result; + result = GetMethodIndicesFromName(tbase, mtCppName); if (result.empty()) { for (TCppIndex_t ibase = 0; ibase < GetNumBases(tbase); ++ibase) { TCppScope_t b = GetScope(GetBaseName(tbase, ibase)); @@ -215,17 +234,22 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct, // add a virtual destructor for good measure, which is allowed to be "overridden" by // the conventional __destruct__ method (note that __del__ is always called, too, if -// provided, but only when the Python object goes away) +// provided, but only when the Python object goes away; furthermore, if the Python +// object goes before the C++ one, only __del__ is called) if (PyMapping_HasKeyString(dct, (char*)"__destruct__")) { code << " virtual ~" << derivedName << "() {\n" + " PyObject* iself = (PyObject*)_internal_self;\n" + " if (!iself || iself == Py_None)\n" + " return;\n" // safe, as destructor always returns void + " Py_INCREF(iself);\n" " PyObject* mtPyName = PyUnicode_FromString(\"__destruct__\");\n" - " PyObject* pyresult = PyObject_CallMethodObjArgs((PyObject*)_internal_self, mtPyName, NULL);\n" - " Py_DECREF(mtPyName);\n"; + " PyObject* pyresult = PyObject_CallMethodObjArgs(iself, mtPyName, NULL);\n" + " Py_DECREF(mtPyName);\n Py_DECREF(iself);\n"; // this being a destructor, print on exception rather than propagate using the // magic C++ exception ... - code << " if (!pyresult) PyErr_Print();\n" - " else { Py_DECREF(pyresult); }\n" + code << " if (!pyresult) PyErr_Print();\n" + " else { Py_DECREF(pyresult); }\n" " }\n"; } else code << " virtual ~" << derivedName << "() {}\n"; @@ -342,10 +366,10 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct, // TODO: should probably invert this looping; but that makes handling overloads clunky PyObject* key = PyList_GET_ITEM(keys, i); std::string mtCppName = CPyCppyy_PyText_AsString(key); - const auto& methods = FindBaseMethod(tbase, mtCppName); - for (auto method : methods) - InjectMethod(method, mtCppName, code); - if (!methods.empty()) { + const auto& v = FindBaseMethod(tbase, mtCppName); + for (auto idx : v) + InjectMethod(Cppyy::GetMethod(tbase, idx), mtCppName, code); + if (!v.empty()) { if (PyDict_DelItem(clbs, key) != 0) PyErr_Clear(); } } From 6cd5c7625865c099f700238a7309887ff6d601a4 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 12:49:28 +0100 Subject: [PATCH 16/28] Sync Executor --- src/Executors.cxx | 82 ++++++++++++++++++++++++++++++++--------------- src/Executors.h | 1 + 2 files changed, 58 insertions(+), 25 deletions(-) diff --git a/src/Executors.cxx b/src/Executors.cxx index 531eae4..d5b7227 100644 --- a/src/Executors.cxx +++ b/src/Executors.cxx @@ -509,34 +509,34 @@ PyObject* CPyCppyy::VoidArrayExecutor::Execute( } //---------------------------------------------------------------------------- -#define CPPYY_IMPL_ARRAY_EXEC(name, type) \ +#define CPPYY_IMPL_ARRAY_EXEC(name, type, suffix) \ PyObject* CPyCppyy::name##ArrayExecutor::Execute( \ Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt) \ { \ - return CreateLowLevelView((type*)GILCallR(method, self, ctxt), fShape); \ + return CreateLowLevelView##suffix((type*)GILCallR(method, self, ctxt), fShape); \ } -CPPYY_IMPL_ARRAY_EXEC(Bool, bool) -CPPYY_IMPL_ARRAY_EXEC(UChar, unsigned char) +CPPYY_IMPL_ARRAY_EXEC(Bool, bool, ) +CPPYY_IMPL_ARRAY_EXEC(UChar, unsigned char, ) #if __cplusplus > 201402L -CPPYY_IMPL_ARRAY_EXEC(Byte, std::byte) +CPPYY_IMPL_ARRAY_EXEC(Byte, std::byte, ) #endif -CPPYY_IMPL_ARRAY_EXEC(Int8, int8_t) -CPPYY_IMPL_ARRAY_EXEC(UInt8, uint8_t) -CPPYY_IMPL_ARRAY_EXEC(Short, short) -CPPYY_IMPL_ARRAY_EXEC(UShort, unsigned short) -CPPYY_IMPL_ARRAY_EXEC(Int, int) -CPPYY_IMPL_ARRAY_EXEC(UInt, unsigned int) -CPPYY_IMPL_ARRAY_EXEC(Long, long) -CPPYY_IMPL_ARRAY_EXEC(ULong, unsigned long) -CPPYY_IMPL_ARRAY_EXEC(LLong, long long) -CPPYY_IMPL_ARRAY_EXEC(ULLong, unsigned long long) -CPPYY_IMPL_ARRAY_EXEC(Float, float) -CPPYY_IMPL_ARRAY_EXEC(Double, double) -CPPYY_IMPL_ARRAY_EXEC(ComplexF, std::complex) -CPPYY_IMPL_ARRAY_EXEC(ComplexD, std::complex) -CPPYY_IMPL_ARRAY_EXEC(ComplexI, std::complex) -CPPYY_IMPL_ARRAY_EXEC(ComplexL, std::complex) +CPPYY_IMPL_ARRAY_EXEC(Int8, int8_t, _i8) +CPPYY_IMPL_ARRAY_EXEC(UInt8, uint8_t, _i8) +CPPYY_IMPL_ARRAY_EXEC(Short, short, ) +CPPYY_IMPL_ARRAY_EXEC(UShort, unsigned short, ) +CPPYY_IMPL_ARRAY_EXEC(Int, int, ) +CPPYY_IMPL_ARRAY_EXEC(UInt, unsigned int, ) +CPPYY_IMPL_ARRAY_EXEC(Long, long, ) +CPPYY_IMPL_ARRAY_EXEC(ULong, unsigned long, ) +CPPYY_IMPL_ARRAY_EXEC(LLong, long long, ) +CPPYY_IMPL_ARRAY_EXEC(ULLong, unsigned long long, ) +CPPYY_IMPL_ARRAY_EXEC(Float, float, ) +CPPYY_IMPL_ARRAY_EXEC(Double, double, ) +CPPYY_IMPL_ARRAY_EXEC(ComplexF, std::complex, ) +CPPYY_IMPL_ARRAY_EXEC(ComplexD, std::complex, ) +CPPYY_IMPL_ARRAY_EXEC(ComplexI, std::complex, ) +CPPYY_IMPL_ARRAY_EXEC(ComplexL, std::complex, ) //- special cases ------------------------------------------------------------ @@ -591,7 +591,7 @@ PyObject* CPyCppyy::STLWStringExecutor::Execute( } PyObject* pyresult = PyUnicode_FromWideChar(result->c_str(), result->size()); - ::operator delete(result); // calls Cppyy::CallO which calls ::operator new + delete result; // Cppyy::CallO allocates and constructs a string, so it must be properly destroyed return pyresult; } @@ -642,6 +642,21 @@ CPyCppyy::IteratorExecutor::IteratorExecutor(Cppyy::TCppScope_t klass) : fFlags |= CPPInstance::kNoWrapConv; // adds to flags from base class } +//---------------------------------------------------------------------------- +PyObject* CPyCppyy::IteratorExecutor::Execute( + Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt) +{ + PyObject* iter = this->InstanceExecutor::Execute(method, self, ctxt); + if (iter && ctxt->fPyContext) { + // set life line to tie iterator life time to the container (which may + // be a temporary) + std::ostringstream attr_name; + attr_name << "__" << (intptr_t)iter; + if (PyObject_SetAttrString(ctxt->fPyContext, (char*)attr_name.str().c_str(), iter)) + PyErr_Clear(); + } + return iter; +} //---------------------------------------------------------------------------- PyObject* CPyCppyy::InstanceRefExecutor::Execute( @@ -850,7 +865,7 @@ CPyCppyy::Executor* CPyCppyy::CreateExecutor(const std::string& fullType, cdims_ // C++ classes and special cases Executor* result = 0; if (Cppyy::TCppType_t klass = Cppyy::GetFullScope(realType)) { - if (resolvedType.find("iterator") != std::string::npos || gIteratorTypes.find(fullType) != gIteratorTypes.end()) { + if (Utility::IsSTLIterator(realType) || gIteratorTypes.find(fullType) != gIteratorTypes.end()) { if (cpd == "") return new IteratorExecutor(klass); } @@ -1012,6 +1027,23 @@ bool CPyCppyy::RegisterExecutor(const std::string& name, ef_t fac) return true; } +//---------------------------------------------------------------------------- +CPYCPPYY_EXPORT +bool CPyCppyy::RegisterExecutorAlias(const std::string& name, const std::string& target) +{ +// register a custom executor that is a reference to an existing converter + auto f = gExecFactories.find(name); + if (f != gExecFactories.end()) + return false; + + auto t = gExecFactories.find(target); + if (t == gExecFactories.end()) + return false; + + gExecFactories[name] = t->second; + return true; +} + //---------------------------------------------------------------------------- CPYCPPYY_EXPORT bool CPyCppyy::UnregisterExecutor(const std::string& name) @@ -1164,9 +1196,9 @@ struct InitExecFactories_t { gf[CCOMPLEX_D " ptr"] = gf["std::complex ptr"]; // factories for special cases - gf["const char*"] = (ef_t)+[]() { static CStringExecutor e{}; return &e; }; + gf["const char*"] = (ef_t)+[](cdims_t) { static CStringExecutor e{}; return &e; }; gf["char*"] = gf["const char*"]; - gf["const char*&"] = (ef_t)+[]() { static CStringRefExecutor e{}; return &e; }; + gf["const char*&"] = (ef_t)+[](cdims_t) { static CStringRefExecutor e{}; return &e; }; gf["char*&"] = gf["const char*&"]; gf["const signed char*"] = gf["const char*"]; gf["signed char*"] = gf["char*"]; diff --git a/src/Executors.h b/src/Executors.h index 0d67c41..38a4b8f 100644 --- a/src/Executors.h +++ b/src/Executors.h @@ -37,6 +37,7 @@ CPYCPPYY_EXPORT Executor* CreateExecutor(Cppyy::TCppType_t type, cdims_t = 0); CPYCPPYY_EXPORT void DestroyExecutor(Executor* p); typedef Executor* (*ef_t) (cdims_t); CPYCPPYY_EXPORT bool RegisterExecutor(const std::string& name, ef_t fac); +CPYCPPYY_EXPORT bool RegisterExecutorAlias(const std::string& name, const std::string& target); CPYCPPYY_EXPORT bool UnregisterExecutor(const std::string& name); // helper for the actual call From 0b9efa7f8ab8a772a16c039e4758c100a9b38dac Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 13:05:35 +0100 Subject: [PATCH 17/28] Sync LowLevelViews --- src/LowLevelViews.cxx | 155 ++++++++++++++++++++++++++++++++++++------ src/LowLevelViews.h | 15 ++-- 2 files changed, 144 insertions(+), 26 deletions(-) diff --git a/src/LowLevelViews.cxx b/src/LowLevelViews.cxx index f43704d..f0ca82a 100644 --- a/src/LowLevelViews.cxx +++ b/src/LowLevelViews.cxx @@ -8,6 +8,7 @@ // Standard #include #include +#include #include @@ -246,14 +247,14 @@ static char* lookup_dimension(Py_buffer& view, char* ptr, int dim, Py_ssize_t in index += nitems; else { PyErr_Format(PyExc_IndexError, - "negative index not supporte on dimension %d with unknown size", dim + 1); + "negative index not supported on dimension %d with unknown size", dim + 1); return nullptr; } } if (view.strides[dim] == CPyCppyy::UNKNOWN_SIZE) { PyErr_Format(PyExc_IndexError, - "multi index not supporte on dimension %d with unknown stride", dim + 1); + "multi index not supported on dimension %d with unknown stride", dim + 1); return nullptr; } @@ -466,11 +467,38 @@ static PyObject* ll_subscript(CPyCppyy::LowLevelView* self, PyObject* key) return ll_item(self, index); } else if (PySlice_Check(key)) { - // TODO: handle slicing. This should be simpler than the memoryview - // case as there is no Python object holding the buffer. - PyErr_SetString(PyExc_NotImplementedError, - "multi-dimensional slicing is not implemented"); - return nullptr; + if (view.ndim == 1) { + Py_ssize_t start, stop, step, slicelen; + if (PySlice_Unpack(key, &start, &stop, &step) < 0) + return nullptr; + + slicelen = PySlice_AdjustIndices(view.shape[0], &start, &stop, step); + if (slicelen <= 0) + slicelen = view.shape[0]; + + char* buf = (char*)self->get_buf(); + char* slice_buf = new char[slicelen*view.itemsize]; + size_t isize = view.itemsize; + for (size_t i=0, cur=0; i < (size_t)slicelen; cur += step, ++i) { + for (size_t j=0; j < isize; ++j) + slice_buf[i*isize+j] = buf[(start+cur)*isize + j]; + } + + CPyCppyy::LowLevelView* ll = self->fCreator(slice_buf, {1, slicelen}); + if (!ll) + delete [] slice_buf; + else + (intptr_t&)ll->fBufInfo.internal |= CPyCppyy::LowLevelView::kIsOwner; + + return (PyObject*)ll; + + } else { + // TODO: handle slicing. This should be simpler than the memoryview + // case as there is no Python object holding the buffer. + PyErr_SetString(PyExc_NotImplementedError, + "multi-dimensional slicing is not implemented"); + return nullptr; + } } else if (is_multiindex(key)) { return ll_item_multi(self, key); @@ -823,10 +851,31 @@ static PyObject* ll_array(CPyCppyy::LowLevelView* self, PyObject* args, PyObject return view; } + +//--------------------------------------------------------------------------- +static PyObject* ll_as_string(CPyCppyy::LowLevelView* self) +{ +// Interpret memory as a null-terminated char string. + Py_buffer& view = self->fBufInfo; + + if (strcmp(view.format, "b") != 0 || view.ndim != 1) { + PyErr_Format(PyExc_TypeError, + "as_string only supported for 1-dim char strings (format: %s, dim: %d)", + view.format, (int)view.ndim); + return nullptr; + } + + char* buf = (char*)self->get_buf(); + size_t sz = strnlen(buf, (size_t)view.shape[0]); + return CPyCppyy_PyText_FromStringAndSize(buf, sz); +} + //--------------------------------------------------------------------------- static PyMethodDef ll_methods[] = { {(char*)"reshape", (PyCFunction)ll_reshape, METH_O, (char*)"change the shape (not layout) of the low level view"}, + {(char*)"as_string", (PyCFunction)ll_as_string, METH_NOARGS, + (char*)"interpret memory as a null-terminated char string and return Python str"}, {(char*)"__array__", (PyCFunction)ll_array, METH_VARARGS | METH_KEYWORDS, (char*)"return a numpy array from the low level view"}, {(char*)nullptr, nullptr, 0, nullptr} @@ -904,6 +953,12 @@ PyTypeObject LowLevelView_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; } // namespace CPyCppyy @@ -923,6 +978,8 @@ template<> struct typecode_traits { template<> struct typecode_traits { static constexpr const char* format = "B"; static constexpr const char* name = "UCharAsInt"; }; #endif +template<> struct typecode_traits { + static constexpr const char* format = "b"; static constexpr const char* name = "char*"; }; template<> struct typecode_traits { static constexpr const char* format = "b"; static constexpr const char* name = "const char*"; }; template<> struct typecode_traits { @@ -959,9 +1016,23 @@ template<> struct typecode_traits> { } // unnamed namespace +//--------------------------------------------------------------------------- +bool CPyCppyy::LowLevelView::resize(size_t sz) +{ + Py_buffer& bi = this->fBufInfo; + if (bi.ndim == 1 && bi.shape) { + bi.len = sz * bi.itemsize; + bi.shape[0] = sz; + return true; + } + + return false; +} + //--------------------------------------------------------------------------- template -static inline PyObject* CreateLowLevelViewT(T* address, CPyCppyy::cdims_t shape) +static inline CPyCppyy::LowLevelView* CreateLowLevelViewT( + T* address, CPyCppyy::cdims_t shape, const char* format = nullptr, const char* name = nullptr, Py_ssize_t itemsize = -1) { using namespace CPyCppyy; Py_ssize_t nx = (shape.ndim() != UNKNOWN_SIZE) ? shape[0] : INT_MAX/sizeof(T); @@ -975,7 +1046,7 @@ static inline PyObject* CreateLowLevelViewT(T* address, CPyCppyy::cdims_t shape) view.buf = address; view.obj = nullptr; view.readonly = 0; - view.format = (char*)typecode_traits::format; + view.format = (char*)(format ? format : typecode_traits::format); view.ndim = int(shape.ndim() != UNKNOWN_SIZE ? shape.ndim() : 1); view.shape = (Py_ssize_t*)PyMem_Malloc(view.ndim * sizeof(Py_ssize_t)); view.shape[0] = nx; // view.len / view.itemsize @@ -989,11 +1060,11 @@ static inline PyObject* CreateLowLevelViewT(T* address, CPyCppyy::cdims_t shape) if (isfix) (intptr_t&)view.internal |= CPyCppyy::LowLevelView::kIsFixed; } - llp->fElemCnv = CreateConverter(typecode_traits::name); + llp->fElemCnv = CreateConverter(name ? name : typecode_traits::name); if (view.ndim == 1) { // simple 1-dim array of the declared type view.len = nx * sizeof(T); - view.itemsize = sizeof(T); + view.itemsize = (itemsize > 0 ? (size_t)itemsize : sizeof(T)); llp->fConverter = llp->fElemCnv; } else { // multi-dim array; sub-views are projected by using more LLViews @@ -1003,34 +1074,42 @@ static inline PyObject* CreateLowLevelViewT(T* address, CPyCppyy::cdims_t shape) view.shape[idim] = shape[idim]; // peel off one dimension and create a new LLView converter - std::string tname{typecode_traits::name}; - tname.append("*"); // make sure to ask for another array + std::string tname{name ? name : typecode_traits::name}; + tname.append("[]"); // make sure to ask for another array // TODO: although this will work, it means that "naive" loops are expensive llp->fConverter = CreateConverter(tname, shape.sub()); } set_strides(view, sizeof(T), isfix); - return (PyObject*)llp; + return llp; } //--------------------------------------------------------------------------- template -static inline PyObject* CreateLowLevelViewT(T** address, CPyCppyy::cdims_t shape) +static inline CPyCppyy::LowLevelView* CreateLowLevelViewT( + T** address, CPyCppyy::cdims_t shape, const char* format = nullptr, const char* name = nullptr) { using namespace CPyCppyy; - LowLevelView* llp = (LowLevelView*)CreateLowLevelViewT((T*)address, shape); + LowLevelView* llp = (LowLevelView*)CreateLowLevelViewT((T*)address, shape, format, name); llp->set_buf((void**)address); - return (PyObject*)llp; + return llp; } //--------------------------------------------------------------------------- +#define CPPYY_RET_W_CREATOR(type, fname) \ + PyObject* (*c)(type, cdims_t) = &fname; \ + ll->fCreator = (LowLevelView::Creator_t)c; \ + return (PyObject*)ll + #define CPPYY_IMPL_VIEW_CREATOR(type) \ PyObject* CPyCppyy::CreateLowLevelView(type* address, cdims_t shape) { \ - return CreateLowLevelViewT(address, shape); \ + LowLevelView* ll = CreateLowLevelViewT(address, shape); \ + CPPYY_RET_W_CREATOR(type*, CreateLowLevelView); \ } \ PyObject* CPyCppyy::CreateLowLevelView(type** address, cdims_t shape) { \ - return CreateLowLevelViewT(address, shape); \ + LowLevelView* ll = CreateLowLevelViewT(address, shape); \ + CPPYY_RET_W_CREATOR(type**, CreateLowLevelView); \ } CPPYY_IMPL_VIEW_CREATOR(bool); @@ -1055,10 +1134,42 @@ CPPYY_IMPL_VIEW_CREATOR(std::complex); CPPYY_IMPL_VIEW_CREATOR(std::complex); CPPYY_IMPL_VIEW_CREATOR(std::complex); +PyObject* CPyCppyy::CreateLowLevelView(char* address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape); + CPPYY_RET_W_CREATOR(char*, CreateLowLevelView); +} + PyObject* CPyCppyy::CreateLowLevelView(char** address, cdims_t shape) { - return CreateLowLevelViewT((char*)address, shape); + LowLevelView* ll = CreateLowLevelViewT(address, shape); + CPPYY_RET_W_CREATOR(char**, CreateLowLevelView); +} + +PyObject* CPyCppyy::CreateLowLevelViewString(char** address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape, nullptr, nullptr, sizeof(char)); + CPPYY_RET_W_CREATOR(char**, CreateLowLevelViewString); +} + +PyObject* CPyCppyy::CreateLowLevelViewString(const char** address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape, nullptr, nullptr, sizeof(char)); + CPPYY_RET_W_CREATOR(const char**, CreateLowLevelViewString); +} + +PyObject* CPyCppyy::CreateLowLevelView_i8(int8_t* address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape, "b", "int8_t"); + CPPYY_RET_W_CREATOR(int8_t*, CreateLowLevelView_i8); +} + +PyObject* CPyCppyy::CreateLowLevelView_i8(int8_t** address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape, "b", "int8_t"); + CPPYY_RET_W_CREATOR(int8_t**, CreateLowLevelView_i8); +} + +PyObject* CPyCppyy::CreateLowLevelView_i8(uint8_t* address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape, "B", "uint8_t"); + CPPYY_RET_W_CREATOR(uint8_t*, CreateLowLevelView_i8); } -PyObject* CPyCppyy::CreateLowLevelView(const char** address, cdims_t shape) { - return CreateLowLevelViewT(address, shape); +PyObject* CPyCppyy::CreateLowLevelView_i8(uint8_t** address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape, "B", "uint8_t"); + CPPYY_RET_W_CREATOR(uint8_t**, CreateLowLevelView_i8); } diff --git a/src/LowLevelViews.h b/src/LowLevelViews.h index 4695d22..002e60b 100644 --- a/src/LowLevelViews.h +++ b/src/LowLevelViews.h @@ -31,9 +31,14 @@ class LowLevelView { Converter* fConverter; Converter* fElemCnv; + typedef LowLevelView* (*Creator_t)(void*, cdims_t); + Creator_t fCreator; // for slicing, which requires copying + public: void* get_buf() { return fBuf ? *fBuf : fBufInfo.buf; } void set_buf(void** buf) { fBuf = buf; fBufInfo.buf = get_buf(); } + + bool resize(size_t sz); }; #define CPPYY_DECL_VIEW_CREATOR(type) \ @@ -47,8 +52,10 @@ CPPYY_DECL_VIEW_CREATOR(unsigned char); #if __cplusplus > 201402L CPPYY_DECL_VIEW_CREATOR(std::byte); #endif -CPPYY_DECL_VIEW_CREATOR(int8_t); -CPPYY_DECL_VIEW_CREATOR(uint8_t); +PyObject* CreateLowLevelView_i8(int8_t*, cdims_t shape); +PyObject* CreateLowLevelView_i8(int8_t**, cdims_t shape); +PyObject* CreateLowLevelView_i8(uint8_t*, cdims_t shape); +PyObject* CreateLowLevelView_i8(uint8_t**, cdims_t shape); CPPYY_DECL_VIEW_CREATOR(short); CPPYY_DECL_VIEW_CREATOR(unsigned short); CPPYY_DECL_VIEW_CREATOR(int); @@ -65,8 +72,8 @@ CPPYY_DECL_VIEW_CREATOR(std::complex); CPPYY_DECL_VIEW_CREATOR(std::complex); CPPYY_DECL_VIEW_CREATOR(std::complex); -PyObject* CreateLowLevelView(char**, cdims_t shape = 0); -PyObject* CreateLowLevelView(const char**, cdims_t shape = 0); +PyObject* CreateLowLevelViewString(char**, cdims_t shape); +PyObject* CreateLowLevelViewString(const char**, cdims_t shape); inline PyObject* CreatePointerView(void* ptr, cdims_t shape = 0) { return CreateLowLevelView((uintptr_t*)ptr, shape); From 9b3f917c44527050676030a3f778f370778b69a8 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 13:27:43 +0100 Subject: [PATCH 18/28] Sync MemoryRegulator --- src/MemoryRegulator.cxx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/MemoryRegulator.cxx b/src/MemoryRegulator.cxx index 49275b7..37cc394 100644 --- a/src/MemoryRegulator.cxx +++ b/src/MemoryRegulator.cxx @@ -99,7 +99,7 @@ CPyCppyy::MemoryRegulator::MemoryRegulator() bool CPyCppyy::MemoryRegulator::RecursiveRemove( Cppyy::TCppObject_t cppobj, Cppyy::TCppType_t klass) { -// if registerd by the framework, called whenever a cppobj gets destroyed +// if registered by the framework, called whenever a cppobj gets destroyed if (!cppobj) return false; @@ -135,6 +135,7 @@ bool CPyCppyy::MemoryRegulator::RecursiveRemove( CPyCppyy_NoneType.tp_traverse = Py_TYPE(pyobj)->tp_traverse; CPyCppyy_NoneType.tp_clear = Py_TYPE(pyobj)->tp_clear; CPyCppyy_NoneType.tp_free = Py_TYPE(pyobj)->tp_free; + CPyCppyy_NoneType.tp_flags |= Py_TYPE(pyobj)->tp_flags; } else if (CPyCppyy_NoneType.tp_traverse != Py_TYPE(pyobj)->tp_traverse) { // TODO: SystemError? std::cerr << "in CPyCppyy::MemoryRegulater, unexpected object of type: " From 495548ebc5be20e5c68dd78d2e08dc5952b5d345 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 13:33:56 +0100 Subject: [PATCH 19/28] Sync ProxyWrappers --- src/ProxyWrappers.cxx | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ProxyWrappers.cxx b/src/ProxyWrappers.cxx index b26cad1..4e2150c 100644 --- a/src/ProxyWrappers.cxx +++ b/src/ProxyWrappers.cxx @@ -163,7 +163,6 @@ static int BuildScopeProxyDict(Cppyy::TCppScope_t scope, PyObject* pyclass, cons bool hasConstructor = false; Cppyy::TCppMethod_t potGetItem = (Cppyy::TCppMethod_t)0; - // load all public methods and data members typedef std::vector Callables_t; typedef std::map CallableCache_t; @@ -176,7 +175,6 @@ static int BuildScopeProxyDict(Cppyy::TCppScope_t scope, PyObject* pyclass, cons // functions in namespaces are properly found through lazy lookup, so do not // create them until needed (the same is not true for data members) std::vector methods; - if (isComplete) Cppyy::GetClassMethods(scope, methods); for (auto &method : methods) { From 28b613d5f442a094d0188e930595eb9932a7a388 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 13:36:24 +0100 Subject: [PATCH 20/28] Sync PyCallable, PyException and PyStrings --- src/PyCallable.h | 4 ++++ src/PyException.cxx | 14 ++++++++------ src/PyStrings.cxx | 7 +++++++ src/PyStrings.h | 1 + 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/PyCallable.h b/src/PyCallable.h index d76e0f5..62ce3aa 100644 --- a/src/PyCallable.h +++ b/src/PyCallable.h @@ -1,6 +1,8 @@ #ifndef CPYCPPYY_PYCALLABLE_H #define CPYCPPYY_PYCALLABLE_H +#include + // Bindings #include "CPyCppyy/Reflex.h" #include "CallContext.h" @@ -38,6 +40,8 @@ class PyCallable { virtual PyCallable* Clone() = 0; + virtual int GetArgMatchScore(PyObject* /* args_tuple */) { return INT_MAX; } + public: virtual PyObject* Call(CPPInstance*& self, CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr) = 0; diff --git a/src/PyException.cxx b/src/PyException.cxx index 6e8d158..2929683 100644 --- a/src/PyException.cxx +++ b/src/PyException.cxx @@ -93,16 +93,18 @@ CPyCppyy::PyException::PyException() if (fMsg.empty()) fMsg = "python exception"; -// only keeping the filename, not the full path - if (!locFile.empty()) + if (!locFile.empty()) { + + // only keeping the filename, not the full path locFile = locFile.substr(locFile.find_last_of("/\\") + 1); - fMsg += " (at " + locFile + ":" + std::to_string(locLine); + fMsg += " (at " + locFile + ":" + std::to_string(locLine); - if (locName != "") - fMsg += " in " + locName; + if (locName != "") + fMsg += " in " + locName; - fMsg += ")"; + fMsg += ")"; + } #ifdef WITH_THREAD PyGILState_Release(state); diff --git a/src/PyStrings.cxx b/src/PyStrings.cxx index 081f23e..e918d44 100644 --- a/src/PyStrings.cxx +++ b/src/PyStrings.cxx @@ -7,6 +7,7 @@ PyObject* CPyCppyy::PyStrings::gAssign = nullptr; PyObject* CPyCppyy::PyStrings::gBases = nullptr; PyObject* CPyCppyy::PyStrings::gBase = nullptr; +PyObject* CPyCppyy::PyStrings::gCppBool = nullptr; PyObject* CPyCppyy::PyStrings::gCppName = nullptr; PyObject* CPyCppyy::PyStrings::gAnnotations = nullptr; PyObject* CPyCppyy::PyStrings::gCastCpp = nullptr; @@ -86,6 +87,11 @@ bool CPyCppyy::CreatePyStrings() { CPPYY_INITIALIZE_STRING(gAssign, __assign__); CPPYY_INITIALIZE_STRING(gBases, __bases__); CPPYY_INITIALIZE_STRING(gBase, __base__); +#if PY_VERSION_HEX < 0x03000000 + CPPYY_INITIALIZE_STRING(gCppBool, __cpp_nonzero__); +#else + CPPYY_INITIALIZE_STRING(gCppBool, __cpp_bool__); +#endif CPPYY_INITIALIZE_STRING(gCppName, __cpp_name__); CPPYY_INITIALIZE_STRING(gAnnotations, __annotations__); CPPYY_INITIALIZE_STRING(gCastCpp, __cast_cpp__); @@ -163,6 +169,7 @@ PyObject* CPyCppyy::DestroyPyStrings() { // Remove all cached python strings. Py_DECREF(PyStrings::gBases); PyStrings::gBases = nullptr; Py_DECREF(PyStrings::gBase); PyStrings::gBase = nullptr; + Py_DECREF(PyStrings::gCppBool); PyStrings::gCppBool = nullptr; Py_DECREF(PyStrings::gCppName); PyStrings::gCppName = nullptr; Py_DECREF(PyStrings::gAnnotations); PyStrings::gAnnotations = nullptr; Py_DECREF(PyStrings::gCType); PyStrings::gCType = nullptr; diff --git a/src/PyStrings.h b/src/PyStrings.h index b0e7330..7012b89 100644 --- a/src/PyStrings.h +++ b/src/PyStrings.h @@ -10,6 +10,7 @@ namespace PyStrings { extern PyObject* gAssign; extern PyObject* gBases; extern PyObject* gBase; + extern PyObject* gCppBool; extern PyObject* gCppName; extern PyObject* gAnnotations; extern PyObject* gCastCpp; From 2f0ee278486bd400985e350d5f87edcffe55bfe3 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 14:23:40 +0100 Subject: [PATCH 21/28] Sync Pythonize --- src/Pythonize.cxx | 60 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index c4e3280..d6e43eb 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -202,6 +202,19 @@ PyObject* FollowGetAttr(PyObject* self, PyObject* name) return result; } +//- pointer checking bool converter ------------------------------------------- +PyObject* NullCheckBool(PyObject* self) +{ + if (!CPPInstance_Check(self)) { + PyErr_SetString(PyExc_TypeError, "C++ object proxy expected"); + return nullptr; + } + + if (!((CPPInstance*)self)->GetObject()) + Py_RETURN_FALSE; + + return PyObject_CallMethodNoArgs(self, PyStrings::gCppBool); +} //- vector behavior as primitives ---------------------------------------------- #if PY_VERSION_HEX < 0x03040000 @@ -345,7 +358,7 @@ static bool FillVector(PyObject* vecin, PyObject* args, ItemGetter* getter) eb_args = PyTuple_New(1); PyTuple_SET_ITEM(eb_args, 0, item); } else if (PyTuple_CheckExact(item)) { - eb_args = item; + eb_args = item; } else if (PyList_CheckExact(item)) { Py_ssize_t isz = PyList_GET_SIZE(item); eb_args = PyTuple_New(isz); @@ -492,7 +505,8 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) PyObject* VectorData(PyObject* self, PyObject*) { PyObject* pydata = CallPyObjMethod(self, "__real_data"); - if (!LowLevelView_Check(pydata)) return pydata; + if (!LowLevelView_Check(pydata) && !CPPInstance_Check(pydata)) + return pydata; PyObject* pylen = PyObject_CallMethodNoArgs(self, PyStrings::gSize); if (!pylen) { @@ -503,12 +517,12 @@ PyObject* VectorData(PyObject* self, PyObject*) long clen = PyInt_AsLong(pylen); Py_DECREF(pylen); -// TODO: should be a LowLevelView helper - Py_buffer& bi = ((LowLevelView*)pydata)->fBufInfo; - bi.len = clen * bi.itemsize; - if (bi.ndim == 1 && bi.shape) - bi.shape[0] = clen; + if (CPPInstance_Check(pydata)) { + ((CPPInstance*)pydata)->CastToArray(clen); + return pydata; + } + ((LowLevelView*)pydata)->resize((size_t)clen); return pydata; } @@ -549,7 +563,18 @@ static PyObject* vector_iter(PyObject* v) { if (PyLong_Check(pyvalue_type)) { Cppyy::TCppType_t value_type = PyLong_AsVoidPtr(pyvalue_type); + value_type = Cppyy::ResolveType(value_type); vi->vi_klass = Cppyy::GetScopeFromType(value_type); + if (!vi->vi_klass) { + // look for a special case of pointer to a class type (which is a builtin, but it + // is more useful to treat it polymorphically by allowing auto-downcasts) + const std::string& clean_type = TypeManip::clean_type(value_type, false, false); + Cppyy::TCppScope_t c = Cppyy::GetScope(clean_type); + if (c && TypeManip::compound(value_type) == "*") { + vi->vi_klass = c; + vi->vi_flags = vectoriterobject::kIsPolymorphic; + } + } if (vi->vi_klass) { vi->vi_converter = nullptr; if (!vi->vi_flags) { @@ -810,7 +835,7 @@ static PyObject* MapFromPairs(PyObject* self, PyObject* pairs) PyObject* MapInit(PyObject* self, PyObject* args, PyObject* /* kwds */) { // Specialized map constructor to allow construction from mapping containers and -// from tuples of pairs ("intializer_list style"). +// from tuples of pairs ("initializer_list style"). // PyMapping_Check is not very discriminatory, as it basically only checks for the // existence of __getitem__, hence the most common cases of tuple and list are @@ -1012,7 +1037,7 @@ PyObject* CheckedGetItem(PyObject* self, PyObject* obj) { // Implement a generic python __getitem__ for STL-like classes that are missing the // reflection info for their iterators. This is then used for iteration by means of -// consecutive indeces, it such index is of integer type. +// consecutive indices, it such index is of integer type. Py_ssize_t size = PySequence_Size(self); Py_ssize_t idx = PyInt_AsSsize_t(obj); if ((size == (Py_ssize_t)-1 || idx == (Py_ssize_t)-1) && PyErr_Occurred()) { @@ -1236,7 +1261,7 @@ PyObject* STLStringContains(CPPInstance* self, PyObject* pyobj) Py_RETURN_FALSE; } -PyObject* STLStringReplace(CPPInstance* self, PyObject* args, PyObject* kwds) +PyObject* STLStringReplace(CPPInstance* self, PyObject* args, PyObject* /*kwds*/) { std::string* obj = GetSTLString(self); if (!obj) @@ -1267,7 +1292,7 @@ PyObject* STLStringReplace(CPPInstance* self, PyObject* args, PyObject* kwds) } #define CPYCPPYY_STRING_FINDMETHOD(name, cppname, pyname) \ -PyObject* STLString##name(CPPInstance* self, PyObject* args, PyObject* kwds) \ +PyObject* STLString##name(CPPInstance* self, PyObject* args, PyObject* /*kwds*/) \ { \ std::string* obj = GetSTLString(self); \ if (!obj) \ @@ -1387,7 +1412,6 @@ PyObject* StringViewInit(PyObject* self, PyObject* args, PyObject* /* kwds */) } - //- STL iterator behavior ---------------------------------------------------- PyObject* STLIterNext(PyObject* self) { @@ -1590,6 +1614,16 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, Cppyy::TCppScope_t scope) else if (HasAttrDirect(pyclass, PyStrings::gFollow) && !Cppyy::IsSmartPtr(klass->fCppType)) Utility::AddToClass(pyclass, "__getattr__", (PyCFunction)FollowGetAttr, METH_O); +// for pre-check of nullptr for boolean types + if (HasAttrDirect(pyclass, PyStrings::gCppBool)) { +#if PY_VERSION_HEX >= 0x03000000 + const char* pybool_name = "__bool__"; +#else + const char* pybool_name = "__nonzero__"; +#endif + Utility::AddToClass(pyclass, pybool_name, (PyCFunction)NullCheckBool, METH_NOARGS); + } + // for STL containers, and user classes modeled after them if (HasAttrDirect(pyclass, PyStrings::gSize)) Utility::AddToClass(pyclass, "__len__", "size"); @@ -1631,7 +1665,7 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, Cppyy::TCppScope_t scope) // Python will iterate over __getitem__ using integers, but C++ operator[] will never raise // a StopIteration. A checked getitem (raising IndexError if beyond size()) works in some // cases but would mess up if operator[] is meant to implement an associative container. So, - // this has to be implemented as an interator protocol. + // this has to be implemented as an iterator protocol. ((PyTypeObject*)pyclass)->tp_iter = (getiterfunc)index_iter; Utility::AddToClass(pyclass, "__iter__", (PyCFunction)index_iter, METH_NOARGS); } From 85ba00a4ecf729ed068cbc625f30b704818d5f70 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 14:27:05 +0100 Subject: [PATCH 22/28] Sync SignalTryCatch --- src/SignalTryCatch.h | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/SignalTryCatch.h b/src/SignalTryCatch.h index a630820..14742b8 100644 --- a/src/SignalTryCatch.h +++ b/src/SignalTryCatch.h @@ -18,7 +18,13 @@ #define NEED_SIGJMP 1 #endif +// By default, the ExceptionContext_t class is expected in the namespace +// CppyyLegacy, If it is expected in no namespace, one can explicitly define +// NO_CPPYY_LEGACY_NAMESPACE at build time (e.g. if one wants to use ROOT). + +#ifndef NO_CPPYY_LEGACY_NAMESPACE namespace CppyyLegacy { +#endif struct ExceptionContext_t { #ifdef NEED_SIGJMP sigjmp_buf fBuf; @@ -26,11 +32,17 @@ struct ExceptionContext_t { jmp_buf fBuf; #endif }; +#ifndef NO_CPPYY_LEGACY_NAMESPACE } +using CppyyExceptionContext_t = CppyyLegacy::ExceptionContext_t; +#else +using CppyyExceptionContext_t = ExceptionContext_t; +#endif + // FIXME: This is a dummy, replace with cling equivalent of gException -static CppyyLegacy::ExceptionContext_t DummyException; -static CppyyLegacy::ExceptionContext_t *gException = &DummyException; +static CppyyExceptionContext_t DummyException; +static CppyyExceptionContext_t *gException = &DummyException; #ifdef NEED_SIGJMP # define CLING_EXCEPTION_SETJMP(buf) sigsetjmp(buf,1) @@ -40,14 +52,14 @@ static CppyyLegacy::ExceptionContext_t *gException = &DummyException; #define CLING_EXCEPTION_RETRY \ { \ - static CppyyLegacy::ExceptionContext_t R__curr, *R__old = gException; \ + static CppyyExceptionContext_t R__curr, *R__old = gException; \ int R__code; \ gException = &R__curr; \ R__code = CLING_EXCEPTION_SETJMP(gException->fBuf); if (R__code) { }; { #define CLING_EXCEPTION_TRY \ { \ - static CppyyLegacy::ExceptionContext_t R__curr, *R__old = gException; \ + static CppyyExceptionContext_t R__curr, *R__old = gException; \ int R__code; \ gException = &R__curr; \ if ((R__code = CLING_EXCEPTION_SETJMP(gException->fBuf)) == 0) { @@ -63,6 +75,6 @@ static CppyyLegacy::ExceptionContext_t *gException = &DummyException; gException = R__old; \ } -CPYCPPYY_IMPORT CppyyLegacy::ExceptionContext_t *gException; +CPYCPPYY_IMPORT CppyyExceptionContext_t *gException; #endif From 5fde53150ad937af9c1eb758d3973c9f884454f1 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 14:30:19 +0100 Subject: [PATCH 23/28] Sync TemplateProxy --- src/TemplateProxy.cxx | 87 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 17 deletions(-) diff --git a/src/TemplateProxy.cxx b/src/TemplateProxy.cxx index 03870ec..0285725 100644 --- a/src/TemplateProxy.cxx +++ b/src/TemplateProxy.cxx @@ -493,7 +493,7 @@ static inline PyObject* SelectAndForward(TemplateProxy* pytmpl, CPPOverload* pym static inline PyObject* CallMethodImp(TemplateProxy* pytmpl, PyObject*& pymeth, CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, bool impOK, uint64_t sighash) { -// Actual call of a given overload: takes care of handling of "self" and +// Actual call of a given overload: takes care of handlign of "self" and // dereferences the overloaded method after use. PyObject* result; @@ -654,13 +654,13 @@ static PyObject* tpp_call(TemplateProxy* pytmpl, PyObject* args, PyObject* kwds) result = SelectAndForward(pytmpl, pytmpl->fTI->fNonTemplated, args, nargsf, kwds, true /* implicitOkay */, false /* use_targs */, sighash, errors); if (result) - return result; + TPPCALL_RETURN; // case 3: select known template overload result = SelectAndForward(pytmpl, pytmpl->fTI->fTemplated, args, nargsf, kwds, false /* implicitOkay */, true /* use_targs */, sighash, errors); if (result) - return result; + TPPCALL_RETURN; // case 4: auto-instantiation from types of arguments for (auto pref : {Utility::kReference, Utility::kPointer, Utility::kValue}) { @@ -681,7 +681,7 @@ static PyObject* tpp_call(TemplateProxy* pytmpl, PyObject* args, PyObject* kwds) result = SelectAndForward(pytmpl, pytmpl->fTI->fLowPriority, args, nargsf, kwds, false /* implicitOkay */, false /* use_targs */, sighash, errors); if (result) - return result; + TPPCALL_RETURN; // error reporting is fraud, given the numerous steps taken, but more details seems better if (!errors.empty()) { @@ -790,20 +790,67 @@ static PyObject* tpp_overload(TemplateProxy* pytmpl, PyObject* args) { // Select and call a specific C++ overload, based on its signature. const char* sigarg = nullptr; + PyObject* sigarg_tuple = nullptr; int want_const = -1; - if (!PyArg_ParseTuple(args, const_cast("s|i:__overload__"), &sigarg, &want_const)) + + Cppyy::TCppScope_t scope = (Cppyy::TCppScope_t) 0; + Cppyy::TCppMethod_t cppmeth = (Cppyy::TCppMethod_t) 0; + std::string proto; + + if (PyArg_ParseTuple(args, const_cast("s|i:__overload__"), &sigarg, &want_const)) { + want_const = PyTuple_GET_SIZE(args) == 1 ? -1 : want_const; + + // check existing overloads in order + PyObject* ol = pytmpl->fTI->fNonTemplated->FindOverload(sigarg, want_const); + if (ol) return ol; + PyErr_Clear(); + ol = pytmpl->fTI->fTemplated->FindOverload(sigarg, want_const); + if (ol) return ol; + PyErr_Clear(); + ol = pytmpl->fTI->fLowPriority->FindOverload(sigarg, want_const); + if (ol) return ol; + + proto = Utility::ConstructTemplateArgs(nullptr, args); + + scope = ((CPPClass*)pytmpl->fTI->fPyClass)->fCppType; + cppmeth = Cppyy::GetMethodTemplate( + scope, pytmpl->fTI->fCppName, proto.substr(1, proto.size()-2)); + } else if (PyArg_ParseTuple(args, const_cast("O|i:__overload__"), &sigarg_tuple, &want_const)) { + PyErr_Clear(); + want_const = PyTuple_GET_SIZE(args) == 1 ? -1 : want_const; + + // check existing overloads in order + PyObject* ol = pytmpl->fTI->fNonTemplated->FindOverload(sigarg_tuple, want_const); + if (ol) return ol; + PyErr_Clear(); + ol = pytmpl->fTI->fTemplated->FindOverload(sigarg_tuple, want_const); + if (ol) return ol; + PyErr_Clear(); + ol = pytmpl->fTI->fLowPriority->FindOverload(sigarg_tuple, want_const); + if (ol) return ol; + + proto.reserve(128); + proto.push_back('<'); + Py_ssize_t n = PyTuple_Size(sigarg_tuple); + for (int i = 0; i < n; i++) { + PyObject *pItem = PyTuple_GetItem(sigarg_tuple, i); + if(!CPyCppyy_PyText_Check(pItem)) { + PyErr_Format(PyExc_LookupError, "argument types should be in string format"); + return (PyObject*) nullptr; + } + proto.append(CPyCppyy_PyText_AsString(pItem)); + if (i < n - 1) + proto.push_back(','); + } + proto.push_back('>'); + + scope = ((CPPClass*)pytmpl->fTI->fPyClass)->fCppType; + cppmeth = Cppyy::GetMethodTemplate( + scope, pytmpl->fTI->fCppName, proto.substr(1, proto.size()-2)); + } else { + PyErr_Format(PyExc_TypeError, "Unexpected arguments to __overload__"); return nullptr; - want_const = PyTuple_GET_SIZE(args) == 1 ? -1 : want_const; - -// check existing overloads in order - PyObject* ol = pytmpl->fTI->fNonTemplated->FindOverload(sigarg, want_const); - if (ol) return ol; - PyErr_Clear(); - ol = pytmpl->fTI->fTemplated->FindOverload(sigarg, want_const); - if (ol) return ol; - PyErr_Clear(); - ol = pytmpl->fTI->fLowPriority->FindOverload(sigarg, want_const); - if (ol) return ol; + } // else attempt instantiation PyObject* pytype = 0, *pyvalue = 0, *pytrace = 0; @@ -877,7 +924,7 @@ PyTypeObject TemplateProxy_Type = { #if PY_VERSION_HEX >= 0x03080000 | Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_METHOD_DESCRIPTOR #endif - , // tp_flags + , // tp_flags (char*)"cppyy template proxy (internal)", // tp_doc (traverseproc)tpp_traverse, // tp_traverse (inquiry)tpp_clear, // tp_clear @@ -912,6 +959,12 @@ PyTypeObject TemplateProxy_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; } // namespace CPyCppyy From a8ea5aba68cd21a44ee4595d751f663289e4fa10 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 14:31:55 +0100 Subject: [PATCH 24/28] Sync TupleOfInstances --- src/TupleOfInstances.cxx | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/TupleOfInstances.cxx b/src/TupleOfInstances.cxx index 8ef8cc5..ccb66eb 100644 --- a/src/TupleOfInstances.cxx +++ b/src/TupleOfInstances.cxx @@ -92,7 +92,7 @@ PyTypeObject InstanceArrayIter_Type = { sizeof(ia_iterobject), // tp_basicsize 0, (destructor)PyObject_GC_Del, // tp_dealloc - 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, &ia_as_mapping, // tp_as_mapping 0, 0, 0, 0, 0, 0, Py_TPFLAGS_DEFAULT | @@ -114,6 +114,12 @@ PyTypeObject InstanceArrayIter_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; @@ -242,6 +248,12 @@ PyTypeObject TupleOfInstances_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif }; } // namespace CPyCppyy From 56ea6e240256b772aee189e5b0e0cbd568eab3a9 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 14:34:15 +0100 Subject: [PATCH 25/28] Sync TypeManip --- src/TypeManip.cxx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/TypeManip.cxx b/src/TypeManip.cxx index 11ab77e..e0832a8 100644 --- a/src/TypeManip.cxx +++ b/src/TypeManip.cxx @@ -167,7 +167,7 @@ std::string CPyCppyy::TypeManip::compound(const std::string& name) // for easy identification of fixed size arrays if (!cpd.empty() && cpd.back() == ']') { if (cpd.front() == '[') - return "[]"; // fixed array any; dimensions handled seperately + return "[]"; // fixed array any; dimensions handled separately std::ostringstream scpd; scpd << cpd.substr(0, cpd.find('[')) << "[]"; @@ -196,7 +196,7 @@ void CPyCppyy::TypeManip::cppscope_to_legalname(std::string& cppscope) { // Change characters illegal in a variable name into '_' to form a legal name. for (char& c : cppscope) { - for (char needle : {':', '>', '<', ' ', ',', '&', '='}) + for (char needle : {':', '>', '<', ' ', ',', '&', '=', '*'}) if (c == needle) c = '_'; } } @@ -266,7 +266,7 @@ std::vector CPyCppyy::TypeManip::extract_arg_types(const std::strin //---------------------------------------------------------------------------- Py_ssize_t CPyCppyy::TypeManip::array_size(const std::string& name) { -// Extrac the array size from a given type name (assumes 1D arrays) +// Extract the array size from a given type name (assumes 1D arrays) std::string cleanName = remove_const(name); if (cleanName[cleanName.size()-1] == ']') { std::string::size_type idx = cleanName.rfind('['); From 89d5e867f1ac63ef0e2bf0f56efc3d2645b89240 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 14:37:41 +0100 Subject: [PATCH 26/28] Sync Utility --- src/Utility.cxx | 81 ++++++++++++++++++++++++++++++++++++++----------- src/Utility.h | 1 + 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/Utility.cxx b/src/Utility.cxx index 2077ecc..809ca15 100644 --- a/src/Utility.cxx +++ b/src/Utility.cxx @@ -115,9 +115,9 @@ namespace { gC2POperatorMapping["="] = "__assign__"; // id. #if PY_VERSION_HEX < 0x03000000 - gC2POperatorMapping["bool"] = "__nonzero__"; + gC2POperatorMapping["bool"] = "__cpp_nonzero__"; #else - gC2POperatorMapping["bool"] = "__bool__"; + gC2POperatorMapping["bool"] = "__cpp_bool__"; #endif } } initOperatorMapping_; @@ -386,6 +386,22 @@ CPyCppyy::PyCallable* CPyCppyy::Utility::FindBinaryOperator( } //---------------------------------------------------------------------------- +static inline std::string AnnotationAsText(PyObject* pyobj) +{ + if (!CPyCppyy_PyText_Check(pyobj)) { + PyObject* pystr = PyObject_GetAttr(pyobj, CPyCppyy::PyStrings::gName); + if (!pystr) { + PyErr_Clear(); + pystr = PyObject_Str(pyobj); + } + + std::string str = CPyCppyy_PyText_AsString(pystr); + Py_DECREF(pystr); + return str; + } + return CPyCppyy_PyText_AsString(pyobj); +} + static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, CPyCppyy::Utility::ArgPreference pref, int* pcnt = nullptr) { @@ -510,14 +526,14 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, if (ret) { // dict is ordered, with the last value being the return type std::ostringstream tpn; - tpn << (CPPScope_Check(ret) ? ClassName(ret) : CPyCppyy_PyText_AsString(ret)) + tpn << (CPPScope_Check(ret) ? ClassName(ret) : AnnotationAsText(ret)) << " (*)("; PyObject* values = PyDict_Values(annot); for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values)-1); ++i) { if (i) tpn << ", "; PyObject* item = PyList_GET_ITEM(values, i); - tpn << (CPPScope_Check(item) ? ClassName(item) : CPyCppyy_PyText_AsString(item)); + tpn << (CPPScope_Check(item) ? ClassName(item) : AnnotationAsText(item)); } Py_DECREF(values); @@ -587,7 +603,7 @@ std::string CPyCppyy::Utility::ConstructTemplateArgs( PyObject* tn = justOne ? tpArgs : PyTuple_GET_ITEM(tpArgs, i); if (CPyCppyy_PyText_Check(tn)) { tmpl_name.append(CPyCppyy_PyText_AsString(tn)); - // some commmon numeric types (separated out for performance: checking for + // some common numeric types (separated out for performance: checking for // __cpp_name__ and/or __name__ is rather expensive) } else { if (!AddTypeName(tmpl_name, tn, (args ? PyTuple_GET_ITEM(args, i) : nullptr), pref, pcnt)) { @@ -1039,14 +1055,18 @@ Py_ssize_t CPyCppyy::Utility::GetBuffer(PyObject* pyobject, char tc, int size, v // new-style buffer interface if (PyObject_CheckBuffer(pyobject)) { + if (PySequence_Check(pyobject) && !PySequence_Size(pyobject)) + return 0; // PyObject_GetBuffer() crashes on some platforms for some zero-sized seqeunces + Py_buffer bufinfo; memset(&bufinfo, 0, sizeof(Py_buffer)); if (PyObject_GetBuffer(pyobject, &bufinfo, PyBUF_FORMAT) == 0) { if (tc == '*' || strchr(bufinfo.format, tc) -#ifdef _WIN32 - // ctypes is inconsistent in format on Windows; either way these types are the same size - || (tc == 'I' && strchr(bufinfo.format, 'L')) || (tc == 'i' && strchr(bufinfo.format, 'l')) -#endif + // if `long int` and `int` are the same size (on Windows and 32bit Linux, + // for example), `ctypes` isn't too picky about the type format, so make + // sure both integer types pass the type check + || (sizeof(long int) == sizeof(int) && ((tc == 'I' && strchr(bufinfo.format, 'L')) || + (tc == 'i' && strchr(bufinfo.format, 'l')))) // complex float is 'Zf' in bufinfo.format, but 'z' in single char || (tc == 'z' && strstr(bufinfo.format, "Zf")) // allow 'signed char' ('b') from array to pass through '?' (bool as from struct) @@ -1227,6 +1247,29 @@ std::string CPyCppyy::Utility::ClassName(PyObject* pyobj) return clname; } +//---------------------------------------------------------------------------- +static std::set sIteratorTypes; +bool CPyCppyy::Utility::IsSTLIterator(const std::string& classname) +{ +// attempt to recognize STL iterators (TODO: probably belongs in the backend) + if (sIteratorTypes.empty()) { + std::string tt = "::"; + for (auto c : {"std::vector", "std::list", "std::deque"}) { + for (auto i : {"iterator", "const_iterator"}) { + const std::string& itname = Cppyy::ResolveName(c+tt+i); + auto pos = itname.find('<'); + if (pos != std::string::npos) + sIteratorTypes.insert(itname.substr(0, pos)); + } + } + } + + auto pos = classname.find('<'); + if (pos != std::string::npos) + return sIteratorTypes.find(classname.substr(0, pos)) != sIteratorTypes.end(); + return false; +} + //---------------------------------------------------------------------------- CPyCppyy::Utility::PyOperators::~PyOperators() @@ -1284,23 +1327,17 @@ void CPyCppyy::Utility::SetDetailedException(std::vector& errors, PyO return; } -// if a _single_ exception was from C++, assume it has priority - PyObject* exc_type = nullptr; +// if a _single_ exception was thrown from C++, assume it has priority (see below) PyError_t* unique_from_cpp = nullptr; for (auto& e : errors) { if (e.fIsCpp) { if (!unique_from_cpp) unique_from_cpp = &e; else { - // two C++ exceptions, resort to default + // two C++ exceptions, resort to default behavior unique_from_cpp = nullptr; - exc_type = defexc; break; } - } else if (!unique_from_cpp) { - // try to consolidate Python exceptions, otherwise select default - if (!exc_type) exc_type = e.fType; - else if (exc_type != e.fType) exc_type = defexc; } } @@ -1314,6 +1351,16 @@ void CPyCppyy::Utility::SetDetailedException(std::vector& errors, PyO Py_INCREF(unique_from_cpp->fType); Py_INCREF(unique_from_cpp->fValue); Py_XINCREF(unique_from_cpp->fTrace); PyErr_Restore(unique_from_cpp->fType, unique_from_cpp->fValue, unique_from_cpp->fTrace); } else { + // try to consolidate Python exceptions, otherwise select default + PyObject* exc_type = nullptr; + for (auto& e : errors) { + if (!exc_type) exc_type = e.fType; + else if (exc_type != e.fType) { + exc_type = defexc; + break; + } + } + // add the details to the topmsg PyObject* separator = CPyCppyy_PyText_FromString("\n "); for (auto& e : errors) { diff --git a/src/Utility.h b/src/Utility.h index 380ff8d..4abd2d3 100644 --- a/src/Utility.h +++ b/src/Utility.h @@ -80,6 +80,7 @@ struct PyOperators { // meta information std::string ClassName(PyObject* pyobj); +bool IsSTLIterator(const std::string& classname); // for threading: save call to PyErr_Occurred() PyObject* PyErr_Occurred_WithGIL(); From 1eb1088e40425eef5e2c3f9e97d778794182aff2 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 28 Oct 2024 14:40:37 +0100 Subject: [PATCH 27/28] Sync CPyCppyy API --- include/CPyCppyy/API.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/include/CPyCppyy/API.h b/include/CPyCppyy/API.h index c0c137f..3a3de47 100644 --- a/include/CPyCppyy/API.h +++ b/include/CPyCppyy/API.h @@ -93,7 +93,7 @@ class Dimensions { // Windows note: NOT exported/imported dim_t* fDims; public: - Dimensions(dim_t ndim = 0, dim_t* dims = nullptr) : fDims(nullptr) {} + Dimensions(dim_t /*ndim*/ = 0, dim_t* /*dims*/ = nullptr) : fDims(nullptr) {} ~Dimensions() { delete [] fDims; } public: @@ -133,6 +133,9 @@ CPYCPPYY_EXTERN void DestroyConverter(Converter* p); typedef Converter* (*ConverterFactory_t)(cdims_t); CPYCPPYY_EXTERN bool RegisterConverter(const std::string& name, ConverterFactory_t); +// register a custom converter that is a reference to an existing converter +CPYCPPYY_EXTERN bool RegisterConverterAlias(const std::string& name, const std::string& target); + // remove a custom converter CPYCPPYY_EXTERN bool UnregisterConverter(const std::string& name); @@ -161,6 +164,9 @@ CPYCPPYY_EXTERN void DestroyConverter(Converter* p); typedef Executor* (*ExecutorFactory_t)(cdims_t); CPYCPPYY_EXTERN bool RegisterExecutor(const std::string& name, ExecutorFactory_t); +// register a custom executor that is a reference to an existing converter +CPYCPPYY_EXTERN bool RegisterExecutorAlias(const std::string& name, const std::string& target); + // remove a custom executor CPYCPPYY_EXTERN bool UnregisterExecutor(const std::string& name); @@ -186,6 +192,9 @@ CPYCPPYY_EXTERN bool Scope_CheckExact(PyObject* pyobject); CPYCPPYY_EXTERN bool Instance_Check(PyObject* pyobject); CPYCPPYY_EXTERN bool Instance_CheckExact(PyObject* pyobject); +// type verifier for sequences +CPYCPPYY_EXTERN bool Sequence_Check(PyObject* pyobject); + // helper to verify expected safety of moving an instance into C++ CPYCPPYY_EXTERN bool Instance_IsLively(PyObject* pyobject); From 8712d9e2cadd0e92cdd62af56c7a058b421f4b59 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Fri, 8 Nov 2024 13:28:41 +0100 Subject: [PATCH 28/28] Update CPyCppyy to use InterOp API This updates converter creation, templateproxy, dispatcher, cppdatamember, cppinstance and cppenum to enable parts of the API that required modifications to work with CppInterOp --- src/CPPDataMember.cxx | 5 ----- src/CPPEnum.cxx | 35 +++++++++++++++++++---------------- src/CPPInstance.cxx | 6 ------ src/Converters.cxx | 3 +-- src/Dispatcher.cxx | 14 +++++++------- src/Executors.cxx | 6 ++---- src/Pythonize.cxx | 4 ++-- src/TemplateProxy.cxx | 5 ----- 8 files changed, 31 insertions(+), 47 deletions(-) diff --git a/src/CPPDataMember.cxx b/src/CPPDataMember.cxx index 8fc6971..0fa7351 100644 --- a/src/CPPDataMember.cxx +++ b/src/CPPDataMember.cxx @@ -48,10 +48,6 @@ static PyObject* dm_get(CPPDataMember* dm, CPPInstance* pyobj, PyObject* /* kls } } -// non-initialized or public data accesses through class (e.g. by help()) - void* address = dm->GetAddress(pyobj); - if (!address || (intptr_t)address == -1 /* Cling error */) - return nullptr; if (dm->fFlags & (kIsEnumPrep | kIsEnumType)) { if (dm->fFlags & kIsEnumPrep) { @@ -92,7 +88,6 @@ static PyObject* dm_get(CPPDataMember* dm, CPPInstance* pyobj, PyObject* /* kls return pyval_from_enum(Cppyy::ResolveEnum(dm->fScope), nullptr, nullptr, dm->fScope); } } - // non-initialized or public data accesses through class (e.g. by help()) void* address = dm->GetAddress(pyobj); if (!address || (intptr_t)address == -1 /* Cling error */) diff --git a/src/CPPEnum.cxx b/src/CPPEnum.cxx index 45a453a..65f4f4b 100644 --- a/src/CPPEnum.cxx +++ b/src/CPPEnum.cxx @@ -19,9 +19,9 @@ static PyObject* pytype_from_enum_type(const std::string& enum_type) } //---------------------------------------------------------------------------- -static PyObject* pyval_from_enum(const std::string& enum_type, PyObject* pytype, - PyObject* btype, Cppyy::TCppEnum_t etype, Cppyy::TCppIndex_t idata) { - long long llval = Cppyy::GetEnumDataValue(etype, idata); +PyObject* CPyCppyy::pyval_from_enum(const std::string& enum_type, PyObject* pytype, + PyObject* btype, Cppyy::TCppScope_t enum_constant) { + long long llval = Cppyy::GetEnumDataValue(enum_constant); if (enum_type == "bool") { PyObject* result = (bool)llval ? Py_True : Py_False; @@ -38,14 +38,15 @@ static PyObject* pyval_from_enum(const std::string& enum_type, PyObject* pytype, else bval = PyLong_FromLongLong(llval); + if (pytype && btype) { PyObject* args = PyTuple_New(1); PyTuple_SET_ITEM(args, 0, bval); - PyObject* result = ((PyTypeObject*)btype)->tp_new((PyTypeObject*)pytype, args, nullptr); + bval = ((PyTypeObject*)btype)->tp_new((PyTypeObject*)pytype, args, nullptr); Py_DECREF(args); - return result; + } + return bval; } - //- enum methods ------------------------------------------------------------- static int enum_setattro(PyObject* /* pyclass */, PyObject* /* pyname */, PyObject* /* pyval */) { @@ -59,6 +60,8 @@ static PyObject* enum_repr(PyObject* self) { using namespace CPyCppyy; + PyObject* kls_scope = PyObject_GetAttr((PyObject*)Py_TYPE(self), PyStrings::gThisModule); + if (!kls_scope) PyErr_Clear(); PyObject* kls_cppname = PyObject_GetAttr((PyObject*)Py_TYPE(self), PyStrings::gCppName); if (!kls_cppname) PyErr_Clear(); PyObject* obj_cppname = PyObject_GetAttr(self, PyStrings::gCppName); @@ -67,7 +70,7 @@ static PyObject* enum_repr(PyObject* self) PyObject* repr = nullptr; if (kls_cppname && obj_cppname && obj_str) { - const std::string resolved = Cppyy::ResolveEnum(CPyCppyy_PyText_AsString(kls_cppname)); + const std::string resolved = Cppyy::ResolveEnum(PyLong_AsVoidPtr(kls_scope)); repr = CPyCppyy_PyText_FromFormat("(%s::%s) : (%s) %s", CPyCppyy_PyText_AsString(kls_cppname), CPyCppyy_PyText_AsString(obj_cppname), resolved.c_str(), CPyCppyy_PyText_AsString(obj_str)); @@ -137,12 +140,12 @@ CPyCppyy::CPPEnum* CPyCppyy::CPPEnum_New(const std::string& name, Cppyy::TCppSco CPPEnum* pyenum = nullptr; - const std::string& ename = scope == Cppyy::gGlobalScope ? name : Cppyy::GetScopedFinalName(scope)+"::"+name; - Cppyy::TCppEnum_t etype = Cppyy::GetEnum(scope, name); + Cppyy::TCppScope_t etype = scope; + const std::string& ename = Cppyy::GetScopedFinalName(scope); if (etype) { // create new enum type with labeled values in place, with a meta-class // to make sure the enum values are read-only - const std::string& resolved = Cppyy::ResolveEnum(ename); + const std::string& resolved = Cppyy::ResolveEnum(etype); PyObject* pyside_type = pytype_from_enum_type(resolved); PyObject* pymetabases = PyTuple_New(1); PyObject* btype = (PyObject*)Py_TYPE(pyside_type); @@ -169,7 +172,7 @@ CPyCppyy::CPPEnum* CPyCppyy::CPPEnum_New(const std::string& name, Cppyy::TCppSco PyObject* pyresolved = CPyCppyy_PyText_FromString(resolved.c_str()); PyDict_SetItem(dct, PyStrings::gUnderlying, pyresolved); Py_DECREF(pyresolved); - + // add the __module__ to allow pickling std::string modname = TypeManip::extract_namespace(ename); TypeManip::cppscope_to_pyscope(modname); // :: -> . @@ -191,10 +194,10 @@ CPyCppyy::CPPEnum* CPyCppyy::CPPEnum_New(const std::string& name, Cppyy::TCppSco ((PyTypeObject*)pyenum)->tp_str = ((PyTypeObject*)pyside_type)->tp_repr; // collect the enum values - Cppyy::TCppIndex_t ndata = Cppyy::GetNumEnumData(etype); - for (Cppyy::TCppIndex_t idata = 0; idata < ndata; ++idata) { - PyObject* val = pyval_from_enum(resolved, pyenum, pyside_type, etype, idata); - PyObject* pydname = CPyCppyy_PyText_FromString(Cppyy::GetEnumDataName(etype, idata).c_str()); + std::vector econstants = Cppyy::GetEnumConstants(etype); + for (auto *econstant : econstants) { + PyObject* val = pyval_from_enum(resolved, pyenum, pyside_type, econstant); + PyObject* pydname = CPyCppyy_PyText_FromString(Cppyy::GetFinalName(econstant).c_str()); PyObject_SetAttr(pyenum, pydname, val); PyObject_SetAttr(val, PyStrings::gCppName, pydname); Py_DECREF(pydname); @@ -215,4 +218,4 @@ CPyCppyy::CPPEnum* CPyCppyy::CPPEnum_New(const std::string& name, Cppyy::TCppSco } return pyenum; -} +} \ No newline at end of file diff --git a/src/CPPInstance.cxx b/src/CPPInstance.cxx index f990661..9d7830e 100644 --- a/src/CPPInstance.cxx +++ b/src/CPPInstance.cxx @@ -358,10 +358,6 @@ static PyObject* op_item(CPPInstance* self, Py_ssize_t idx) return nullptr; } - Py_ssize_t idx = PyInt_AsSsize_t(pyidx); - if (idx == (Py_ssize_t)-1 && PyErr_Occurred()) - return nullptr; - if (idx < 0) { // this is debatable, and probably should not care, but the use case is pretty // circumscribed anyway, so might as well keep the functionality simple @@ -413,8 +409,6 @@ static PyMethodDef op_methods[] = { (char*)"dispatch to selected overload"}, {(char*)"__smartptr__", (PyCFunction)op_get_smartptr, METH_NOARGS, (char*)"get associated smart pointer, if any"}, - {(char*)"__getitem__", (PyCFunction)op_getitem, METH_O, - (char*)"pointer dereferencing"}, {(char*)"__reshape__", (PyCFunction)op_reshape, METH_O, (char*)"cast pointer to 1D array type"}, {(char*)nullptr, nullptr, 0, nullptr} diff --git a/src/Converters.cxx b/src/Converters.cxx index 7117114..fc940b2 100644 --- a/src/Converters.cxx +++ b/src/Converters.cxx @@ -3373,8 +3373,7 @@ CPyCppyy::Converter* CPyCppyy::CreateConverter(Cppyy::TCppType_t type, cdims_t d } else cnv = CreateConverter(value_type); if (cnv || use_byte_cnv) - return new InitializerListConverter(Cppyy::GetScopeFromType(realType), - CreateConverter(value_type), cnv, Cppyy::SizeOf(value_type)); + return new InitializerListConverter(Cppyy::GetScopeFromType(realType), value_type); } //-- still nothing? use a generalized converter diff --git a/src/Dispatcher.cxx b/src/Dispatcher.cxx index ad627e3..287c166 100644 --- a/src/Dispatcher.cxx +++ b/src/Dispatcher.cxx @@ -146,11 +146,11 @@ namespace { using namespace Cppyy; static inline -std::vector FindBaseMethod(TCppScope_t tbase, const std::string mtCppName) +std::vector FindBaseMethod(TCppScope_t tbase, const std::string mtCppName) { // Recursively walk the inheritance tree to find the overloads of the named method - std::vector result; - result = GetMethodIndicesFromName(tbase, mtCppName); + std::vector result; + result = GetMethodsFromName(tbase, mtCppName); if (result.empty()) { for (TCppIndex_t ibase = 0; ibase < GetNumBases(tbase); ++ibase) { TCppScope_t b = GetScope(GetBaseName(tbase, ibase)); @@ -366,10 +366,10 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct, // TODO: should probably invert this looping; but that makes handling overloads clunky PyObject* key = PyList_GET_ITEM(keys, i); std::string mtCppName = CPyCppyy_PyText_AsString(key); - const auto& v = FindBaseMethod(tbase, mtCppName); - for (auto idx : v) - InjectMethod(Cppyy::GetMethod(tbase, idx), mtCppName, code); - if (!v.empty()) { + const auto& methods = FindBaseMethod(tbase, mtCppName); + for (auto method : methods) + InjectMethod(method, mtCppName, code); + if (!methods.empty()) { if (PyDict_DelItem(clbs, key) != 0) PyErr_Clear(); } } diff --git a/src/Executors.cxx b/src/Executors.cxx index d5b7227..14f9b01 100644 --- a/src/Executors.cxx +++ b/src/Executors.cxx @@ -809,10 +809,8 @@ CPyCppyy::Executor* CPyCppyy::CreateExecutor(const std::string& fullType, cdims_ // // If all fails, void is used, which will cause the return type to be ignored on use - // FIXME: - //assert(!fullType.empty() && "This routine assumes non-empty fullType"); - if (fullType.empty()) - return nullptr; + if (fullType.empty()) + return nullptr; // an exactly matching executor is best ExecFactories_t::iterator h = gExecFactories.find(fullType); diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index d6e43eb..d809a6b 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -568,9 +568,9 @@ static PyObject* vector_iter(PyObject* v) { if (!vi->vi_klass) { // look for a special case of pointer to a class type (which is a builtin, but it // is more useful to treat it polymorphically by allowing auto-downcasts) - const std::string& clean_type = TypeManip::clean_type(value_type, false, false); + const std::string& clean_type = TypeManip::clean_type(Cppyy::GetTypeAsString(value_type), false, false); Cppyy::TCppScope_t c = Cppyy::GetScope(clean_type); - if (c && TypeManip::compound(value_type) == "*") { + if (c && TypeManip::compound(Cppyy::GetTypeAsString(value_type)) == "*") { vi->vi_klass = c; vi->vi_flags = vectoriterobject::kIsPolymorphic; } diff --git a/src/TemplateProxy.cxx b/src/TemplateProxy.cxx index 0285725..df9bd2c 100644 --- a/src/TemplateProxy.cxx +++ b/src/TemplateProxy.cxx @@ -856,11 +856,6 @@ static PyObject* tpp_overload(TemplateProxy* pytmpl, PyObject* args) PyObject* pytype = 0, *pyvalue = 0, *pytrace = 0; PyErr_Fetch(&pytype, &pyvalue, &pytrace); - std::string proto = Utility::ConstructTemplateArgs(nullptr, args); - Cppyy::TCppScope_t scope = ((CPPClass*)pytmpl->fTI->fPyClass)->fCppType; - Cppyy::TCppMethod_t cppmeth = Cppyy::GetMethodTemplate( - scope, pytmpl->fTI->fCppName, proto.substr(1, proto.size()-2)); - if (!cppmeth) { PyErr_Restore(pytype, pyvalue, pytrace); return nullptr;