Simplify Python extension loading and other Python improvements

.gitignore

@@ -16,7 +16,7 @@ include/cantera/base/system.h.gch
 include/cantera/ext/
 interfaces/matlab/ctpath.m
 interfaces/matlab/Contents.m
-src/extensions/pythonExtensions.h
+src/extensions/delegator.h
 stage/
 .sconsign.dblite
 .sconf_temp

include/cantera/cython/funcWrapper.h

@@ -162,10 +162,14 @@ class Func1Py : public Cantera::Func1
     void* m_pyobj;
 };
 
+// @todo Remove the second case here when Cython 0.29.x is no longer supported
+#if defined(CYTHON_HEX_VERSION) && CYTHON_HEX_VERSION >= 0x001DFFFF
+extern PyObject* pyCanteraError;
+#else
 extern "C" {
     extern PyObject* pyCanteraError;
 }
-
+#endif
 
 
 //! Take a function which requires Python function information (as a PyFuncInfo
@@ -221,6 +225,8 @@ inline int translate_exception()
         PyErr_SetObject(exn.m_type, exn.m_value);
     } catch (const std::out_of_range& exn) {
         PyErr_SetString(PyExc_IndexError, exn.what());
+    } catch (const Cantera::NotImplementedError& exn) {
+        PyErr_SetString(PyExc_NotImplementedError, exn.what());
     } catch (const Cantera::CanteraError& exn) {
         PyErr_SetString(pyCanteraError, exn.what());
     } catch (const std::exception& exn) {

include/cantera/extensions/PythonExtensionManager.h

@@ -11,7 +11,6 @@
 namespace Cantera
 {
 
-
 //! Class for managing user-defined Cantera extensions written in Python
 //!
 //! Handles Python initialization if the main application is not the Python interpreter.
@@ -26,7 +25,6 @@ namespace Cantera
 class PythonExtensionManager : public ExtensionManager
 {
 public:
-    PythonExtensionManager();
     virtual void registerRateBuilders(const std::string& extensionName) override;
 
     void registerRateBuilder(const string& moduleName,
@@ -38,7 +36,7 @@ class PythonExtensionManager : public ExtensionManager
         const string& className, const string& rateName) override;
 
 private:
-    static bool s_imported;
+    PythonExtensionManager() = default;
 };
 
 }

interfaces/cython/SConscript

@@ -40,17 +40,27 @@ for pyxfile in multi_glob(localenv, "cantera", "pyx"):
     else:
         cython_env = localenv
 
+    cython_output = f"cantera/{pyxfile.name.replace('.pyx', '.cpp')}"
+    if pyxfile.name == "delegator.pyx":
+        cython_output = [cython_output, "cantera/delegator.h"]
+
     cythonized = cython_env.Command(
-         f"cantera/{pyxfile.name.replace('.pyx', '.cpp')}", pyxfile,
+         cython_output, pyxfile,
          f'''${{python_cmd}} -c "import Cython.Build; Cython.Build.cythonize(r'${{SOURCE}}', compiler_directives={directives!r})"'''
     )
     for pxd in multi_glob(cython_env, "cantera", "pxd"):
         cython_env.Depends(cythonized, pxd)
 
     obj = cython_env.SharedObject(
-        f"#build/temp-py/{pyxfile.name.split('.')[0]}", cythonized)
+        f"#build/temp-py/{pyxfile.name.split('.')[0]}", cythonized[0])
     cython_obj.append(obj)
-cython_obj.extend(env['python_ext_objects'])
+
+copy_header = localenv.Command('#src/extensions/delegator.h', '#build/python/cantera/delegator.h',
+                          Copy('$TARGET', '$SOURCE'))
+ext_manager = localenv.SharedObject("#src/extensions/PythonExtensionManager.cpp")
+
+cython_obj.extend(ext_manager)
+env.Depends(ext_manager, copy_header)
 
 module_ext = localenv["py_module_ext"]
 ext = localenv.LoadableModule(f"cantera/_cantera{module_ext}",

interfaces/cython/cantera/_onedim.pyx

@@ -25,6 +25,7 @@ cdef class Domain1D:
             raise TypeError("Can't instantiate abstract class Domain1D.")
 
         self.gas = phase
+        # Block species from being added to the phase as long as this object exists
         self.gas._references[self._weakref_proxy] = True
         self.set_default_tolerances()
 
@@ -441,6 +442,7 @@ cdef class ReactingSurface1D(Boundary1D):
             super().__init__(gas, name=name)
 
         self.surface = phase
+        # Block species from being added to the phase as long as this object exists
         self.surface._references[self._weakref_proxy] = True
 
     def __dealloc__(self):

interfaces/cython/cantera/delegator.pxd

@@ -69,6 +69,13 @@ cdef extern from "cantera/base/ExtensionManager.h" namespace "Cantera":
 
         shared_ptr[CxxExtensionManager] build(string&)
 
+
+cdef extern from "cantera/base/ExtensionManagerFactory.h" namespace "Cantera":
+    cdef cppclass CxxExtensionManagerFactory "Cantera::ExtensionManagerFactory":
+        @staticmethod
+        shared_ptr[CxxExtensionManager] build(string&)
+
+
 cdef extern from "cantera/extensions/PythonExtensionManager.h" namespace "Cantera":
     cdef cppclass CxxPythonExtensionManager "Cantera::PythonExtensionManager" (CxxExtensionManager):
         @staticmethod
@@ -84,4 +91,4 @@ cdef extern from "cantera/base/ExtensionManagerFactory.h" namespace "Cantera":
 ctypedef CxxDelegator* CxxDelegatorPtr
 
 cdef int assign_delegates(object, CxxDelegator*) except -1
-cdef void callback_v(PyFuncInfo& funcInfo)
+cdef void callback_v(PyFuncInfo& funcInfo) noexcept

interfaces/cython/cantera/delegator.pyx

@@ -3,11 +3,18 @@
 
 import inspect
 import sys
+import importlib
+
+from libc.stdlib cimport malloc
+from libc.string cimport strcpy
 
 from ._utils import CanteraError
 from ._utils cimport stringify, pystr, anymap_to_dict
 from .units cimport Units
-from .reaction import ExtensibleRate, ExtensibleRateData
+# from .reaction import ExtensibleRate, ExtensibleRateData
+from .reaction cimport (ExtensibleRate, ExtensibleRateData, CxxReaction,
+    CxxReactionRateDelegator, CxxReactionDataDelegator)
+from .solutionbase cimport CxxSolution, _assign_Solution
 from cython.operator import dereference as deref
 
 # ## Implementation for each delegated function type
@@ -78,7 +85,7 @@ from cython.operator import dereference as deref
 # function.
 
 # Wrapper for functions of type void()
-cdef void callback_v(PyFuncInfo& funcInfo):
+cdef void callback_v(PyFuncInfo& funcInfo) noexcept:
     try:
         (<object>funcInfo.func())()
     except BaseException as e:
@@ -87,7 +94,7 @@ cdef void callback_v(PyFuncInfo& funcInfo):
         funcInfo.setExceptionValue(<PyObject*>exc_value)
 
 # Wrapper for functions of type void(double)
-cdef void callback_v_d(PyFuncInfo& funcInfo, double arg):
+cdef void callback_v_d(PyFuncInfo& funcInfo, double arg) noexcept:
     try:
         (<object>funcInfo.func())(arg)
     except BaseException as e:
@@ -96,7 +103,7 @@ cdef void callback_v_d(PyFuncInfo& funcInfo, double arg):
         funcInfo.setExceptionValue(<PyObject*>exc_value)
 
 # Wrapper for functions of type void(bool)
-cdef void callback_v_b(PyFuncInfo& funcInfo, cbool arg):
+cdef void callback_v_b(PyFuncInfo& funcInfo, cbool arg) noexcept:
     try:
         (<object>funcInfo.func())(arg)
     except BaseException as e:
@@ -106,7 +113,7 @@ cdef void callback_v_b(PyFuncInfo& funcInfo, cbool arg):
 
 # Wrapper for functions of type void(const AnyMap&, const UnitStack&)
 cdef void callback_v_cAMr_cUSr(PyFuncInfo& funcInfo, const CxxAnyMap& arg1,
-                               const CxxUnitStack& arg2):
+                               const CxxUnitStack& arg2) noexcept:
 
     pyArg1 = anymap_to_dict(<CxxAnyMap&>arg1)  # cast away constness
     pyArg2 = Units.copy(arg2.product())
@@ -118,7 +125,7 @@ cdef void callback_v_cAMr_cUSr(PyFuncInfo& funcInfo, const CxxAnyMap& arg1,
         funcInfo.setExceptionValue(<PyObject*>exc_value)
 
 # Wrapper for functions of type void(double*)
-cdef void callback_v_dp(PyFuncInfo& funcInfo, size_array1 sizes, double* arg):
+cdef void callback_v_dp(PyFuncInfo& funcInfo, size_array1 sizes, double* arg) noexcept:
     cdef double[:] view = <double[:sizes[0]]>arg if sizes[0] else None
 
     try:
@@ -130,7 +137,7 @@ cdef void callback_v_dp(PyFuncInfo& funcInfo, size_array1 sizes, double* arg):
 
 # Wrapper for functions of type void(double, double*)
 cdef void callback_v_d_dp(PyFuncInfo& funcInfo, size_array1 sizes, double arg1,
-                                double* arg2):
+                          double* arg2) noexcept:
     cdef double[:] view = <double[:sizes[0]]>arg2 if sizes[0] else None
 
     try:
@@ -142,7 +149,7 @@ cdef void callback_v_d_dp(PyFuncInfo& funcInfo, size_array1 sizes, double arg1,
 
 # Wrapper for functions of type void(double*, double*, double*)
 cdef void callback_v_dp_dp_dp(PyFuncInfo& funcInfo,
-        size_array3 sizes, double* arg1, double* arg2, double* arg3):
+        size_array3 sizes, double* arg1, double* arg2, double* arg3) noexcept:
 
     cdef double[:] view1 = <double[:sizes[0]]>arg1 if sizes[0] else None
     cdef double[:] view2 = <double[:sizes[1]]>arg2 if sizes[1] else None
@@ -155,7 +162,7 @@ cdef void callback_v_dp_dp_dp(PyFuncInfo& funcInfo,
         funcInfo.setExceptionValue(<PyObject*>exc_value)
 
 # Wrapper for functions of type double(void*)
-cdef int callback_d_vp(PyFuncInfo& funcInfo, double& out, void* obj):
+cdef int callback_d_vp(PyFuncInfo& funcInfo, double& out, void* obj) noexcept:
     try:
         ret = (<object>funcInfo.func())(<object>obj)
         if ret is None:
@@ -170,7 +177,7 @@ cdef int callback_d_vp(PyFuncInfo& funcInfo, double& out, void* obj):
     return -1
 
 # Wrapper for functions of type string(size_t)
-cdef int callback_s_sz(PyFuncInfo& funcInfo, string& out, size_t arg):
+cdef int callback_s_sz(PyFuncInfo& funcInfo, string& out, size_t arg) noexcept:
     try:
         ret = (<object>funcInfo.func())(arg)
         if ret is None:
@@ -185,7 +192,7 @@ cdef int callback_s_sz(PyFuncInfo& funcInfo, string& out, size_t arg):
     return -1
 
 # Wrapper for functions of type size_t(string&)
-cdef int callback_sz_csr(PyFuncInfo& funcInfo, size_t& out, const string& arg):
+cdef int callback_sz_csr(PyFuncInfo& funcInfo, size_t& out, const string& arg) noexcept:
     try:
         ret = (<object>funcInfo.func())(pystr(arg))
         if ret is None:
@@ -201,7 +208,7 @@ cdef int callback_sz_csr(PyFuncInfo& funcInfo, size_t& out, const string& arg):
 
 # Wrapper for functions of type void(double, double*, double*)
 cdef void callback_v_d_dp_dp(PyFuncInfo& funcInfo, size_array2 sizes, double arg1,
-                                double* arg2, double* arg3):
+                             double* arg2, double* arg3) noexcept:
     cdef double[:] view1 = <double[:sizes[0]]>arg2 if sizes[0] else None
     cdef double[:] view2 = <double[:sizes[1]]>arg3 if sizes[1] else None
 
@@ -339,12 +346,66 @@ cdef int assign_delegates(obj, CxxDelegator* delegator) except -1:
 
     return 0
 
+
 # Specifications for ReactionRate delegators that have not yet been registered with
 # ReactionRateFactory. This list is read by PythonExtensionManager::registerRateBuilders
 # and then cleared.
 _rate_delegators = []
 _rate_data_delegators = []
 
+
+cdef public char* ct_getExceptionString(object exType, object exValue, object exTraceback):
+    import traceback
+    result = str(exValue) + "\n\n"
+    result += "".join(traceback.format_exception(exType, exValue, exTraceback))
+    tmp = bytes(result.encode())
+    cdef char* c_string = <char*> malloc((len(tmp) + 1) * sizeof(char))
+    strcpy(c_string, tmp)
+    return c_string
+
+
+cdef public object ct_newPythonExtensibleRate(CxxReactionRateDelegator* delegator,
+                                              const string& module_name,
+                                              const string& class_name):
+
+    mod = importlib.import_module(module_name.decode())
+    cdef ExtensibleRate rate = getattr(mod, class_name.decode())(init=False)
+    rate.set_cxx_object(delegator)
+    return rate
+
+
+cdef public object ct_newPythonExtensibleRateData(CxxReactionDataDelegator* delegator,
+                                                  const string& module_name,
+                                                  const string& class_name):
+
+    mod = importlib.import_module(module_name.decode())
+    cdef ExtensibleRateData data = getattr(mod, class_name.decode())()
+    data.set_cxx_object(delegator)
+    return data
+
+
+cdef public ct_registerReactionDelegators():
+    cdef shared_ptr[CxxExtensionManager] mgr = (
+        CxxExtensionManagerFactory.build(stringify("python")))
+
+    for module, cls, name in _rate_delegators:
+        mgr.get().registerRateBuilder(stringify(module), stringify(cls), stringify(name))
+
+    _rate_delegators.clear()
+
+    for module, cls, name in _rate_data_delegators:
+        mgr.get().registerRateDataBuilder(stringify(module), stringify(cls), stringify(name))
+
+    _rate_data_delegators.clear()
+
+
+cdef public object ct_wrapSolution(shared_ptr[CxxSolution] soln):
+    from .composite import Solution
+    pySoln = Solution(init=False)
+    _assign_Solution(pySoln, soln, False, weak=True)
+    return pySoln
+
+
 CxxPythonExtensionManager.registerSelf()
 
 def extension(*, name, data=None):

interfaces/cython/cantera/func1.pxd

@@ -7,7 +7,7 @@
 from .ctcxx cimport *
 
 cdef extern from "cantera/cython/funcWrapper.h":
-    ctypedef double (*callback_wrapper)(double, void*, void**)
+    ctypedef double (*callback_wrapper)(double, void*, void**) except? 0.0
     cdef int translate_exception()
 
     cdef cppclass CxxFunc1 "Func1Py":

interfaces/cython/cantera/func1.pyx

@@ -7,7 +7,7 @@ import numpy as np
 
 from ._utils cimport *
 
-cdef double func_callback(double t, void* obj, void** err):
+cdef double func_callback(double t, void* obj, void** err) except? 0.0:
     """
     This function is called from C/C++ to evaluate a `Func1` object ``obj``,
     returning the value of the function at ``t``. If an exception occurs while

interfaces/cython/cantera/mixture.pyx

@@ -52,6 +52,7 @@ cdef class Mixture:
             phases = [(p, 1 if i == 0 else 0) for i,p in enumerate(phases)]
 
         for phase,moles in phases:
+            # Block species from being added to the phase as long as this object exists
             phase._references[self._weakref_proxy] = True
             self.mix.addPhase(phase.thermo, moles)
             self._phases.append(phase)

interfaces/cython/cantera/reaction.pxd

@@ -6,21 +6,23 @@
 
 from .ctcxx cimport *
 from .func1 cimport *
+from .delegator cimport CxxDelegator
 
-cdef extern from "<map>" namespace "std":
+# @todo Replace with import from libcpp.map once Cython 0.29.x is no longer supported
+cdef extern from "<map>" namespace "std" nogil:
     cdef cppclass multimap[T, U]:
         cppclass iterator:
-            pair[T, U]& operator*() nogil
-            iterator operator++() nogil
-            iterator operator--() nogil
-            bint operator==(iterator) nogil
-            bint operator!=(iterator) nogil
-        multimap() nogil except +
-        U& operator[](T&) nogil
-        iterator begin() nogil
-        iterator end() nogil
-        pair[iterator, bint] insert(pair[T, U]) nogil
-        iterator find(T&) nogil
+            pair[T, U]& operator*()
+            iterator operator++()
+            iterator operator--()
+            bint operator==(iterator)
+            bint operator!=(iterator)
+        multimap() except +
+        U& operator[](T&)
+        iterator begin()
+        iterator end()
+        pair[iterator, bint] insert(pair[T, U])
+        iterator find(T&)
 
 
 cdef extern from "cantera/kinetics/ReactionRateFactory.h" namespace "Cantera":
@@ -189,10 +191,10 @@ cdef extern from "cantera/kinetics/Custom.h" namespace "Cantera":
 
 
 cdef extern from "cantera/kinetics/ReactionRateDelegator.h" namespace "Cantera":
-    cdef cppclass CxxReactionDataDelegator "Cantera::ReactionDataDelegator":
+    cdef cppclass CxxReactionDataDelegator "Cantera::ReactionDataDelegator" (CxxDelegator):
         CxxReactionDataDelegator()
 
-    cdef cppclass CxxReactionRateDelegator "Cantera::ReactionRateDelegator" (CxxReactionRate):
+    cdef cppclass CxxReactionRateDelegator "Cantera::ReactionRateDelegator" (CxxDelegator, CxxReactionRate):
         CxxReactionRateDelegator()
         void setType(string&)
 

interfaces/cython/cantera/reactor.pyx

@@ -53,6 +53,7 @@ cdef class ReactorBase:
         properties and kinetic rates for this reactor.
         """
         self._thermo = solution
+        # Block species from being added to the phase as long as this object exists
         self._thermo._references[self._weakref_proxy] = True
         self.rbase.setThermoMgr(deref(solution.thermo))