Python 11 and 12 support

.github/workflows/lsp-tests.yml

@@ -69,7 +69,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.12"
       - name: Run language server Python tests without PyLint
         run: ./gradlew core:integrationTest --tests org.lflang.tests.lsp.LspTests.pythonValidationTestSyntaxOnly core:integrationTestCodeCoverageReport
       - name: Install pylint

.github/workflows/py-tests.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.12'
       - name: Install dependencies OS X
         run: |
           brew install coreutils

core/src/main/java/org/lflang/federated/extensions/PythonExtension.java

@@ -85,6 +85,11 @@ protected String generateSerializationIncludes(
     return code.getCode();
   }
 
+  @Override
+  public String getNetworkBufferType() {
+    return "PyObject*";
+  }
+
   @Override
   public String generateNetworkSenderBody(
       VarRef sendingPort,

core/src/main/java/org/lflang/generator/c/CCmakeGenerator.java

@@ -69,7 +69,7 @@ public class CCmakeGenerator {
 
   private final FileConfig fileConfig;
   private final List<String> additionalSources;
-  private final SetUpMainTarget setUpMainTarget;
+  private SetUpMainTarget setUpMainTarget;
   private final String installCode;
 
   public CCmakeGenerator(FileConfig fileConfig, List<String> additionalSources) {
@@ -90,6 +90,15 @@ public CCmakeGenerator(
     this.installCode = installCode;
   }
 
+  /**
+   * Set the code generator for the CMake main target.
+   *
+   * @param setUpMainTarget
+   */
+  public void setCmakeGenerator(SetUpMainTarget setUpMainTarget) {
+    this.setUpMainTarget = setUpMainTarget;
+  }
+
   /**
    * Generate the contents of a CMakeLists.txt that builds the provided LF C 'sources'. Any error
    * will be reported in the 'errorReporter'.

core/src/main/java/org/lflang/generator/c/CGenerator.java

@@ -319,7 +319,7 @@ public class CGenerator extends GeneratorBase {
 
   private final CTypes types;
 
-  private final CCmakeGenerator cmakeGenerator;
+  protected CCmakeGenerator cmakeGenerator;
 
   protected CGenerator(
       LFGeneratorContext context,

core/src/main/java/org/lflang/generator/python/PythonGenerator.java

@@ -65,6 +65,7 @@
 import org.lflang.target.Target;
 import org.lflang.target.property.DockerProperty;
 import org.lflang.target.property.ProtobufsProperty;
+import org.lflang.target.property.PythonVersionProperty;
 import org.lflang.util.FileUtil;
 import org.lflang.util.LFCommand;
 
@@ -82,14 +83,17 @@
  *
  * @author Soroush Bateni
  */
-public class PythonGenerator extends CGenerator {
+public class PythonGenerator extends CGenerator implements CCmakeGenerator.SetUpMainTarget {
 
   // Used to add statements that come before reactor classes and user code
   private final CodeBuilder pythonPreamble = new CodeBuilder();
 
   // Used to add module requirements to setup.py (delimited with ,)
   private final List<String> pythonRequiredModules = new ArrayList<>();
 
+  /** Indicator that we have already generated top-level preambles. */
+  private Set<Model> generatedTopLevelPreambles = new HashSet<Model>();
+
   private final PythonTypes types;
 
   public PythonGenerator(LFGeneratorContext context) {
@@ -104,8 +108,9 @@ public PythonGenerator(LFGeneratorContext context) {
                 "lib/python_tag.c",
                 "lib/python_time.c",
                 "lib/pythontarget.c"),
-            PythonGenerator::setUpMainTarget,
+            null, // Temporarily, because can't pass this.
             generateCmakeInstall(context.getFileConfig())));
+    cmakeGenerator.setCmakeGenerator(this);
   }
 
   private PythonGenerator(
@@ -186,6 +191,7 @@ public String generatePythonCode(String pyModuleName) {
         "\n",
         "import os",
         "import sys",
+        "print(\"******* Using Python version: %s.%s.%s\" % sys.version_info[:3])",
         "sys.path.append(os.path.dirname(__file__))",
         "# List imported names, but do not use pylint's --extension-pkg-allow-list option",
         "# so that these names will be assumed present without having to compile and install.",
@@ -270,7 +276,12 @@ protected String generateTopLevelPreambles(Reactor ignored) {
       models.add((Model) ASTUtils.toDefinition(this.mainDef.getReactorClass()).eContainer());
     }
     for (Model m : models) {
-      pythonPreamble.pr(PythonPreambleGenerator.generatePythonPreambles(m.getPreambles()));
+      // In the generated Python code, unlike C, all reactors go into the same file.
+      // Therefore, we do not need to generate this if it has already been generated.
+      if (!generatedTopLevelPreambles.contains(m)) {
+        generatedTopLevelPreambles.add(m);
+        pythonPreamble.pr(PythonPreambleGenerator.generatePythonPreambles(m.getPreambles()));
+      }
     }
     return PythonPreambleGenerator.generateCIncludeStatements(
         targetConfig, targetLanguageIsCpp(), hasModalReactors);
@@ -486,9 +497,6 @@ protected void generateUserPreamblesForReactor(Reactor reactor, CodeBuilder src)
   @Override
   protected void generateReactorClassHeaders(
       TypeParameterizedReactor tpr, String headerName, CodeBuilder header, CodeBuilder src) {
-    header.pr(
-        PythonPreambleGenerator.generateCIncludeStatements(
-            targetConfig, targetLanguageIsCpp(), hasModalReactors));
     super.generateReactorClassHeaders(tpr, headerName, header, src);
   }
 
@@ -639,15 +647,27 @@ protected void additionalPostProcessingForModes() {
     PythonModeGenerator.generateResetReactionsIfNeeded(reactors);
   }
 
-  private static String setUpMainTarget(
-      boolean hasMain, String executableName, Stream<String> cSources) {
+  public String getCmakeCode(boolean hasMain, String executableName, Stream<String> cSources) {
+    // According to https://cmake.org/cmake/help/latest/module/FindPython.html#hints, the following
+    // should work to select the version of Python given in your virtual environment.
+    // However, this does not work for me (macOS Sequoia 15.0.1).
+    // As a consequence, the python-version target property can be used to specify the exact Python
+    // version.
+    var pythonVersion =
+        "3.10.0"; // Allows 3.10 or later. Change to "3.10.0...<3.11.0" to require 3.10 by default.
+    if (targetConfig.isSet(PythonVersionProperty.INSTANCE)) {
+      pythonVersion = targetConfig.get(PythonVersionProperty.INSTANCE) + " EXACT";
+    }
     return ("""
             set(CMAKE_POSITION_INDEPENDENT_CODE ON)
             add_compile_definitions(_PYTHON_TARGET_ENABLED)
             add_subdirectory(core)
             set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR})
             set(LF_MAIN_TARGET <pyModuleName>)
-            find_package(Python 3.10.0...<3.11.0 REQUIRED COMPONENTS Interpreter Development)
+            set(Python_FIND_VIRTUALENV FIRST)
+            set(Python_FIND_STRATEGY LOCATION)
+            set(Python_FIND_FRAMEWORK LAST)
+            find_package(Python <pyVersion> REQUIRED COMPONENTS Interpreter Development)
             Python_add_library(
                 ${LF_MAIN_TARGET}
                 MODULE
@@ -667,7 +687,8 @@ private static String setUpMainTarget(
 target_link_libraries(${LF_MAIN_TARGET} PRIVATE ${Python_LIBRARIES})
 target_compile_definitions(${LF_MAIN_TARGET} PUBLIC MODULE_NAME=<pyModuleName>)
 """)
-        .replace("<pyModuleName>", generatePythonModuleName(executableName));
+        .replace("<pyModuleName>", generatePythonModuleName(executableName))
+        .replace("<pyVersion>", pythonVersion);
     // The use of fileConfig.name will break federated execution, but that's fine
   }
 
@@ -677,6 +698,9 @@ private static String generateCmakeInstall(FileConfig fileConfig) {
     // need to replace '\' with '\\' on Windwos for proper escaping in cmake
     final var pyMainName = pyMainPath.toString().replace("\\", "\\\\");
     return """
+  if (NOT DEFINED CMAKE_INSTALL_BINDIR)
+    set(CMAKE_INSTALL_BINDIR "bin")
+  endif()
   if(WIN32)
     file(GENERATE OUTPUT <fileName>.bat CONTENT
       "@echo off

core/src/main/java/org/lflang/generator/python/PythonPortGenerator.java

@@ -177,7 +177,6 @@ public static String generatePythonListForContainedBank(
         "    }",
         "    /* Release the thread. No Python API allowed beyond this point. */",
         "    PyGILState_Release(gstate);",
-        "    Py_FinalizeEx();",
         "    exit(1);",
         "}",
         "for (int i = 0; i < " + generateWidthVariable(reactorName) + "; i++) {",
@@ -193,7 +192,6 @@ public static String generatePythonListForContainedBank(
         "        }",
         "        /* Release the thread. No Python API allowed beyond this point. */",
         "        PyGILState_Release(gstate);",
-        "        Py_FinalizeEx();",
         "        exit(1);",
         "    }",
         "}");

core/src/main/java/org/lflang/generator/python/PythonPreambleGenerator.java

@@ -44,8 +44,8 @@ public static String generateCDefineDirectives(
   public static String generateCIncludeStatements(
       TargetConfig targetConfig, boolean CCppMode, boolean hasModalReactors) {
     CodeBuilder code = new CodeBuilder();
-    code.pr(CPreambleGenerator.generateIncludeStatements(targetConfig, CCppMode));
     code.pr("#include \"pythontarget.h\"");
+    code.pr(CPreambleGenerator.generateIncludeStatements(targetConfig, CCppMode));
     if (hasModalReactors) {
       code.pr("#include \"include/modal_models/definitions.h\"");
     }

core/src/main/java/org/lflang/generator/python/PythonReactionGenerator.java

@@ -102,10 +102,16 @@ private static String generateCPythonFunctionCaller(
                 + "."
                 + pythonFunctionName
                 + "\");",
+            "PyObject *arglist = Py_BuildValue(\"("
+                + "O".repeat(pyObjects.size())
+                + ")\""
+                + pyObjectsJoined
+                + ");",
             "PyObject *rValue = PyObject_CallObject(",
             "    self->" + cpythonFunctionName + ", ",
-            "    Py_BuildValue(\"(" + "O".repeat(pyObjects.size()) + ")\"" + pyObjectsJoined + ")",
+            "    arglist",
             ");",
+            "Py_DECREF(arglist);",
             "if (rValue == NULL) {",
             "    lf_print_error(\"FATAL: Calling reaction "
                 + reactorDeclName

core/src/main/java/org/lflang/target/Target.java

@@ -26,39 +26,7 @@
 import java.util.Set;
 import net.jcip.annotations.Immutable;
 import org.lflang.lf.TargetDecl;
-import org.lflang.target.property.AuthProperty;
-import org.lflang.target.property.BuildCommandsProperty;
-import org.lflang.target.property.BuildTypeProperty;
-import org.lflang.target.property.CargoDependenciesProperty;
-import org.lflang.target.property.CargoFeaturesProperty;
-import org.lflang.target.property.ClockSyncModeProperty;
-import org.lflang.target.property.ClockSyncOptionsProperty;
-import org.lflang.target.property.CmakeIncludeProperty;
-import org.lflang.target.property.CompileDefinitionsProperty;
-import org.lflang.target.property.CompilerProperty;
-import org.lflang.target.property.CoordinationOptionsProperty;
-import org.lflang.target.property.CoordinationProperty;
-import org.lflang.target.property.DockerProperty;
-import org.lflang.target.property.ExportDependencyGraphProperty;
-import org.lflang.target.property.ExternalRuntimePathProperty;
-import org.lflang.target.property.FilesProperty;
-import org.lflang.target.property.KeepaliveProperty;
-import org.lflang.target.property.NoRuntimeValidationProperty;
-import org.lflang.target.property.NoSourceMappingProperty;
-import org.lflang.target.property.PlatformProperty;
-import org.lflang.target.property.PrintStatisticsProperty;
-import org.lflang.target.property.ProtobufsProperty;
-import org.lflang.target.property.Ros2DependenciesProperty;
-import org.lflang.target.property.Ros2Property;
-import org.lflang.target.property.RuntimeVersionProperty;
-import org.lflang.target.property.RustIncludeProperty;
-import org.lflang.target.property.SchedulerProperty;
-import org.lflang.target.property.SingleFileProjectProperty;
-import org.lflang.target.property.SingleThreadedProperty;
-import org.lflang.target.property.TracePluginProperty;
-import org.lflang.target.property.TracingProperty;
-import org.lflang.target.property.VerifyProperty;
-import org.lflang.target.property.WorkersProperty;
+import org.lflang.target.property.*;
 
 /**
  * Enumeration of targets and their associated properties.
@@ -634,6 +602,7 @@ public void initialize(TargetConfig config) {
               KeepaliveProperty.INSTANCE,
               NoSourceMappingProperty.INSTANCE,
               ProtobufsProperty.INSTANCE,
+              PythonVersionProperty.INSTANCE,
               SchedulerProperty.INSTANCE,
               SingleThreadedProperty.INSTANCE,
               TracingProperty.INSTANCE,

core/src/main/java/org/lflang/target/property/PythonVersionProperty.java

@@ -0,0 +1,17 @@
+package org.lflang.target.property;
+
+/** A specific Python version to use. */
+public final class PythonVersionProperty extends StringProperty {
+
+  /** Singleton target property instance. */
+  public static final PythonVersionProperty INSTANCE = new PythonVersionProperty();
+
+  private PythonVersionProperty() {
+    super();
+  }
+
+  @Override
+  public String name() {
+    return "python-version";
+  }
+}

test/Python/src/serialization/CustomSerializer.lf

@@ -5,7 +5,10 @@ target Python {
 }
 
 preamble {=
+  # Note that both federates will try to install the pickle_serializer package. One will likely fail,
+  # but the other will succeed.
   os.system("pip install ./src/serialization/pickle_serializer/ --user")
+  import pickle_serializer
 =}
 
 reactor Client {
@@ -66,5 +69,5 @@ federated reactor {
   client = new Client()
   server = new Server()
   server.server_message -> client.server_message after 100 ms serializer "pickle_serializer"
-  client.client_message -> server.client_message serializer "pickle_serializer"
+  client.client_message -> server.client_message after 100 ms serializer "pickle_serializer"
 }