Merge branch 'master' into pre-build-cmd

.github/scripts/run-zephyr-tests.sh

@@ -1,15 +1,7 @@
 #!/bin/bash
-timeout=300 # 5min timeout is hopefully enough
-verbose=false
 
-# Function to recursively find all folders containing the top-level CMakeLists.txt
-overall_success=true
-num_successes=0
-num_failures=0
-failed_tests=""
-
-# Skip tests doing file IO and tracing
-skip=("FileReader" "FilePkgReader" "Tracing" "ThreadedThreaded" "CountTest" "AsyncCallback")
+# Skip
+skip=("FileReader" "FilePkgReader")
 
 find_kconfig_folders() {
     if [ -f "$folder/CMakeLists.txt" ]; then
@@ -20,17 +12,15 @@ find_kconfig_folders() {
             echo "Skipping: $test_name"
         else
             echo "Running: $test_name"
-            if run_qemu_zephyr_test "$folder"; then
-                echo "Test $test_name successful"
+            run_qemu_zephyr_test $folder
+            if [ "$?" -eq 0 ]; then
+                echo "Test passed"
                 let "num_successes+=1"
             else
-                echo "Test $test_name failed"
-                let "num_failures+=1"
-                failed_tests+="$test_name, "
-                overall_success=false
+                echo "Test failed."
+                exit 1
             fi
         fi
-
       return
     fi
     for folder in "$1"/*; do
@@ -40,88 +30,55 @@ find_kconfig_folders() {
     done
 }
 
-run_native_zephyr_test() {
-    return_val=0
-    pushd $1/build
-
-    rm -f res.text
-
-    timeout 60s make run | tee res.txt
-    result=$?
-
-    if [ $result -eq 0 ]; then
-        echo "Command completed within the timeout."
-        return_val=0
-    else
-        echo "Command terminated or timed out."
-        echo "Test output:"
-        echo "----------------------------------------------------------------"
-        cat res.txt
-        echo "----------------------------------------------------------------"
-        return_val=1
-    fi
-
-    popd
-    return "$return_val"
-
-
-
-
-}
-
-# Run Zephyr test until either: Timeout or finds match in output
-# https://www.unix.com/shell-programming-and-scripting/171401-kill-process-if-grep-match-found.html
+# Run Zephyr test until either: Timeout or finds match in output using expect
+# https://spin.atomicobject.com/monitoring-stdout-with-a-timeout/
 run_qemu_zephyr_test() {
     success=false
     pushd $1/build
-
-    rm -f /tmp/procfifo
-    rm -f res.text
-    mkfifo /tmp/procfifo
-
-    make run | tee res.txt >  /tmp/procfifo &
-    PID=$!
-    SECONDS=0
-    while IFS= read -t $timeout line
-    do
-        if [ "$verbose" = "true" ]; then  echo $line; fi
-
-        if echo "$line" | grep -q 'FATAL ERROR';
+    res="_res.txt"
+    # Run test program in background and pipe results to a file.
+    make run > $res &
+    if [ $? -ne 0 ]; then
+        echo "ERROR: make run failed."
+        exit 1
+    fi
+    pid=$!
+    return_val=2
+    wait_count=0
+    timeout=60
+    # Parse the file and match on known outputs. With timeout.
+    while [ "$wait_count" -le "$timeout" ]; do
+        sleep 1
+        if grep --quiet 'FATAL ERROR' "$res"
         then
-            echo "Matched on ERROR"
-            echo $line
-            success=false
-            pkill -P $$
+            cat $res
+            echo "-----------------------------------------------------------------------------------------------------------"
+            echo "ERROR: Found 'FATAL ERROR'"
+            return_val=1
             break
         fi
-
-        if echo "$line" | grep -q '^exit';
+        if grep --quiet '^exit' "$res"
         then
-                echo "Matched on exit"
-                success=true
-                pkill -P $$
-                break
-        fi
-
-        if (($SECONDS > $timeout)) ; then
-            echo "Timeout without freeze"
-            success=false
+            cat $res
+            echo "-----------------------------------------------------------------------------------------------------------"
+            echo "SUCCESS: Found 'exit'"
+            return_val=0
             break
         fi
-    done < /tmp/procfifo
 
-    return_val=0
-    if [ "$success" = false ]; then
-        echo "General Timeout"
-        pkill -P $$
-        echo "Test output:"
-        echo "----------------------------------------------------------------"
-        cat res.txt
-        echo "----------------------------------------------------------------"
-        return_val=1
+        ((wait_count++))
+    done
+    kill $pid
+    if [ $? -ne 0 ]; then
+        echo "ERROR: Could not kill qemu process"
+        exit 1
     fi
 
-    rm -f /tmp/procfifo
+    if [ "$return_val" -eq 2 ]; then
+        cat $res
+        echo "-----------------------------------------------------------------------------------------------------------"
+        echo "ERROR: Timed out"
+    fi
     popd
     return "$return_val"
 }
@@ -137,14 +94,8 @@ cd -
 
 # Print report
 echo "================================================================================================================"
-
-if [ "$overall_success" = false ]; then
-    echo "Results: FAIL"
-else
-    echo "Results: PASS"
-fi
+echo "Tests passed!"
 echo "Number of passes: $num_successes"
-echo "Number of fails: $num_failures"
 echo "Skipped tests: ${skip[@]}"
 
 if [ "$overall_success" = false ]; then

.github/workflows/custom-build.yml

@@ -0,0 +1,43 @@
+name: Custom build
+
+# Trigger the workflow every day at 5 AM (UTC).
+on:
+  workflow_dispatch:
+
+jobs:
+  custom-build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out lingua-franca repository
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          fetch-depth: 0
+      - name: Prepare build environment
+        uses: ./.github/actions/prepare-build-env
+      - name: Modify property file to contain the commit SHA
+        shell: bash
+        run: |
+          TIMESTAMP="$(date +'%Y-%m-%d')"
+          SHA="$(git rev-parse --short HEAD)"
+          sed --regexp-extended --in-place "s/^(VERSION = .*)$/\1 ${SHA} ${TIMESTAMP}/" core/src/main/resources/org/lflang/StringsBundle.properties
+      - name: Build and package lf cli tools (nightly build)
+        shell: bash
+        run: |
+          export TIMESTAMP="$(date +'%Y%m%d%H%M%S')"
+          echo "timestamp=$TIMESTAMP" >> "$GITHUB_ENV"
+          ./gradlew build -Pnightly=$TIMESTAMP -PtargetOS=Linux -PtargetArch=x86_64
+          ./gradlew assemble -Pnightly=$TIMESTAMP -PtargetOS=Linux -PtargetArch=aarch64
+          ./gradlew assemble -Pnightly=$TIMESTAMP -PtargetOS=MacOS -PtargetArch=x86_64
+          ./gradlew assemble -Pnightly=$TIMESTAMP -PtargetOS=MacOS -PtargetArch=aarch64
+          ./gradlew assemble -Pnightly=$TIMESTAMP -PtargetOS=Windows -PtargetArch=x86_64
+      - name: Deploy nightly release
+        uses: marvinpinto/action-automatic-releases@latest
+        with:
+          repo_token: "${{ secrets.NIGHTLY_BUILD }}"
+          automatic_release_tag: '${{ github.ref_name }}'
+          prerelease: true
+          draft: true
+          title: "Custom Lingua Franca build for ${{ github.ref_name }} branch"
+          files: |
+            build/distributions/*

core/src/main/java/org/lflang/MessageReporterBase.java

@@ -40,11 +40,17 @@ public Stage2 at(Path file, Range range) {
 
   @Override
   public Stage2 at(EObject node) {
+    if (node == null) {
+      return nowhere();
+    }
     return wrap((severity, message) -> reportOnNode(node, severity, message));
   }
 
   @Override
   public Stage2 at(EObject node, EStructuralFeature feature) {
+    if (node == null) {
+      return nowhere();
+    }
     return wrap((severity, message) -> reportOnNode(node, feature, severity, message));
   }
 

core/src/main/java/org/lflang/federated/extensions/CExtensionUtils.java

@@ -201,6 +201,7 @@ public static void handleCompileDefinitions(
     }
     definitions.put("NUMBER_OF_FEDERATES", String.valueOf(numOfFederates));
     definitions.put("EXECUTABLE_PREAMBLE", "");
+    definitions.put("FEDERATE_ID", String.valueOf(federate.id));
 
     CompileDefinitionsProperty.INSTANCE.update(federate.targetConfig, definitions);
 
@@ -303,7 +304,8 @@ public static void generateCMakeInclude(
         "add_compile_definitions(LF_SOURCE_DIRECTORY=\"" + fileConfig.srcPath + "\")");
     cmakeIncludeCode.pr(
         "add_compile_definitions(LF_PACKAGE_DIRECTORY=\"" + fileConfig.srcPkgPath + "\")");
-
+    cmakeIncludeCode.pr(
+        "add_compile_definitions(LF_SOURCE_GEN_DIRECTORY=\"" + fileConfig.getSrcGenPath() + "\")");
     try (var srcWriter = Files.newBufferedWriter(cmakeIncludePath)) {
       srcWriter.write(cmakeIncludeCode.getCode());
     }

core/src/main/java/org/lflang/federated/launcher/FedLauncherGenerator.java

@@ -124,7 +124,7 @@ public void doGenerate(List<FederateInstance> federates, RtiConfig rtiConfig) {
       target = user + "@" + host;
     }
 
-    shCode.append("#### Host is ").append(host);
+    shCode.append("#### Host is ").append(host).append("\n");
 
     // Launch the RTI in the foreground.
     if (host.equals("localhost") || host.equals("0.0.0.0")) {

core/src/main/java/org/lflang/generator/CodeBuilder.java

@@ -126,7 +126,8 @@ public void prSourceLineNumber(EObject eObject, boolean suppress) {
         }
       }
       // Extract the filename from eResource, an astonishingly difficult thing to do.
-      String filePath = CommonPlugin.resolve(eObject.eResource().getURI()).path();
+      String filePath =
+          CommonPlugin.resolve(eObject.eResource().getURI()).path().replace("\\", "\\\\");
       pr("#line " + (node.getStartLine() + offset) + " \"" + filePath + "\"");
     }
   }
@@ -558,7 +559,11 @@ public CodeMap writeToFile(String path) throws IOException {
     for (var line : (Iterable<String>) () -> s.lines().iterator()) {
       lineNumber++;
       if (line.contains(END_SOURCE_LINE_NUMBER_TAG) && !path.endsWith(".ino")) {
-        out.append("#line ").append(lineNumber).append(" \"").append(path).append("\"");
+        out.append("#line ")
+            .append(lineNumber)
+            .append(" \"")
+            .append(path.replace("\\", "\\\\"))
+            .append("\"");
       } else {
         out.append(line);
       }

core/src/main/java/org/lflang/generator/c/CCmakeGenerator.java

@@ -422,6 +422,14 @@ CodeBuilder generateCMakeCode(
     }
     cMakeCode.newLine();
 
+    // Add definition of directory where the main CMakeLists.txt file resides because this is where
+    // any files specified by the `file` target directive will be put.
+    cMakeCode.pr(
+        "# Define directory in which files from the 'files' target directive will be put.");
+    cMakeCode.pr(
+        "target_compile_definitions(${LF_MAIN_TARGET} PUBLIC"
+            + " LF_TARGET_FILES_DIRECTORY=\"${CMAKE_CURRENT_LIST_DIR}\")");
+
     cMakeCode.pr(cMakeExtras);
     cMakeCode.newLine();
 

core/src/main/java/org/lflang/generator/c/CCompiler.java

@@ -42,6 +42,7 @@
 import org.lflang.target.property.CompilerProperty;
 import org.lflang.target.property.PlatformProperty;
 import org.lflang.target.property.PlatformProperty.PlatformOptions;
+import org.lflang.target.property.TracePluginProperty;
 import org.lflang.target.property.type.BuildTypeType.BuildType;
 import org.lflang.target.property.type.PlatformType.Platform;
 import org.lflang.util.FileUtil;
@@ -219,11 +220,13 @@ private static List<String> cmakeOptions(TargetConfig targetConfig, FileConfig f
     String maybeQuote = ""; // Windows seems to require extra level of quoting.
     String srcPath = fileConfig.srcPath.toString(); // Windows requires escaping the backslashes.
     String rootPath = fileConfig.srcPkgPath.toString();
+    String srcGenPath = fileConfig.getSrcGenPath().toString();
     if (separator.equals("\\")) {
       separator = "\\\\\\\\";
       maybeQuote = "\\\"";
       srcPath = srcPath.replaceAll("\\\\", "\\\\\\\\");
       rootPath = rootPath.replaceAll("\\\\", "\\\\\\\\");
+      srcGenPath = srcGenPath.replaceAll("\\\\", "\\\\\\\\");
     }
     arguments.addAll(
         List.of(
@@ -235,13 +238,22 @@ private static List<String> cmakeOptions(TargetConfig targetConfig, FileConfig f
             "-DCMAKE_INSTALL_BINDIR="
                 + FileUtil.toUnixString(fileConfig.getOutPath().relativize(fileConfig.binPath)),
             "-DLF_FILE_SEPARATOR=\"" + maybeQuote + separator + maybeQuote + "\""));
+    var tracePlugin = targetConfig.getOrDefault(TracePluginProperty.INSTANCE);
+    if (tracePlugin != null) {
+      arguments.add(
+          "-DLF_TRACE_PLUGIN="
+              + targetConfig
+                  .getOrDefault(TracePluginProperty.INSTANCE)
+                  .getImplementationArchiveFile());
+    }
     // Add #define for source file directory.
     // Do not do this for federated programs because for those, the definition is put
     // into the cmake file (and fileConfig.srcPath is the wrong directory anyway).
     if (!fileConfig.srcPath.toString().contains("fed-gen")) {
       // Do not convert to Unix path
       arguments.add("-DLF_SOURCE_DIRECTORY=\"" + maybeQuote + srcPath + maybeQuote + "\"");
       arguments.add("-DLF_PACKAGE_DIRECTORY=\"" + maybeQuote + rootPath + maybeQuote + "\"");
+      arguments.add("-DLF_SOURCE_GEN_DIRECTORY=\"" + maybeQuote + srcGenPath + maybeQuote + "\"");
     }
     arguments.add(FileUtil.toUnixString(fileConfig.getSrcGenPath()));
 

core/src/main/java/org/lflang/generator/c/CConstructorGenerator.java

@@ -16,7 +16,7 @@ public static String generateConstructor(TypeParameterizedReactor tpr, String co
     code.pr(structType + "* new_" + CUtil.getName(tpr) + "() {");
     code.indent();
     code.pr(
-        structType + "* self = (" + structType + "*)_lf_new_reactor(sizeof(" + structType + "));");
+        structType + "* self = (" + structType + "*)lf_new_reactor(sizeof(" + structType + "));");
     code.pr(constructorCode);
     code.pr("return self;");
     code.unindent();

core/src/main/java/org/lflang/generator/c/CEnvironmentFunctionGenerator.java

@@ -78,7 +78,7 @@ private String generateEnvironmentEnum() {
   private String generateCreateEnvironments() {
     CodeBuilder code = new CodeBuilder();
     code.pr("// 'Create' and initialize the environments in the program");
-    code.pr("void _lf_create_environments() {");
+    code.pr("void lf_create_environments() {");
     code.indent();
     for (ReactorInstance enclave : enclaves) {
       // Decide the number of workers to use. If this is the top-level