diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index da1990ca3f..fce0ae6ea9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,6 @@ jobs: epoch: uses: lf-lang/epoch/.github/workflows/build.yml@main with: - lingua-franca-ref: ${{ github.head_ref || github.ref_name }} + lingua-franca-ref: pre-build-cmd # ${{ github.head_ref || github.ref_name }} lingua-franca-repo: ${{ github.event.pull_request.head.repo.full_name }} upload-artifacts: false diff --git a/core/build.gradle b/core/build.gradle index 2979ab494b..f35dcff7f8 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -23,6 +23,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-core:$fasterxmlVersion" implementation "com.fasterxml.jackson.core:jackson-annotations:$fasterxmlVersion" implementation "com.fasterxml.jackson.core:jackson-databind:$fasterxmlVersion" + implementation "org.apache.commons:commons-text:$commonsTextVersion" implementation ("de.cau.cs.kieler.klighd:de.cau.cs.kieler.klighd.lsp:$klighdVersion") { exclude group: 'org.eclipse.platform', module: 'org.eclipse.swt.*' diff --git a/core/src/main/java/org/lflang/analyses/uclid/UclidGenerator.java b/core/src/main/java/org/lflang/analyses/uclid/UclidGenerator.java index 540a2b5c81..9253856fcc 100644 --- a/core/src/main/java/org/lflang/analyses/uclid/UclidGenerator.java +++ b/core/src/main/java/org/lflang/analyses/uclid/UclidGenerator.java @@ -70,6 +70,7 @@ import org.lflang.generator.TargetTypes; import org.lflang.generator.TimerInstance; import org.lflang.generator.TriggerInstance; +import org.lflang.generator.docker.DockerGenerator; import org.lflang.lf.AttrParm; import org.lflang.lf.Attribute; import org.lflang.lf.Connection; @@ -1744,4 +1745,9 @@ public TargetTypes getTargetTypes() { throw new UnsupportedOperationException( "This method is not applicable for this generator since Uclid5 is not an LF target."); } + + @Override + protected DockerGenerator getDockerGenerator(LFGeneratorContext context) { + return null; + } } diff --git a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java index 2f4a4dfb0a..e1f36a9ad1 100644 --- a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java +++ b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java @@ -221,11 +221,7 @@ private void buildUsingDocker(LFGeneratorContext context, List subCo try { var dockerGen = new FedDockerComposeGenerator(context, rtiConfig.getHost()); dockerGen.writeDockerComposeFile(createDockerFiles(context, subContexts)); - if (dockerGen.build()) { - dockerGen.createLauncher(); - } else { - context.getErrorReporter().nowhere().error("Docker build failed."); - } + dockerGen.buildIfRequested(); } catch (IOException e) { context .getErrorReporter() @@ -279,6 +275,7 @@ private List createDockerFiles( var dockerData = dockerGenerator.generateDockerData(); try { dockerData.writeDockerFile(); + dockerData.copyScripts(context); } catch (IOException e) { throw new RuntimeIOException(e); } @@ -366,9 +363,7 @@ time allowed for the test expires (currently two hours). new TargetConfig( subFileConfig.resource, GeneratorArguments.none(), subContextMessageReporter); - if (targetConfig.get(DockerProperty.INSTANCE).enabled() - && targetConfig.target.buildsUsingDocker() - || fed.isRemote) { + if (targetConfig.get(DockerProperty.INSTANCE).enabled() || fed.isRemote) { NoCompileProperty.INSTANCE.override(subConfig, true); } // Disabled Docker for the federate and put federation in charge. diff --git a/core/src/main/java/org/lflang/generator/GeneratorBase.java b/core/src/main/java/org/lflang/generator/GeneratorBase.java index ea99722a9a..6911f5a4ed 100644 --- a/core/src/main/java/org/lflang/generator/GeneratorBase.java +++ b/core/src/main/java/org/lflang/generator/GeneratorBase.java @@ -51,6 +51,8 @@ import org.lflang.analyses.uclid.UclidGenerator; import org.lflang.ast.ASTUtils; import org.lflang.ast.AstTransformation; +import org.lflang.generator.docker.DockerComposeGenerator; +import org.lflang.generator.docker.DockerGenerator; import org.lflang.graph.InstantiationGraph; import org.lflang.lf.Attribute; import org.lflang.lf.Connection; @@ -627,6 +629,30 @@ protected void cleanIfNeeded(LFGeneratorContext context) { } } + /** Return a {@code DockerGenerator} instance suitable for the target. */ + protected abstract DockerGenerator getDockerGenerator(LFGeneratorContext context); + + /** Create Dockerfiles and docker-compose.yml, build, and create a launcher. */ + protected boolean buildUsingDocker() { + // Create docker file. + var dockerCompose = new DockerComposeGenerator(context); + var dockerData = getDockerGenerator(context).generateDockerData(); + try { + dockerData.writeDockerFile(); + dockerData.copyScripts(context); + dockerCompose.writeDockerComposeFile(List.of(dockerData)); + } catch (IOException e) { + context + .getErrorReporter() + .nowhere() + .error( + "Error while writing Docker files: " + + (e.getMessage() == null ? "No cause given" : e.getMessage())); + return false; + } + return dockerCompose.buildIfRequested(); + } + /** * Check if @property is used. If so, instantiate a UclidGenerator. The verification model needs * to be generated before the target code since code generation changes LF program (desugar diff --git a/core/src/main/java/org/lflang/generator/c/CGenerator.java b/core/src/main/java/org/lflang/generator/c/CGenerator.java index 298a637098..58624b353d 100644 --- a/core/src/main/java/org/lflang/generator/c/CGenerator.java +++ b/core/src/main/java/org/lflang/generator/c/CGenerator.java @@ -69,7 +69,6 @@ import org.lflang.generator.TimerInstance; import org.lflang.generator.TriggerInstance; import org.lflang.generator.docker.CDockerGenerator; -import org.lflang.generator.docker.DockerComposeGenerator; import org.lflang.generator.docker.DockerGenerator; import org.lflang.generator.python.PythonGenerator; import org.lflang.lf.Action; @@ -577,27 +576,6 @@ public void doGenerate(Resource resource, LFGeneratorContext context) { GeneratorUtils.refreshProject(resource, context.getMode()); } - /** Create Dockerfiles and docker-compose.yml, build, and create a launcher. */ - private boolean buildUsingDocker() { - // Create docker file. - var dockerCompose = new DockerComposeGenerator(context); - var dockerData = getDockerGenerator(context).generateDockerData(); - try { - dockerData.writeDockerFile(); - dockerCompose.writeDockerComposeFile(List.of(dockerData)); - } catch (IOException e) { - throw new RuntimeException("Error while writing Docker files", e); - } - var success = dockerCompose.build(); - if (!success) { - messageReporter.nowhere().error("Docker-compose build failed."); - } - if (success && mainDef != null) { - dockerCompose.createLauncher(); - } - return success; - } - private void generateCodeFor(String lfModuleName) throws IOException { code.pr(generateDirectives()); code.pr(new CMainFunctionGenerator(targetConfig).generateCode()); @@ -1966,6 +1944,7 @@ public TargetTypes getTargetTypes() { * @param context * @return */ + @Override protected DockerGenerator getDockerGenerator(LFGeneratorContext context) { return new CDockerGenerator(context); } diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index 41f459776c..afe67d8147 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -1,16 +1,18 @@ package org.lflang.generator.docker; -import org.eclipse.xtext.xbase.lib.IterableExtensions; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; import org.lflang.generator.LFGeneratorContext; import org.lflang.generator.c.CCompiler; +import org.lflang.generator.c.CFileConfig; import org.lflang.target.Target; -import org.lflang.target.property.BuildCommandsProperty; -import org.lflang.util.StringUtil; /** * Generate the docker file related code for the C and CCpp target. * * @author Hou Seng Wong + * @author Marten Lohstroh */ public class CDockerGenerator extends DockerGenerator { @@ -30,79 +32,47 @@ public String defaultImage() { return DEFAULT_BASE_IMAGE; } - /** Generate the contents of the docker file. */ @Override - protected String generateDockerFileContent() { - var lfModuleName = context.getFileConfig().name; - var config = context.getTargetConfig(); - var compileCommand = - IterableExtensions.isNullOrEmpty(config.get(BuildCommandsProperty.INSTANCE)) - ? generateCompileCommand() - : StringUtil.joinObjects(config.get(BuildCommandsProperty.INSTANCE), " "); - var baseImage = baseImage(); - return String.join( - "\n", - "# For instructions, see: https://www.lf-lang.org/docs/handbook/containerized-execution", - "FROM " + baseImage + " AS builder", - "WORKDIR /lingua-franca/" + lfModuleName, - generateRunForBuildDependencies(), - "COPY . src-gen", - compileCommand, - "", - "FROM " + baseImage, - "WORKDIR /lingua-franca", - "RUN mkdir bin", - "COPY --from=builder /lingua-franca/" - + lfModuleName - + "/bin/" - + lfModuleName - + " ./bin/" - + lfModuleName, - "", - "# Use ENTRYPOINT not CMD so that command-line arguments go through", - "ENTRYPOINT [\"./bin/" + lfModuleName + "\"]", - ""); + public List defaultEntryPoint() { + return List.of("./bin/" + context.getFileConfig().name); } @Override - protected String generateRunForBuildDependencies() { + protected String generateRunForInstallingDeps() { var config = context.getTargetConfig(); var compiler = config.target == Target.CCPP ? "g++" : "gcc"; - if (baseImage().equals(defaultImage())) { - return """ - # Install build dependencies - RUN set -ex && apk add --no-cache %s musl-dev cmake make - # Optional user specified run command - %s - """ - .formatted(compiler, userRunCommand()); + if (builderBase().equals(defaultImage())) { + return "RUN set -ex && apk add --no-cache %s musl-dev cmake make".formatted(compiler); } else { - return """ - # Optional user specified run command - %s - # Check for build dependencies - RUN which make && which cmake && which %s - """ - .formatted(userRunCommand(), compiler); + return "# (Skipping installation of build dependencies; custom base image.)"; } } - /** Return the default compile command for the C docker container. */ - protected String generateCompileCommand() { - var ccompile = - new CCompiler( - context.getTargetConfig(), - context.getFileConfig(), - context.getErrorReporter(), - context.getTargetConfig().target == Target.C); - return String.join( - "\n", - "RUN set -ex && \\", - "mkdir bin && \\", - String.format( - "%s -DCMAKE_INSTALL_BINDIR=./bin -S src-gen -B bin && \\", - ccompile.compileCmakeCommand()), - "cd bin && \\", - "make all"); + @Override + protected List defaultBuildCommands() { + try { + var ccompile = + new CCompiler( + context.getTargetConfig(), + new CFileConfig( + context.getFileConfig().resource, + Path.of("/lingua-franca", context.getFileConfig().name), + false), + context.getErrorReporter(), + context.getTargetConfig().target == Target.CCPP); + return List.of( + "mkdir -p bin", + String.format( + "%s -DCMAKE_INSTALL_BINDIR=./bin -S src-gen -B bin", ccompile.compileCmakeCommand()), + "cd bin", + "make all", + "cd .."); + } catch (IOException e) { + context + .getErrorReporter() + .nowhere() + .error("Unable to create file configuration for Docker container"); + throw new RuntimeException(e); + } } } diff --git a/core/src/main/java/org/lflang/generator/docker/DockerComposeGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerComposeGenerator.java index 9b6f684f75..2255f3ce3b 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerComposeGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerComposeGenerator.java @@ -7,6 +7,7 @@ import java.util.Objects; import java.util.stream.Collectors; import org.lflang.generator.LFGeneratorContext; +import org.lflang.target.property.DockerProperty; import org.lflang.util.FileUtil; import org.lflang.util.LFCommand; @@ -46,7 +47,6 @@ protected String generateDockerNetwork(String networkName) { */ protected String generateDockerServices(List services) { return """ - version: "3.9" services: %s """ @@ -153,4 +153,19 @@ public void createLauncher() { messageReporter.nowhere().warning("Unable to make launcher script executable."); } } + + /** + * Build, unless building was disabled. + * + * @return {@code false} if building failed, {@code true} otherwise + */ + public boolean buildIfRequested() { + if (!context.getTargetConfig().get(DockerProperty.INSTANCE).noBuild()) { + if (build()) { + createLauncher(); + } else context.getErrorReporter().nowhere().error("Docker build failed."); + return false; + } + return true; + } } diff --git a/core/src/main/java/org/lflang/generator/docker/DockerData.java b/core/src/main/java/org/lflang/generator/docker/DockerData.java index 7891eb74d6..75338630b5 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerData.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerData.java @@ -3,7 +3,10 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import org.lflang.FileConfig; import org.lflang.generator.LFGeneratorContext; +import org.lflang.target.property.DockerProperty; import org.lflang.util.FileUtil; /** @@ -44,4 +47,26 @@ public void writeDockerFile() throws IOException { FileUtil.writeToFile(dockerFileContent, dockerFilePath); context.getErrorReporter().nowhere().info("Dockerfile written to " + dockerFilePath); } + + /** Copy the pre-build, post-build, and pre-run scripts, if specified */ + public void copyScripts(LFGeneratorContext context) throws IOException { + var prop = context.getTargetConfig().get(DockerProperty.INSTANCE); + copyScripts( + context.getFileConfig(), + List.of(prop.preBuildScript(), prop.postBuildScript(), prop.preRunScript())); + } + + /** Copy the given list of scripts */ + private void copyScripts(FileConfig fileConfig, List scripts) throws IOException { + for (var script : scripts) { + if (!script.isEmpty()) { + var found = FileUtil.findInPackage(Path.of(script), fileConfig); + if (found != null) { + var destination = dockerFilePath.getParent().resolve(found.getFileName()); + FileUtil.copyFile(found, destination); + this.context.getErrorReporter().nowhere().info("Script written to " + destination); + } + } + } + } } diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index a9a7b71d52..40f9fa7f37 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -1,8 +1,17 @@ package org.lflang.generator.docker; import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.text.StringEscapeUtils; +import org.lflang.LocalStrings; import org.lflang.generator.LFGeneratorContext; +import org.lflang.generator.SubContext; +import org.lflang.target.property.BuildCommandsProperty; import org.lflang.target.property.DockerProperty; +import org.lflang.target.property.DockerProperty.DockerOptions; +import org.lflang.util.StringUtil; /** * A class for generating docker files. @@ -24,31 +33,179 @@ public DockerGenerator(LFGeneratorContext context) { this.context = context; } - /** Generate the contents of a Dockerfile. */ - protected abstract String generateDockerFileContent(); + /** Generate the contents of the docker file. */ + protected String generateDockerFileContent() { + var lfModuleName = context.getFileConfig().name; + return String.join( + "\n", + generateHeader(), + "FROM " + builderBase() + " AS builder", + "WORKDIR /lingua-franca/" + lfModuleName, + generateRunForInstallingDeps(), + generateCopyForSources(), + generateRunForBuild(), + "", + "FROM " + runnerBase(), + "WORKDIR /lingua-franca", + "RUN mkdir scripts", + generateCopyOfScript(), + generateRunForMakingExecutableDir(), + generateCopyOfExecutable(), + generateEntryPoint(), + ""); + } + + /** Return a RUN command for making a directory to place executables in. */ + protected String generateRunForMakingExecutableDir() { + return "RUN mkdir bin"; + } + + /** Return a COPY command for copying sources from host into container. */ + protected String generateCopyForSources() { + return "COPY . src-gen"; + } /** Return a RUN command for installing/checking build dependencies. */ - protected abstract String generateRunForBuildDependencies(); + protected abstract String generateRunForInstallingDeps(); - /** Return the default base image. */ - public abstract String defaultImage(); + /** Return the default compile commands for the C docker container. */ + protected abstract List defaultBuildCommands(); - /** Return the selected base image, or the default one if none was selected. */ - public String baseImage() { - var baseImage = context.getTargetConfig().get(DockerProperty.INSTANCE).from(); - if (baseImage != null && !baseImage.isEmpty()) { - return baseImage; + /** Return the commands used to build */ + protected List getBuildCommands() { + if (context.getTargetConfig().isSupported(BuildCommandsProperty.INSTANCE)) { + var customBuildCommands = context.getTargetConfig().get(BuildCommandsProperty.INSTANCE); + if (customBuildCommands != null && !customBuildCommands.isEmpty()) { + return customBuildCommands; + } + } + return defaultBuildCommands(); + } + + /** Return the command that sources the pre-build script, if there is one. */ + protected List getPreBuildCommand() { + var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildScript(); + if (!script.isEmpty()) { + return List.of("source src-gen/" + StringEscapeUtils.escapeXSI(script)); + } + return List.of(); + } + + /** Return the command that sources the post-build script, if there is one. */ + protected List getPostBuildCommand() { + var script = context.getTargetConfig().get(DockerProperty.INSTANCE).postBuildScript(); + if (!script.isEmpty()) { + return List.of("source src-gen/" + StringEscapeUtils.escapeXSI(script)); + } + return List.of(); + } + + /** Generate a header to print at the top of the Dockerfile. */ + protected String generateHeader() { + return """ + # Generated by the Lingua Franca compiler version %s + # - Docs: https://www.lf-lang.org/docs/handbook/containerized-execution" + """ + .formatted(LocalStrings.VERSION); + } + + /** Return the Docker RUN command used for building. */ + protected String generateRunForBuild() { + return "RUN " + + StringUtil.joinObjects( + Stream.of( + List.of("set -ex"), + getPreBuildCommand(), + getBuildCommands(), + getPostBuildCommand()) + .flatMap(java.util.Collection::stream) + .collect(Collectors.toList()), + " \\\n\t&& "); + } + + /** Return the ENTRYPOINT command. */ + protected String generateEntryPoint() { + return "ENTRYPOINT [" + + getEntryPointCommands().stream() + .map(cmd -> "\"" + cmd + "\"") + .collect(Collectors.joining(",")) + + "]"; + } + + /** Return a COPY command to copy the executable from the builder to the runner. */ + protected String generateCopyOfExecutable() { + var lfModuleName = context.getFileConfig().name; + // safe because context.getFileConfig().name never contains spaces + return "COPY --from=builder /lingua-franca/%s/bin/%s ./bin/%s" + .formatted(lfModuleName, lfModuleName, lfModuleName); + } + + /** Return a COPY command to copy the scripts from the builder to the runner. */ + protected String generateCopyOfScript() { + var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preRunScript(); + if (!script.isEmpty()) { + return "COPY --from=builder /lingua-franca/%s/src-gen/%s ./scripts/" + .formatted(context.getFileConfig().name, StringEscapeUtils.escapeXSI(script)); } - return defaultImage(); + return "# (No pre-run script provided.)"; } - public String userRunCommand() { - var runCmd = context.getTargetConfig().get(DockerProperty.INSTANCE).run(); - if (runCmd != null && !runCmd.isEmpty()) { - return "RUN " + runCmd; + /** + * Return a list of strings used to construct and entrypoint. If this is done for a federate, then + * also include additional parameters to pass in the federation ID. + */ + protected List entryPoint() { + if (context instanceof SubContext) { + return Stream.concat(defaultEntryPoint().stream(), List.of("-i", "1").stream()).toList(); } else { - return ""; + return defaultEntryPoint(); + } + } + + /** + * Return a list of commands to be used to construct an ENTRYPOINT, taking into account the + * existence of a possible pre-run script. + */ + protected final List getEntryPointCommands() { + var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preRunScript(); + if (!script.isEmpty()) { + return List.of( + DockerOptions.DEFAULT_SHELL, + "-c", + "source scripts/" + + StringEscapeUtils.escapeXSI(script) + + " && " + + entryPoint().stream().collect(Collectors.joining(" "))); + } + return entryPoint(); + } + + /** The default list of commands to construct an ENTRYPOINT out of. Different for each target. */ + public abstract List defaultEntryPoint(); + + /** Return the default base image. */ + public abstract String defaultImage(); + + /** Return the base image to be used during the building stage. */ + protected String builderBase() { + return baseImage( + context.getTargetConfig().get(DockerProperty.INSTANCE).builderBase(), defaultImage()); + } + + /** Return the base image to be used during the running stage. */ + protected String runnerBase() { + return baseImage( + context.getTargetConfig().get(DockerProperty.INSTANCE).runnerBase(), + baseImage( + context.getTargetConfig().get(DockerProperty.INSTANCE).builderBase(), defaultImage())); + } + + /** Return the selected base image, or the default one if none was selected. */ + private String baseImage(String name, String defaultImage) { + if (name != null && !name.isEmpty()) { + return name; } + return defaultImage; } /** @@ -82,4 +239,12 @@ public static DockerGenerator dockerGeneratorFactory(LFGeneratorContext context) throw new IllegalArgumentException("No Docker support for " + target + " yet."); }; } + + /** + * Convert an argument list, starting with the command to execute, into a string that can be + * executed by a POSIX-compliant shell. + */ + public static String argListToCommand(List args) { + return args.stream().map(it -> "\"" + it + "\"").collect(Collectors.joining(" ")); + } } diff --git a/core/src/main/java/org/lflang/generator/docker/FedDockerComposeGenerator.java b/core/src/main/java/org/lflang/generator/docker/FedDockerComposeGenerator.java index 1e5deb3860..2eb298b2de 100644 --- a/core/src/main/java/org/lflang/generator/docker/FedDockerComposeGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/FedDockerComposeGenerator.java @@ -65,7 +65,6 @@ protected String generateDockerServices(List services) { protected String getServiceDescription(DockerData data) { return """ %s\ - command: "-i 1" depends_on: - rti """ diff --git a/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java index 371c880771..6a55669752 100644 --- a/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java @@ -1,5 +1,6 @@ package org.lflang.generator.docker; +import java.util.List; import org.lflang.generator.LFGeneratorContext; /** @@ -20,32 +21,25 @@ public String defaultImage() { } @Override - protected String generateRunForBuildDependencies() { - if (baseImage().equals(defaultImage())) { - return """ - # Install build dependencies - RUN set -ex && apt-get update && apt-get install -y python3-pip && pip install cmake - """; + protected String generateRunForInstallingDeps() { + if (builderBase().equals(defaultImage())) { + return "RUN set -ex && apt-get update && apt-get install -y python3-pip && pip install cmake"; } else { - return """ - # Check for build dependencies - RUN which make && which cmake && which gcc - """; + return "# (Skipping installation of build dependencies; custom base image.)"; } } - /** Generates the contents of the docker file. */ @Override - protected String generateDockerFileContent() { + protected String generateCopyOfExecutable() { + var lfModuleName = context.getFileConfig().name; return String.join( "\n", - "# For instructions, see:" - + " https://www.lf-lang.org/docs/handbook/containerized-execution?target=py", - "FROM " + baseImage(), - "WORKDIR /lingua-franca/" + context.getFileConfig().name, - generateRunForBuildDependencies(), - "COPY . src-gen", - super.generateCompileCommand(), - "ENTRYPOINT [\"python3\", \"-u\", \"src-gen/" + context.getFileConfig().name + ".py\"]"); + super.generateCopyOfExecutable(), + "COPY --from=builder /lingua-franca/%s/src-gen ./src-gen".formatted(lfModuleName)); + } + + @Override + public List defaultEntryPoint() { + return List.of("python3", "-u", "src-gen/" + context.getFileConfig().name + ".py"); } } diff --git a/core/src/main/java/org/lflang/generator/docker/RtiDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/RtiDockerGenerator.java index d281cc896c..c936e529ae 100644 --- a/core/src/main/java/org/lflang/generator/docker/RtiDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/RtiDockerGenerator.java @@ -28,7 +28,7 @@ protected String generateDockerFileContent() { } @Override - public String baseImage() { + public String builderBase() { return defaultImage(); } } diff --git a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java index 5fae515778..1e81439886 100644 --- a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java @@ -1,11 +1,13 @@ package org.lflang.generator.docker; +import java.util.List; import org.lflang.generator.LFGeneratorContext; /** * Generates the docker file related code for the Typescript target. * * @author Hou Seng Wong + * @author Marten Lohstroh */ public class TSDockerGenerator extends DockerGenerator { @@ -14,21 +16,35 @@ public TSDockerGenerator(LFGeneratorContext context) { super(context); } - /** Return the content of the docker file for [tsFileName]. */ - public String generateDockerFileContent() { - return """ - FROM %s - WORKDIR /linguafranca/$name - %s - COPY . . - ENTRYPOINT ["node", "dist/%s.js"] - """ - .formatted(baseImage(), generateRunForBuildDependencies(), context.getFileConfig().name); + @Override + protected String generateCopyOfExecutable() { + var lfModuleName = context.getFileConfig().name; + return "COPY --from=builder /lingua-franca/%s .".formatted(lfModuleName); + } + + @Override + protected String generateRunForMakingExecutableDir() { + return "RUN mkdir dist"; + } + + @Override + protected String generateCopyForSources() { + return "COPY . ."; + } + + @Override + public List defaultEntryPoint() { + return List.of("node", "dist/%s.js".formatted(context.getFileConfig().name)); + } + + @Override + protected String generateRunForInstallingDeps() { + return "RUN apk add git && npm install -g pnpm"; } @Override - protected String generateRunForBuildDependencies() { - return "RUN which node && node --version"; + protected List defaultBuildCommands() { + return List.of("pnpm install", "pnpm run build"); } @Override diff --git a/core/src/main/java/org/lflang/target/Target.java b/core/src/main/java/org/lflang/target/Target.java index 6467023621..30048f5a58 100644 --- a/core/src/main/java/org/lflang/target/Target.java +++ b/core/src/main/java/org/lflang/target/Target.java @@ -497,14 +497,6 @@ public boolean supportsReactionDeclarations() { return this.equals(Target.C) || this.equals(Target.CPP); } - /** Return true if this code for this target should be built using Docker if Docker is used. */ - public boolean buildsUsingDocker() { - return switch (this) { - case TS -> false; - case C, CCPP, CPP, Python, Rust -> true; - }; - } - /** * Whether the target requires using an equal sign to assign a default value to a parameter, or * initialize a state variable. All targets mandate an equal sign when passing arguments to a @@ -618,6 +610,7 @@ public void initialize(TargetConfig config) { BuildTypeProperty.INSTANCE, CmakeIncludeProperty.INSTANCE, CompilerProperty.INSTANCE, + DockerProperty.INSTANCE, ExportDependencyGraphProperty.INSTANCE, ExportToYamlProperty.INSTANCE, ExternalRuntimePathProperty.INSTANCE, diff --git a/core/src/main/java/org/lflang/target/TargetConfig.java b/core/src/main/java/org/lflang/target/TargetConfig.java index f3dc1695f6..a5ce42c6cc 100644 --- a/core/src/main/java/org/lflang/target/TargetConfig.java +++ b/core/src/main/java/org/lflang/target/TargetConfig.java @@ -170,6 +170,11 @@ protected void load(JsonObject jsonObject, MessageReporter messageReporter) { } } + /** Return {@code true} if the given target property is supported, {@code false} otherwise. */ + public boolean isSupported(TargetProperty p) { + return properties.containsKey(p); + } + /** * Report that a target property is not supported by the current target. * diff --git a/core/src/main/java/org/lflang/target/property/DockerProperty.java b/core/src/main/java/org/lflang/target/property/DockerProperty.java index c58e2bebf9..2aa539207e 100644 --- a/core/src/main/java/org/lflang/target/property/DockerProperty.java +++ b/core/src/main/java/org/lflang/target/property/DockerProperty.java @@ -34,9 +34,14 @@ public DockerOptions initialValue() { @Override public DockerOptions fromAst(Element node, MessageReporter reporter) { var enabled = false; - var from = ""; - var run = ""; + var noBuild = false; + var builderBase = ""; + var runnerBase = ""; var rti = DockerOptions.DOCKERHUB_RTI_IMAGE; + var shell = DockerOptions.DEFAULT_SHELL; + var preBuildScript = ""; + var postBuildScript = ""; + var runScript = ""; if (node.getLiteral() != null) { if (ASTUtils.toBoolean(node)) { @@ -47,13 +52,28 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { for (KeyValuePair entry : node.getKeyvalue().getPairs()) { DockerOption option = (DockerOption) DictionaryType.DOCKER_DICT.forName(entry.getName()); switch (option) { - case FROM -> from = ASTUtils.elementToSingleString(entry.getValue()); - case RUN -> run = ASTUtils.elementToSingleString(entry.getValue()); + case NO_BUILD -> noBuild = ASTUtils.toBoolean(entry.getValue()); + case BUILDER_BASE -> builderBase = ASTUtils.elementToSingleString(entry.getValue()); + case RUNNER_BASE -> runnerBase = ASTUtils.elementToSingleString(entry.getValue()); + case PRE_BUILD_SCRIPT -> + preBuildScript = ASTUtils.elementToSingleString(entry.getValue()); + case PRE_RUN_SCRIPT -> runScript = ASTUtils.elementToSingleString(entry.getValue()); + case POST_BUILD_SCRIPT -> + postBuildScript = ASTUtils.elementToSingleString(entry.getValue()); case RTI_IMAGE -> rti = ASTUtils.elementToSingleString(entry.getValue()); } } } - return new DockerOptions(enabled, from, run, rti); + return new DockerOptions( + enabled, + noBuild, + builderBase, + runnerBase, + rti, + shell, + preBuildScript, + postBuildScript, + runScript); } @Override @@ -82,8 +102,12 @@ public Element toAstElement(DockerOptions value) { KeyValuePair pair = LfFactory.eINSTANCE.createKeyValuePair(); pair.setName(opt.toString()); switch (opt) { - case FROM -> pair.setValue(ASTUtils.toElement(value.from)); - case RUN -> pair.setValue(ASTUtils.toElement(value.run)); + case NO_BUILD -> pair.setValue(ASTUtils.toElement(value.noBuild)); + case BUILDER_BASE -> pair.setValue(ASTUtils.toElement(value.builderBase)); + case RUNNER_BASE -> pair.setValue(ASTUtils.toElement(value.runnerBase)); + case PRE_BUILD_SCRIPT -> pair.setValue(ASTUtils.toElement(value.preBuildScript)); + case PRE_RUN_SCRIPT -> pair.setValue(ASTUtils.toElement(value.preRunScript)); + case POST_BUILD_SCRIPT -> pair.setValue(ASTUtils.toElement(value.postBuildScript)); case RTI_IMAGE -> pair.setValue(ASTUtils.toElement(value.rti)); } kvp.getPairs().add(pair); @@ -102,16 +126,27 @@ public String name() { } /** Settings related to Docker options. */ - public record DockerOptions(boolean enabled, String from, String run, String rti) { + public record DockerOptions( + boolean enabled, + boolean noBuild, + String builderBase, + String runnerBase, + String rti, + String shell, + String preBuildScript, + String postBuildScript, + String preRunScript) { /** Default location to pull the rti from. */ public static final String DOCKERHUB_RTI_IMAGE = "lflang/rti:rti"; + public static final String DEFAULT_SHELL = "/bin/sh"; + /** String to indicate a local build of the rti. */ public static final String LOCAL_RTI_IMAGE = "rti:local"; public DockerOptions(boolean enabled) { - this(enabled, "", "", DOCKERHUB_RTI_IMAGE); + this(enabled, false, "", "", DOCKERHUB_RTI_IMAGE, DEFAULT_SHELL, "", "", ""); } } @@ -121,9 +156,13 @@ public DockerOptions(boolean enabled) { * @author Edward A. Lee */ public enum DockerOption implements DictionaryElement { - FROM("FROM", PrimitiveType.STRING), - RUN("RUN", PrimitiveType.STRING), - RTI_IMAGE("rti-image", PrimitiveType.STRING); + NO_BUILD("no-build", PrimitiveType.BOOLEAN), + BUILDER_BASE("builder-base", PrimitiveType.STRING), + RUNNER_BASE("runner-base", PrimitiveType.STRING), + RTI_IMAGE("rti-image", PrimitiveType.STRING), + PRE_BUILD_SCRIPT("pre-build-script", PrimitiveType.STRING), + PRE_RUN_SCRIPT("pre-run-script", PrimitiveType.STRING), + POST_BUILD_SCRIPT("post-build-script", PrimitiveType.STRING); public final PrimitiveType type; diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt index 204b6e9324..adfbbb9dda 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt @@ -27,14 +27,16 @@ package org.lflang.generator.cpp import org.eclipse.emf.ecore.resource.Resource -import org.lflang.target.Target import org.lflang.generator.* import org.lflang.generator.GeneratorUtils.canGenerate import org.lflang.generator.LFGeneratorContext.Mode +import org.lflang.generator.docker.DockerGenerator import org.lflang.isGeneric import org.lflang.scoping.LFGlobalScopeProvider +import org.lflang.target.Target import org.lflang.target.property.* import org.lflang.util.FileUtil +import java.io.IOException import java.nio.file.Files import java.nio.file.Path @@ -58,6 +60,7 @@ class CppGenerator( const val MINIMUM_CMAKE_VERSION = "3.5" const val CPP_VERSION = "20" + } override fun doGenerate(resource: Resource, context: LFGeneratorContext) { @@ -66,8 +69,7 @@ class CppGenerator( if (!canGenerate(errorsOccurred(), mainDef, messageReporter, context)) return // create a platform-specific generator - val platformGenerator: CppPlatformGenerator = - if (targetConfig.get(Ros2Property.INSTANCE)) CppRos2Generator(this) else CppStandaloneGenerator(this) + val platformGenerator: CppPlatformGenerator = getPlatformGenerator() // generate all core files generateFiles(platformGenerator.srcGenPath, getAllImportedResources(resource)) @@ -82,7 +84,6 @@ class CppGenerator( context.reportProgress( "Code generation complete. Validating generated code...", IntegratedBuilder.GENERATED_PERCENT_PROGRESS ) - if (platformGenerator.doCompile(context)) { CppValidator(fileConfig, messageReporter, codeMaps).doValidate(context) context.finish(GeneratorResult.GENERATED_NO_EXECUTABLE.apply(context, codeMaps)) @@ -93,11 +94,45 @@ class CppGenerator( context.reportProgress( "Code generation complete. Compiling...", IntegratedBuilder.GENERATED_PERCENT_PROGRESS ) - if (platformGenerator.doCompile(context)) { - context.finish(GeneratorResult.Status.COMPILED, codeMaps) + if (targetConfig.get(DockerProperty.INSTANCE).enabled) { + copySrcGenBaseDirIntoDockerDir() + buildUsingDocker() } else { - context.unsuccessfulFinish() + if (platformGenerator.doCompile(context)) { + context.finish(GeneratorResult.Status.COMPILED, codeMaps) + } else { + context.unsuccessfulFinish() + } + } + } + } + + /** + * Copy the contents of the entire src-gen directory to a nested src-gen directory next to the generated Dockerfile. + */ + private fun copySrcGenBaseDirIntoDockerDir() { + FileUtil.deleteDirectory(context.fileConfig.srcGenPath.resolve("src-gen")) + try { + // We need to copy in two steps via a temporary directory, as the target directory + // is located within the source directory. Without the temporary directory, copying + // fails as we modify the source while writing the target. + val tempDir = Files.createTempDirectory(context.fileConfig.outPath, "src-gen-directory") + try { + FileUtil.copyDirectoryContents(context.fileConfig.srcGenBasePath, tempDir, false) + FileUtil.copyDirectoryContents(tempDir, context.fileConfig.srcGenPath.resolve("src-gen"), false) + } catch (e: IOException) { + context.errorReporter.nowhere() + .error("Failed to copy sources to make them accessible to Docker: " + if (e.message == null) "No cause given" else e.message) + e.printStackTrace() + } finally { + FileUtil.deleteDirectory(tempDir) + } + if (errorsOccurred()) { + return } + } catch (e: IOException) { + context.errorReporter.nowhere().error("Failed to create temporary directory.") + e.printStackTrace() } } @@ -183,8 +218,11 @@ class CppGenerator( } } + private fun getPlatformGenerator() = if (targetConfig.get(Ros2Property.INSTANCE)) CppRos2Generator(this) else CppStandaloneGenerator(this) + override fun getTarget() = Target.CPP override fun getTargetTypes(): TargetTypes = CppTypes -} + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator = getPlatformGenerator().getDockerGenerator(context) +} diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt index 220770b6a9..799d0125c2 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt @@ -4,6 +4,7 @@ import org.lflang.MessageReporter import org.lflang.target.TargetConfig import org.lflang.generator.GeneratorCommandFactory import org.lflang.generator.LFGeneratorContext +import org.lflang.generator.docker.DockerGenerator import org.lflang.target.property.BuildTypeProperty import org.lflang.target.property.LoggingProperty import org.lflang.target.property.NoRuntimeValidationProperty @@ -37,4 +38,6 @@ abstract class CppPlatformGenerator(protected val generator: CppGenerator) { "-DREACTOR_CPP_LOG_LEVEL=${targetConfig.get(LoggingProperty.INSTANCE).severity}", "-DLF_SRC_PKG_PATH=${fileConfig.srcPkgPath}", ) + + abstract fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator } diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt index 76969269c0..4be02961ef 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt @@ -1,6 +1,9 @@ package org.lflang.generator.cpp import org.lflang.generator.LFGeneratorContext +import org.lflang.generator.docker.DockerGenerator +import org.lflang.target.property.DockerProperty +import org.lflang.toUnixString import org.lflang.util.FileUtil import java.nio.file.Path @@ -12,6 +15,10 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator private val nodeGenerator = CppRos2NodeGenerator(mainReactor, targetConfig, fileConfig); private val packageGenerator = CppRos2PackageGenerator(generator, nodeGenerator.nodeName) + companion object { + const val DEFAULT_BASE_IMAGE: String = "ros:rolling-ros-base" + } + override fun generatePlatformFiles() { FileUtil.writeToFile( nodeGenerator.generateHeader(), @@ -30,7 +37,11 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator packagePath.resolve("CMakeLists.txt"), true ) - val scriptPath = fileConfig.binPath.resolve(fileConfig.name); + val scriptPath = + if (targetConfig.get(DockerProperty.INSTANCE).enabled) + fileConfig.srcGenPath.resolve("bin").resolve(fileConfig.name) + else + fileConfig.binPath.resolve(fileConfig.name) FileUtil.writeToFile(packageGenerator.generateBinScript(), scriptPath) scriptPath.toFile().setExecutable(true); } @@ -46,24 +57,57 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator ) return false } - val colconCommand = commandFactory.createCommand( - "colcon", listOf( + "colcon", colconArgs(), fileConfig.outPath) + val returnCode = colconCommand?.run(context.cancelIndicator) + if (returnCode != 0 && !messageReporter.errorsOccurred) { + // If errors occurred but none were reported, then the following message is the best we can do. + messageReporter.nowhere().error("colcon failed with error code $returnCode") + } + + return !messageReporter.errorsOccurred + } + + private fun colconArgs(): List { + return listOf( "build", "--packages-select", fileConfig.name, packageGenerator.reactorCppName, "--cmake-args", "-DLF_REACTOR_CPP_SUFFIX=${packageGenerator.reactorCppSuffix}", - ) + cmakeArgs, - fileConfig.outPath - ) - val returnCode = colconCommand?.run(context.cancelIndicator); - if (returnCode != 0 && !messageReporter.errorsOccurred) { - // If errors occurred but none were reported, then the following message is the best we can do. - messageReporter.nowhere().error("colcon failed with error code $returnCode") - } + ) + cmakeArgs + } - return !messageReporter.errorsOccurred + inner class CppDockerGenerator(context: LFGeneratorContext?) : DockerGenerator(context) { + override fun generateCopyForSources() = + """ + COPY src-gen src-gen + COPY bin bin + """.trimIndent() + + override fun defaultImage(): String = DEFAULT_BASE_IMAGE + + override fun generateRunForInstallingDeps(): String = "" + + override fun defaultEntryPoint(): List = listOf(fileConfig.outPath.relativize(fileConfig.binPath).toUnixString() + "/" + fileConfig.name) + + override fun generateCopyOfExecutable(): String = + """ + ${super.generateCopyOfExecutable()} + COPY --from=builder lingua-franca/${fileConfig.name}/install install + """.trimIndent() + + override fun defaultBuildCommands(): List { + val commands = listOf( + listOf(".", "/opt/ros/rolling/setup.sh"), + listOf("mkdir", "-p", "build"), + listOf("colcon") + colconArgs(), + ) + return commands.map { argListToCommand(it) } + } } + + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator = CppDockerGenerator(context) + } diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2PackageGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2PackageGenerator.kt index ea7568e2c4..c3ce84c747 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2PackageGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2PackageGenerator.kt @@ -113,7 +113,7 @@ class CppRos2PackageGenerator(generator: CppGenerator, private val nodeName: Str return """ |#!/bin/bash |script_dir="$S(dirname -- "$S(readlink -f -- "${S}0")")" - |source "$S{script_dir}/$relPath/install/setup.sh" + |source "$S{script_dir}/$relPath/install/setup.bash" |ros2 run ${fileConfig.name} ${fileConfig.name}_exe """.trimMargin() } diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneCmakeGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneCmakeGenerator.kt index 2c041fe6d2..5d36e8e327 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneCmakeGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneCmakeGenerator.kt @@ -153,6 +153,8 @@ class CppStandaloneCmakeGenerator(private val targetConfig: TargetConfig, privat |cmake_minimum_required(VERSION 3.5) |project(${fileConfig.name} VERSION 0.0.0 LANGUAGES CXX) | + |option(REACTOR_CPP_LINK_EXECINFO "Link against execinfo" OFF) + | |${if (targetConfig.get(ExternalRuntimePathProperty.INSTANCE) != null) "find_package(reactor-cpp PATHS ${targetConfig.get(ExternalRuntimePathProperty.INSTANCE)})" else ""} | |set(LF_MAIN_TARGET ${fileConfig.name}) @@ -167,6 +169,10 @@ class CppStandaloneCmakeGenerator(private val targetConfig: TargetConfig, privat |) |target_link_libraries($S{LF_MAIN_TARGET} $reactorCppTarget) | + |if(REACTOR_CPP_LINK_EXECINFO) + | target_link_libraries($S{LF_MAIN_TARGET} execinfo) + |endif() + | |if(MSVC) | target_compile_options($S{LF_MAIN_TARGET} PRIVATE /W4) |else() diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt index dd03db376d..64fdf32ee0 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt @@ -2,6 +2,7 @@ package org.lflang.generator.cpp import org.lflang.generator.CodeMap import org.lflang.generator.LFGeneratorContext +import org.lflang.generator.docker.DockerGenerator import org.lflang.target.property.BuildTypeProperty import org.lflang.target.property.CompilerProperty import org.lflang.target.property.type.BuildTypeType.BuildType @@ -16,6 +17,15 @@ import java.nio.file.Paths class CppStandaloneGenerator(generator: CppGenerator) : CppPlatformGenerator(generator) { + companion object { + fun buildTypeToCmakeConfig(type: BuildType) = when (type) { + BuildType.TEST -> "Debug" + else -> type.toString() + } + + const val DEFAULT_BASE_IMAGE: String = "alpine:latest" + } + override fun generatePlatformFiles() { // generate the main source file (containing main()) @@ -62,16 +72,22 @@ class CppStandaloneGenerator(generator: CppGenerator) : Files.createDirectories(fileConfig.buildPath) val version = checkCmakeVersion() + var parallelize = true + if (version != null && version.compareVersion("3.12.0") < 0) { + messageReporter.nowhere().warning("CMAKE is older than version 3.12. Parallel building is not supported.") + parallelize = false + } + if (version != null) { val cmakeReturnCode = runCmake(context) if (cmakeReturnCode == 0 && runMake) { // If cmake succeeded, run make - val makeCommand = createMakeCommand(fileConfig.buildPath, version, fileConfig.name) + val makeCommand = createMakeCommand(fileConfig.buildPath, parallelize, fileConfig.name) val makeReturnCode = CppValidator(fileConfig, messageReporter, codeMaps).run(makeCommand, context.cancelIndicator) var installReturnCode = 0 if (makeReturnCode == 0) { - val installCommand = createMakeCommand(fileConfig.buildPath, version, "install") + val installCommand = createMakeCommand(fileConfig.buildPath, parallelize, "install") installReturnCode = installCommand.run(context.cancelIndicator) if (installReturnCode == 0) { println("SUCCESS (compiling generated C++ code)") @@ -132,43 +148,43 @@ class CppStandaloneGenerator(generator: CppGenerator) : return 0 } - private fun buildTypeToCmakeConfig(type: BuildType) = when (type) { - BuildType.TEST -> "Debug" - else -> type.toString() - } - - private fun createMakeCommand(buildPath: Path, version: String, target: String): LFCommand { - val makeArgs: List - if (version.compareVersion("3.12.0") < 0) { - messageReporter.nowhere().warning("CMAKE is older than version 3.12. Parallel building is not supported.") - makeArgs = - listOf("--build", ".", "--target", target, "--config", buildTypeToCmakeConfig(targetConfig.get(BuildTypeProperty.INSTANCE))) - } else { - val cores = Runtime.getRuntime().availableProcessors() - makeArgs = listOf( - "--build", - ".", - "--target", - target, - "--parallel", - cores.toString(), - "--config", - buildTypeToCmakeConfig(targetConfig.get(BuildTypeProperty.INSTANCE)) - ) + private fun createMakeCommand(buildPath: Path, parallelize: Boolean, target: String): LFCommand { + val cmakeConfig = buildTypeToCmakeConfig(targetConfig.get(BuildTypeProperty.INSTANCE)) + val makeArgs: MutableList = listOf( + "--build", + buildPath.fileName.toString(), + "--target", + target, + "--config", + cmakeConfig + ).toMutableList() + + if (parallelize) { + makeArgs.addAll(listOf("--parallel", Runtime.getRuntime().availableProcessors().toString())) } - return commandFactory.createCommand("cmake", makeArgs, buildPath) + return commandFactory.createCommand("cmake", makeArgs, buildPath.parent) } - private fun createCmakeCommand(buildPath: Path, outPath: Path): LFCommand { + private fun createCmakeCommand( + buildPath: Path, + outPath: Path, + additionalCmakeArgs: List = listOf(), + sourcesRoot: String? = null + ): LFCommand { val cmd = commandFactory.createCommand( "cmake", - cmakeArgs + listOf( + cmakeArgs + additionalCmakeArgs + listOf( "-DCMAKE_INSTALL_PREFIX=${outPath.toUnixString()}", - "-DCMAKE_INSTALL_BINDIR=${outPath.relativize(fileConfig.binPath).toUnixString()}", - fileConfig.srcGenBasePath.toUnixString() + "-DCMAKE_INSTALL_BINDIR=${ + if (outPath.isAbsolute) outPath.relativize(fileConfig.binPath).toUnixString() else fileConfig.binPath.fileName.toString() + }", + "-S", + sourcesRoot ?: fileConfig.srcGenBasePath.toUnixString(), + "-B", + buildPath.fileName.toString() ), - buildPath + buildPath.parent ) // prepare cmake @@ -177,4 +193,49 @@ class CppStandaloneGenerator(generator: CppGenerator) : } return cmd } + + inner class StandaloneDockerGenerator(context: LFGeneratorContext?) : DockerGenerator(context) { + + override fun generateCopyForSources(): String = "COPY src-gen src-gen" + + override fun defaultImage(): String = DEFAULT_BASE_IMAGE + + override fun generateRunForInstallingDeps(): String { + return if (builderBase() == defaultImage()) { + ("RUN set -ex && apk add --no-cache g++ musl-dev cmake make && apk add --no-cache" + + " --update --repository=https://dl-cdn.alpinelinux.org/alpine/v3.16/main/" + + " libexecinfo-dev") + } else { + "# (Skipping installation of build dependencies; custom base image.)" + } + } + + override fun defaultEntryPoint(): List = listOf("./bin/" + context.fileConfig.name) + + override fun generateCopyOfExecutable(): String = + """ + ${super.generateCopyOfExecutable()} + COPY --from=builder /usr/local/lib /usr/local/lib + COPY --from=builder /usr/lib /usr/lib + COPY --from=builder /lingua-franca . + """.trimIndent() + + override fun defaultBuildCommands(): List { + val mkdirCommand = listOf("mkdir", "-p", "build") + val commands = listOf( + mkdirCommand, + createCmakeCommand( + Path.of("./build"), + Path.of("."), + listOf("-DREACTOR_CPP_LINK_EXECINFO=ON"), + "src-gen" + ).command(), + createMakeCommand(fileConfig.buildPath, true, fileConfig.name).command(), + createMakeCommand(Path.of("./build"), true, "install").command() + ) + return commands.map { argListToCommand(it) } + } + } + + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator = StandaloneDockerGenerator(context) } diff --git a/core/src/main/kotlin/org/lflang/generator/rust/RustGenerator.kt b/core/src/main/kotlin/org/lflang/generator/rust/RustGenerator.kt index b64d849945..b0d275d48a 100644 --- a/core/src/main/kotlin/org/lflang/generator/rust/RustGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/rust/RustGenerator.kt @@ -28,6 +28,7 @@ import org.eclipse.emf.ecore.resource.Resource import org.lflang.target.Target import org.lflang.generator.* import org.lflang.generator.GeneratorUtils.canGenerate +import org.lflang.generator.docker.DockerGenerator import org.lflang.joinWithCommas import org.lflang.scoping.LFGlobalScopeProvider import org.lflang.target.property.* @@ -148,5 +149,8 @@ class RustGenerator( override fun getTarget(): Target = Target.Rust override fun getTargetTypes(): TargetTypes = RustTypes + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator { + TODO("Not yet implemented") + } } diff --git a/core/src/main/kotlin/org/lflang/generator/ts/TSGenerator.kt b/core/src/main/kotlin/org/lflang/generator/ts/TSGenerator.kt index 0404f5818b..91db8810e1 100644 --- a/core/src/main/kotlin/org/lflang/generator/ts/TSGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/ts/TSGenerator.kt @@ -28,15 +28,17 @@ package org.lflang.generator.ts import com.google.common.base.Strings import org.eclipse.emf.ecore.resource.Resource import org.eclipse.xtext.util.CancelIndicator -import org.lflang.target.Target import org.lflang.TimeValue import org.lflang.ast.DelayedConnectionTransformation import org.lflang.generator.* import org.lflang.generator.GeneratorUtils.canGenerate +import org.lflang.generator.docker.CDockerGenerator import org.lflang.generator.docker.DockerComposeGenerator +import org.lflang.generator.docker.DockerGenerator import org.lflang.generator.docker.TSDockerGenerator import org.lflang.lf.Preamble import org.lflang.model +import org.lflang.target.Target import org.lflang.target.property.DockerProperty import org.lflang.target.property.NoCompileProperty import org.lflang.target.property.ProtobufsProperty @@ -44,8 +46,7 @@ import org.lflang.target.property.RuntimeVersionProperty import org.lflang.util.FileUtil import java.nio.file.Files import java.nio.file.Path -import java.util.LinkedList -import kotlin.collections.HashMap +import java.util.* private const val NO_NPM_MESSAGE = "The TypeScript target requires npm >= 6.14.4. " + "For installation instructions, see: https://www.npmjs.com/get-npm. \n" + @@ -98,6 +99,10 @@ class TSGenerator( } + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator { + return TSDockerGenerator(context) + } + /** Generate TypeScript code from the Lingua Franca model contained by the * specified resource. This is the main entry point for code * generation. @@ -156,6 +161,8 @@ class TSGenerator( return; } context.finish(GeneratorResult.GENERATED_NO_EXECUTABLE.apply(context, codeMaps)) + } else if (targetConfig.get(DockerProperty.INSTANCE).enabled) { + buildUsingDocker() } else { compile(validator, resource, parsingContext) concludeCompilation(context, codeMaps) @@ -279,10 +286,13 @@ class TSGenerator( /** * Return whether it is advisable to install dependencies. */ - private fun shouldCollectDependencies(context: LFGeneratorContext): Boolean = - (context.mode != LFGeneratorContext.Mode.LSP_MEDIUM - && !targetConfig.get(NoCompileProperty.INSTANCE)) - || !fileConfig.srcGenPkgPath.resolve("node_modules").toFile().exists() + private fun shouldCollectDependencies(context: LFGeneratorContext): Boolean { + if (targetConfig.get(NoCompileProperty.INSTANCE) || targetConfig.get(DockerProperty.INSTANCE).enabled) { + return false; + } + return ((context.mode != LFGeneratorContext.Mode.LSP_MEDIUM && !targetConfig.get(NoCompileProperty.INSTANCE) + || !fileConfig.srcGenPkgPath.resolve("node_modules").toFile().exists())) + } /** * Collect the dependencies in package.json and their @@ -445,7 +455,7 @@ class TSGenerator( val jsPath = fileConfig.srcGenPath.resolve("dist").resolve("${fileConfig.name}.js") FileUtil.writeToFile("#!/bin/sh\nnode $jsPath", shScriptPath) shScriptPath.toFile().setExecutable(true) - messageReporter.nowhere().info("Script for executing the compiled program: $shScriptPath.") + messageReporter.nowhere().info("Script for running the program: $shScriptPath.") } } diff --git a/core/src/test/java/org/lflang/tests/compiler/LinguaFrancaValidationTest.java b/core/src/test/java/org/lflang/tests/compiler/LinguaFrancaValidationTest.java index 6132e7d0d1..8a51d4a027 100644 --- a/core/src/test/java/org/lflang/tests/compiler/LinguaFrancaValidationTest.java +++ b/core/src/test/java/org/lflang/tests/compiler/LinguaFrancaValidationTest.java @@ -1457,7 +1457,9 @@ public void recognizeHostNames() throws Exception { LfPackage.eINSTANCE.getKeyValuePair(), DictionaryType.DOCKER_DICT), List.of( - "{FROM: [1, 2, 3]}", LfPackage.eINSTANCE.getElement(), PrimitiveType.STRING)), + "{builder-base: [1, 2, 3]}", + LfPackage.eINSTANCE.getElement(), + PrimitiveType.STRING)), UnionType.TRACING_UNION, List.of( List.of("foo", LfPackage.eINSTANCE.getKeyValuePair(), UnionType.TRACING_UNION), diff --git a/gradle.properties b/gradle.properties index 307b046644..75e2c2dfd9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -22,6 +22,7 @@ freehepVersion=2.4 swtVersion=3.124.0 spotbugsToolVersion=4.7.3 jcipVersion=1.0 +commonsTextVersion=1.11.0 [manifestPropertyNames] org.eclipse.xtext=xtextVersion diff --git a/test/C/src/docker/federated/DockerOptions.lf b/test/C/src/docker/federated/DockerOptions.lf new file mode 100644 index 0000000000..a63608191f --- /dev/null +++ b/test/C/src/docker/federated/DockerOptions.lf @@ -0,0 +1,30 @@ +target C { + logging: WARN, + timeout: 1 s, + coordination: centralized, + docker: { + rti-image: "rti:local", + pre-build-script: "foo.sh", + pre-run-script: "bar.sh", + post-build-script: "baz.sh", + no-build: false + }, + cmake-include: "cmake-check-environment-variable.cmake" +} + +reactor Hello { + reaction(startup) {= + printf("Hello World!\n"); + // crash if env var "bar" is not set + if (getenv("BAR") == NULL) { + printf("bar is not set\n"); + exit(1); + } + printf("success: bar = %s\n", getenv("BAR")); + =} +} + +federated reactor { + a = new Hello() + b = new Hello() +} diff --git a/test/C/src/docker/federated/EscapedScriptName.lf b/test/C/src/docker/federated/EscapedScriptName.lf new file mode 100644 index 0000000000..25b44164b3 --- /dev/null +++ b/test/C/src/docker/federated/EscapedScriptName.lf @@ -0,0 +1,19 @@ +target C { + logging: DEBUG, + timeout: 1 s, + coordination: centralized, + docker: { + rti-image: "rti:local", + pre-build-script: "foo ish.sh" + } +} + +reactor Hello { + reaction(startup) {= + printf("Hello World!\n"); + =} +} + +federated reactor { + a = new Hello() +} diff --git a/test/C/src/docker/federated/bar.sh b/test/C/src/docker/federated/bar.sh new file mode 100755 index 0000000000..fcd6eed11a --- /dev/null +++ b/test/C/src/docker/federated/bar.sh @@ -0,0 +1,3 @@ +#!/bin/bash +echo "Hello from pre-run script!" +export BAR=bar diff --git a/test/C/src/docker/federated/baz.sh b/test/C/src/docker/federated/baz.sh new file mode 100644 index 0000000000..f640992ee4 --- /dev/null +++ b/test/C/src/docker/federated/baz.sh @@ -0,0 +1,2 @@ +#!/bin/bash +echo "Hello from post-build script!" diff --git a/test/C/src/docker/federated/cmake-check-environment-variable.cmake b/test/C/src/docker/federated/cmake-check-environment-variable.cmake new file mode 100644 index 0000000000..72c9aff868 --- /dev/null +++ b/test/C/src/docker/federated/cmake-check-environment-variable.cmake @@ -0,0 +1,6 @@ +# fail if the environment variable FOO is not set +if(DEFINED ENV{FOO}) + message("FOO is set to $ENV{FOO}") +else() + message(FATAL_ERROR "FOO is not set") +endif() diff --git a/test/C/src/docker/federated/foo ish.sh b/test/C/src/docker/federated/foo ish.sh new file mode 100644 index 0000000000..c070ce28b4 --- /dev/null +++ b/test/C/src/docker/federated/foo ish.sh @@ -0,0 +1,2 @@ +#!/bin/bash +echo "Hello from pre-build script with a foolish name!" diff --git a/test/C/src/docker/federated/foo.sh b/test/C/src/docker/federated/foo.sh new file mode 100644 index 0000000000..9e8fa941ef --- /dev/null +++ b/test/C/src/docker/federated/foo.sh @@ -0,0 +1,3 @@ +#!/bin/bash +echo "Hello from pre-build script!" +export FOO=foo diff --git a/test/Cpp/src/docker/HelloWorldContainerized.lf b/test/Cpp/src/docker/HelloWorldContainerized.lf new file mode 100644 index 0000000000..a7ddb6205c --- /dev/null +++ b/test/Cpp/src/docker/HelloWorldContainerized.lf @@ -0,0 +1,11 @@ +target Cpp { + logging: error, // To test generating a custom trace file name. + docker: true, + build-type: Debug +} + +import HelloWorld2 from "../HelloWorld.lf" + +main reactor { + a = new HelloWorld2() +}