From dc346135c031e1c80eebaf86327b80d89f14c55b Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Wed, 6 Mar 2024 22:54:25 -0800 Subject: [PATCH 01/48] Add `pre-build-cmd` to `docker` target property --- .../generator/docker/CDockerGenerator.java | 25 ++++++-------- .../generator/docker/DockerGenerator.java | 33 +++++++++++++++++++ .../generator/docker/TSDockerGenerator.java | 6 ++++ .../target/property/DockerProperty.java | 32 +++++++++++++----- .../DistributedCountContainerized.lf | 3 +- 5 files changed, 75 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index ab04e6cfd0..62c61ed216 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -1,10 +1,8 @@ package org.lflang.generator.docker; -import org.eclipse.xtext.xbase.lib.IterableExtensions; +import java.util.List; import org.lflang.generator.LFGeneratorContext; import org.lflang.target.Target; -import org.lflang.target.property.BuildCommandsProperty; -import org.lflang.util.StringUtil; /** * Generate the docker file related code for the C and CCpp target. @@ -34,10 +32,8 @@ public String defaultImage() { protected String generateDockerFileContent() { var lfModuleName = context.getFileConfig().name; var config = context.getTargetConfig(); - var compileCommand = - IterableExtensions.isNullOrEmpty(config.get(BuildCommandsProperty.INSTANCE)) - ? generateCompileCommand() - : StringUtil.joinObjects(config.get(BuildCommandsProperty.INSTANCE), " "); + var compileCommand = generateCompileCommand(); + var compiler = config.target == Target.CCPP ? "g++" : "gcc"; var baseImage = baseImage(); return String.join( "\n", @@ -82,14 +78,13 @@ protected String generateRunForBuildDependencies() { } } - /** Return the default compile command for the C docker container. */ - protected String generateCompileCommand() { - return String.join( - "\n", - "RUN set -ex && \\", - "mkdir bin && \\", - "cmake -DCMAKE_INSTALL_BINDIR=./bin -S src-gen -B bin && \\", - "cd bin && \\", + @Override + protected List defaultBuildCommands() { + return List.of( + "set -ex", + "mkdir -p bin", + "cmake -DCMAKE_INSTALL_BINDIR=./bin -S src-gen -B bin", + "cd bin", "make all"); } } diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index 203c5f94d7..b529342c26 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -1,8 +1,11 @@ package org.lflang.generator.docker; import java.nio.file.Path; +import java.util.List; import org.lflang.generator.LFGeneratorContext; +import org.lflang.target.property.BuildCommandsProperty; import org.lflang.target.property.DockerProperty; +import org.lflang.util.StringUtil; /** * A class for generating docker files. @@ -30,6 +33,28 @@ public DockerGenerator(LFGeneratorContext context) { /** Return a RUN command for installing/checking build dependencies. */ protected abstract String generateRunForBuildDependencies(); + /** Return the default compile commands for the C docker container. */ + protected abstract List defaultBuildCommands(); + + protected List getBuildCommands() { + var customBuildCommands = context.getTargetConfig().get(BuildCommandsProperty.INSTANCE); + if (customBuildCommands != null && !customBuildCommands.isEmpty()) { + return customBuildCommands; + } + return defaultBuildCommands(); + } + + /** Return the default compile command for the C docker container. */ + protected String generateCompileCommand() { + var preBuildCmd = preBuildCmd(); + var run = "RUN "; + var del = " && "; + if (!preBuildCmd.isEmpty()) { + run = run + preBuildCmd + del; + } + return run + StringUtil.joinObjects(getBuildCommands(), del); + } + /** Return the default base image. */ public abstract String defaultImage(); @@ -42,6 +67,14 @@ public String baseImage() { return defaultImage(); } + public String preBuildCmd() { + var preBuildCmd = context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildCmd(); + if (preBuildCmd != null) { + return preBuildCmd; + } + return ""; + } + /** * Produce a DockerData object, which bundles all information needed to output a Dockerfile. * diff --git a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java index cdb6ddcf4b..6f943bbb30 100644 --- a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java @@ -1,5 +1,6 @@ package org.lflang.generator.docker; +import java.util.List; import org.lflang.generator.LFGeneratorContext; /** @@ -31,6 +32,11 @@ protected String generateRunForBuildDependencies() { return "RUN which node && node --version"; } + @Override + protected List defaultBuildCommands() { + return List.of("pnpm install"); // FIXME: actually build using docker, not natively. + } + @Override public String defaultImage() { return "node:alpine"; diff --git a/core/src/main/java/org/lflang/target/property/DockerProperty.java b/core/src/main/java/org/lflang/target/property/DockerProperty.java index 1dc47724fe..c7eadcb1a3 100644 --- a/core/src/main/java/org/lflang/target/property/DockerProperty.java +++ b/core/src/main/java/org/lflang/target/property/DockerProperty.java @@ -36,6 +36,7 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { var enabled = false; var from = ""; var rti = DockerOptions.DOCKERHUB_RTI_IMAGE; + var preBuildCmd = ""; if (node.getLiteral() != null) { if (ASTUtils.toBoolean(node)) { @@ -45,15 +46,23 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { enabled = true; for (KeyValuePair entry : node.getKeyvalue().getPairs()) { DockerOption option = (DockerOption) DictionaryType.DOCKER_DICT.forName(entry.getName()); - if (option != null && option.equals(DockerOption.FROM)) { - from = ASTUtils.elementToSingleString(entry.getValue()); + if (option == null) { + continue; } - if (option != null && option.equals(DockerOption.RTI_IMAGE)) { - rti = ASTUtils.elementToSingleString(entry.getValue()); + switch (option) { + case FROM: + from = ASTUtils.elementToSingleString(entry.getValue()); + break; + case RTI_IMAGE: + rti = ASTUtils.elementToSingleString(entry.getValue()); + break; + case PRE_BUILD_CMD: + preBuildCmd = ASTUtils.elementToSingleString(entry.getValue()); + break; } } } - return new DockerOptions(enabled, from, rti); + return new DockerOptions(enabled, from, rti, preBuildCmd); } @Override @@ -86,6 +95,12 @@ public Element toAstElement(DockerOptions value) { } pair.setValue(ASTUtils.toElement(value.rti)); } + if (opt == DockerOption.PRE_BUILD_CMD) { + if (!value.preBuildCmd.isEmpty()) { + continue; + } + pair.setValue(ASTUtils.toElement(value.preBuildCmd)); + } kvp.getPairs().add(pair); } e.setKeyvalue(kvp); @@ -102,7 +117,7 @@ public String name() { } /** Settings related to Docker options. */ - public record DockerOptions(boolean enabled, String from, String rti) { + public record DockerOptions(boolean enabled, String from, String rti, String preBuildCmd) { /** Default location to pull the rti from. */ public static final String DOCKERHUB_RTI_IMAGE = "lflang/rti:rti"; @@ -111,7 +126,7 @@ public record DockerOptions(boolean enabled, String from, String rti) { public static final String LOCAL_RTI_IMAGE = "rti:local"; public DockerOptions(boolean enabled) { - this(enabled, "", DOCKERHUB_RTI_IMAGE); + this(enabled, "", DOCKERHUB_RTI_IMAGE, ""); } } @@ -122,7 +137,8 @@ public DockerOptions(boolean enabled) { */ public enum DockerOption implements DictionaryElement { FROM("FROM", PrimitiveType.STRING), - RTI_IMAGE("rti-image", PrimitiveType.STRING); + RTI_IMAGE("rti-image", PrimitiveType.STRING), + PRE_BUILD_CMD("pre-build-cmd", PrimitiveType.STRING); public final PrimitiveType type; diff --git a/test/C/src/docker/federated/DistributedCountContainerized.lf b/test/C/src/docker/federated/DistributedCountContainerized.lf index dfcf9472cf..0429d84d7d 100644 --- a/test/C/src/docker/federated/DistributedCountContainerized.lf +++ b/test/C/src/docker/federated/DistributedCountContainerized.lf @@ -9,7 +9,8 @@ target C { logging: DEBUG, coordination: centralized, docker: { - rti-image: "rti:local" + rti-image: "rti:local", + pre-build-cmd: "ls" } } From 5700b0f715f45ae7d4dce18f6ea1d9052129216e Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Mon, 11 Mar 2024 15:18:01 -0700 Subject: [PATCH 02/48] Minor cleanup --- .../org/lflang/generator/docker/CDockerGenerator.java | 9 +++------ .../org/lflang/generator/docker/DockerGenerator.java | 4 ++-- .../lflang/generator/docker/PythonDockerGenerator.java | 6 +++--- .../org/lflang/generator/docker/RtiDockerGenerator.java | 2 +- .../org/lflang/generator/docker/TSDockerGenerator.java | 4 ++-- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index 62c61ed216..02876769f6 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -31,18 +31,15 @@ public String defaultImage() { @Override protected String generateDockerFileContent() { var lfModuleName = context.getFileConfig().name; - var config = context.getTargetConfig(); - var compileCommand = generateCompileCommand(); - var compiler = config.target == Target.CCPP ? "g++" : "gcc"; var baseImage = baseImage(); return String.join( "\n", "# For instructions, see: https://www.lf-lang.org/docs/handbook/containerized-execution", "FROM " + baseImage + " AS builder", "WORKDIR /lingua-franca/" + lfModuleName, - generateRunForBuildDependencies(), + generateRunForInstallingDeps(), "COPY . src-gen", - compileCommand, + generateRunForBuild(), "", "FROM " + baseImage, "WORKDIR /lingua-franca", @@ -60,7 +57,7 @@ protected String generateDockerFileContent() { } @Override - protected String generateRunForBuildDependencies() { + protected String generateRunForInstallingDeps() { var config = context.getTargetConfig(); var compiler = config.target == Target.CCPP ? "g++" : "gcc"; if (baseImage().equals(defaultImage())) { diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index b529342c26..9db4915dc9 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -31,7 +31,7 @@ public DockerGenerator(LFGeneratorContext context) { protected abstract String generateDockerFileContent(); /** Return a RUN command for installing/checking build dependencies. */ - protected abstract String generateRunForBuildDependencies(); + protected abstract String generateRunForInstallingDeps(); /** Return the default compile commands for the C docker container. */ protected abstract List defaultBuildCommands(); @@ -45,7 +45,7 @@ protected List getBuildCommands() { } /** Return the default compile command for the C docker container. */ - protected String generateCompileCommand() { + protected String generateRunForBuild() { var preBuildCmd = preBuildCmd(); var run = "RUN "; var del = " && "; diff --git a/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java index 4b68c36cdd..c5766ec02e 100644 --- a/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java @@ -20,7 +20,7 @@ public String defaultImage() { } @Override - protected String generateRunForBuildDependencies() { + protected String generateRunForInstallingDeps() { if (baseImage().equals(defaultImage())) { return """ # Install build dependencies @@ -43,9 +43,9 @@ protected String generateDockerFileContent() { + " https://www.lf-lang.org/docs/handbook/containerized-execution?target=py", "FROM " + baseImage(), "WORKDIR /lingua-franca/" + context.getFileConfig().name, - generateRunForBuildDependencies(), + generateRunForInstallingDeps(), "COPY . src-gen", - super.generateCompileCommand(), + super.generateRunForBuild(), "ENTRYPOINT [\"python3\", \"-u\", \"src-gen/" + context.getFileConfig().name + ".py\"]"); } } diff --git a/core/src/main/java/org/lflang/generator/docker/RtiDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/RtiDockerGenerator.java index fb863eed4c..49abefd84d 100644 --- a/core/src/main/java/org/lflang/generator/docker/RtiDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/RtiDockerGenerator.java @@ -32,7 +32,7 @@ protected String generateDockerFileContent() { # Use ENTRYPOINT not CMD so that command-line arguments go through ENTRYPOINT ["RTI"] """ - .formatted(baseImage(), generateRunForBuildDependencies()); + .formatted(baseImage(), generateRunForInstallingDeps()); } @Override diff --git a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java index 6f943bbb30..ddeb0e35ca 100644 --- a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java @@ -24,11 +24,11 @@ public String generateDockerFileContent() { |COPY . . |ENTRYPOINT ["node", "dist/%s.js"] """ - .formatted(baseImage(), generateRunForBuildDependencies(), context.getFileConfig().name); + .formatted(baseImage(), generateRunForInstallingDeps(), context.getFileConfig().name); } @Override - protected String generateRunForBuildDependencies() { + protected String generateRunForInstallingDeps() { return "RUN which node && node --version"; } From 593ef2e28c964c3bbadf1906d3d318d0af94f514 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Fri, 5 Apr 2024 00:10:37 -0700 Subject: [PATCH 03/48] Use `pre-build-script` and `run-script` instead --- .../generator/docker/CDockerGenerator.java | 18 +++++++---- .../generator/docker/DockerGenerator.java | 24 ++++++-------- .../target/property/DockerProperty.java | 32 +++++++++++++------ 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index 02876769f6..5e2c2555d3 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -1,6 +1,8 @@ package org.lflang.generator.docker; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.lflang.generator.LFGeneratorContext; import org.lflang.target.Target; @@ -77,11 +79,15 @@ protected String generateRunForInstallingDeps() { @Override protected List defaultBuildCommands() { - return List.of( - "set -ex", - "mkdir -p bin", - "cmake -DCMAKE_INSTALL_BINDIR=./bin -S src-gen -B bin", - "cd bin", - "make all"); + return Stream.of( + List.of("set -ex"), + getPreBuildCommand(), + List.of( + "mkdir -p bin", + "cmake -DCMAKE_INSTALL_BINDIR=./bin -S src-gen -B bin", + "cd bin", + "make all")) + .flatMap(java.util.Collection::stream) + .collect(Collectors.toList()); } } diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index 9db4915dc9..33b607797f 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -44,15 +44,17 @@ protected List getBuildCommands() { return defaultBuildCommands(); } + protected List getPreBuildCommand() { + var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildScript(); + if (!script.isEmpty()) { + return List.of("source " + script); + } + return List.of(); + } + /** Return the default compile command for the C docker container. */ protected String generateRunForBuild() { - var preBuildCmd = preBuildCmd(); - var run = "RUN "; - var del = " && "; - if (!preBuildCmd.isEmpty()) { - run = run + preBuildCmd + del; - } - return run + StringUtil.joinObjects(getBuildCommands(), del); + return "RUN " + StringUtil.joinObjects(getBuildCommands(), " && "); } /** Return the default base image. */ @@ -67,14 +69,6 @@ public String baseImage() { return defaultImage(); } - public String preBuildCmd() { - var preBuildCmd = context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildCmd(); - if (preBuildCmd != null) { - return preBuildCmd; - } - return ""; - } - /** * Produce a DockerData object, which bundles all information needed to output a Dockerfile. * diff --git a/core/src/main/java/org/lflang/target/property/DockerProperty.java b/core/src/main/java/org/lflang/target/property/DockerProperty.java index c7eadcb1a3..b391ec2e97 100644 --- a/core/src/main/java/org/lflang/target/property/DockerProperty.java +++ b/core/src/main/java/org/lflang/target/property/DockerProperty.java @@ -36,7 +36,8 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { var enabled = false; var from = ""; var rti = DockerOptions.DOCKERHUB_RTI_IMAGE; - var preBuildCmd = ""; + var preBuildScript = ""; + var runScript = ""; if (node.getLiteral() != null) { if (ASTUtils.toBoolean(node)) { @@ -56,13 +57,16 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { case RTI_IMAGE: rti = ASTUtils.elementToSingleString(entry.getValue()); break; - case PRE_BUILD_CMD: - preBuildCmd = ASTUtils.elementToSingleString(entry.getValue()); + case PRE_BUILD_SCRIPT: + preBuildScript = ASTUtils.elementToSingleString(entry.getValue()); + break; + case RUN_SCRIPT: + runScript = ASTUtils.elementToSingleString(entry.getValue()); break; } } } - return new DockerOptions(enabled, from, rti, preBuildCmd); + return new DockerOptions(enabled, from, rti, preBuildScript, runScript); } @Override @@ -95,11 +99,17 @@ public Element toAstElement(DockerOptions value) { } pair.setValue(ASTUtils.toElement(value.rti)); } - if (opt == DockerOption.PRE_BUILD_CMD) { - if (!value.preBuildCmd.isEmpty()) { + if (opt == DockerOption.RUN_SCRIPT) { + if (!value.runScript.isEmpty()) { + continue; + } + pair.setValue(ASTUtils.toElement(value.runScript)); + } + if (opt == DockerOption.PRE_BUILD_SCRIPT) { + if (!value.preBuildScript.isEmpty()) { continue; } - pair.setValue(ASTUtils.toElement(value.preBuildCmd)); + pair.setValue(ASTUtils.toElement(value.preBuildScript)); } kvp.getPairs().add(pair); } @@ -117,7 +127,8 @@ public String name() { } /** Settings related to Docker options. */ - public record DockerOptions(boolean enabled, String from, String rti, String preBuildCmd) { + public record DockerOptions( + boolean enabled, String from, String rti, String preBuildScript, String runScript) { /** Default location to pull the rti from. */ public static final String DOCKERHUB_RTI_IMAGE = "lflang/rti:rti"; @@ -126,7 +137,7 @@ public record DockerOptions(boolean enabled, String from, String rti, String pre public static final String LOCAL_RTI_IMAGE = "rti:local"; public DockerOptions(boolean enabled) { - this(enabled, "", DOCKERHUB_RTI_IMAGE, ""); + this(enabled, "", DOCKERHUB_RTI_IMAGE, "", ""); } } @@ -138,7 +149,8 @@ public DockerOptions(boolean enabled) { public enum DockerOption implements DictionaryElement { FROM("FROM", PrimitiveType.STRING), RTI_IMAGE("rti-image", PrimitiveType.STRING), - PRE_BUILD_CMD("pre-build-cmd", PrimitiveType.STRING); + PRE_BUILD_SCRIPT("pre-build-script", PrimitiveType.STRING), + RUN_SCRIPT("run-script", PrimitiveType.STRING); public final PrimitiveType type; From 2d657765b82bb3b5caf02eef17bb5feaec3ae952 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Fri, 5 Apr 2024 00:22:19 -0700 Subject: [PATCH 04/48] Update reactor-c submodule --- core/src/main/resources/lib/c/reactor-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/resources/lib/c/reactor-c b/core/src/main/resources/lib/c/reactor-c index 277d3f2951..ec06c930a6 160000 --- a/core/src/main/resources/lib/c/reactor-c +++ b/core/src/main/resources/lib/c/reactor-c @@ -1 +1 @@ -Subproject commit 277d3f295167b9f9f3b45356937af8cf57bef3b8 +Subproject commit ec06c930a65d05420d103a6773fca8a442d58124 From bf3ef86316921f0d73c294ffc780aae654a9bad8 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Fri, 5 Apr 2024 00:23:16 -0700 Subject: [PATCH 05/48] Update reactor-cpp submodule --- core/src/main/resources/lib/cpp/reactor-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/resources/lib/cpp/reactor-cpp b/core/src/main/resources/lib/cpp/reactor-cpp index 3b25ffe3e2..f0f62ad339 160000 --- a/core/src/main/resources/lib/cpp/reactor-cpp +++ b/core/src/main/resources/lib/cpp/reactor-cpp @@ -1 +1 @@ -Subproject commit 3b25ffe3e270ab10181552a7e08ba37a471d8868 +Subproject commit f0f62ad339c82dc137955f7dcae617326b644622 From 30bc253e02f666004032e6215209dbce39801bf5 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Sat, 20 Apr 2024 09:17:43 -0700 Subject: [PATCH 06/48] WIP, amend later --- .../java/org/lflang/generator/docker/DockerData.java | 11 +++++++++++ .../org/lflang/target/property/DockerProperty.java | 4 ++++ .../docker/federated/DistributedCountContainerized.lf | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/lflang/generator/docker/DockerData.java b/core/src/main/java/org/lflang/generator/docker/DockerData.java index 7891eb74d6..d830a1d52b 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerData.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerData.java @@ -4,6 +4,7 @@ import java.nio.file.Files; import java.nio.file.Path; import org.lflang.generator.LFGeneratorContext; +import org.lflang.target.property.DockerProperty; import org.lflang.util.FileUtil; /** @@ -40,6 +41,16 @@ public DockerData( /** Write a docker file based on this data. */ public void writeDockerFile() throws IOException { + var pbs = context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildScript(); + if (!pbs.isEmpty()) { + var found = FileUtil.findInPackage(Path.of(pbs), context.getFileConfig()); // FIXME: the FileConfig points to fed-gen and the file hasn't been copied there + if (found != null) { + System.out.println(">>>>>>>>>>>>>>>>>>>!!!!!!!!!: " + found.toString()); + System.out.println(dockerFilePath.getParent().resolve(found.getFileName())); + FileUtil.copyFile(found, dockerFilePath.getParent().resolve(found.getFileName())); + } + } + Files.deleteIfExists(dockerFilePath); FileUtil.writeToFile(dockerFileContent, dockerFilePath); context.getErrorReporter().nowhere().info("Dockerfile written to " + dockerFilePath); diff --git a/core/src/main/java/org/lflang/target/property/DockerProperty.java b/core/src/main/java/org/lflang/target/property/DockerProperty.java index b391ec2e97..e6551bbc3c 100644 --- a/core/src/main/java/org/lflang/target/property/DockerProperty.java +++ b/core/src/main/java/org/lflang/target/property/DockerProperty.java @@ -1,5 +1,9 @@ package org.lflang.target.property; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.lflang.MessageReporter; import org.lflang.ast.ASTUtils; import org.lflang.lf.Element; diff --git a/test/C/src/docker/federated/DistributedCountContainerized.lf b/test/C/src/docker/federated/DistributedCountContainerized.lf index 0429d84d7d..2dc87df455 100644 --- a/test/C/src/docker/federated/DistributedCountContainerized.lf +++ b/test/C/src/docker/federated/DistributedCountContainerized.lf @@ -10,7 +10,7 @@ target C { coordination: centralized, docker: { rti-image: "rti:local", - pre-build-cmd: "ls" + pre-build-script: "foo.sh" } } From bbd9eb662bbb98f24e09431aa502e23545ebdfbb Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Tue, 11 Jun 2024 23:30:02 -0700 Subject: [PATCH 07/48] Handle pre-build script --- .../federated/generator/FedGenerator.java | 1 + .../lflang/generator/docker/DockerData.java | 27 ++++++++++++------- .../generator/docker/DockerGenerator.java | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java index 2d06f21436..77d97d8674 100644 --- a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java +++ b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java @@ -274,6 +274,7 @@ private List createDockerFiles( var dockerData = dockerGenerator.generateDockerData(); try { dockerData.writeDockerFile(); + dockerData.copyScripts(context); } catch (IOException e) { throw new RuntimeIOException(e); } diff --git a/core/src/main/java/org/lflang/generator/docker/DockerData.java b/core/src/main/java/org/lflang/generator/docker/DockerData.java index d830a1d52b..5c51158e1b 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerData.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerData.java @@ -6,6 +6,7 @@ import org.lflang.generator.LFGeneratorContext; import org.lflang.target.property.DockerProperty; import org.lflang.util.FileUtil; +import org.lflang.FileConfig; /** * Build configuration of a docker service. @@ -41,18 +42,24 @@ public DockerData( /** Write a docker file based on this data. */ public void writeDockerFile() throws IOException { - var pbs = context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildScript(); - if (!pbs.isEmpty()) { - var found = FileUtil.findInPackage(Path.of(pbs), context.getFileConfig()); // FIXME: the FileConfig points to fed-gen and the file hasn't been copied there - if (found != null) { - System.out.println(">>>>>>>>>>>>>>>>>>>!!!!!!!!!: " + found.toString()); - System.out.println(dockerFilePath.getParent().resolve(found.getFileName())); - FileUtil.copyFile(found, dockerFilePath.getParent().resolve(found.getFileName())); - } - } - Files.deleteIfExists(dockerFilePath); FileUtil.writeToFile(dockerFileContent, dockerFilePath); context.getErrorReporter().nowhere().info("Dockerfile written to " + dockerFilePath); } + + public void copyScripts(LFGeneratorContext context) throws IOException { + copyScript(context.getFileConfig(), context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildScript()); + copyScript(context.getFileConfig(), context.getTargetConfig().get(DockerProperty.INSTANCE).runScript()); + } + + private void copyScript(FileConfig fileConfig, String script) throws IOException { + if (!script.isEmpty()) { + var found = FileUtil.findInPackage(Path.of(script), fileConfig); + if (found != null) { + var destination = dockerFilePath.getParent().resolve(found.getFileName()); + FileUtil.copyFile(found, destination); + this.context.getErrorReporter().nowhere().info("Script written to " + destination); + } + } + } } diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index 33b607797f..4022c96fda 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -47,7 +47,7 @@ protected List getBuildCommands() { protected List getPreBuildCommand() { var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildScript(); if (!script.isEmpty()) { - return List.of("source " + script); + return List.of("source src-gen/" + script); } return List.of(); } From 9f25bf5a8b7297a17b1db7310758dde9dcd58612 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Wed, 12 Jun 2024 17:42:24 -0700 Subject: [PATCH 08/48] Various cleanups and bugfixes --- .../generator/docker/CDockerGenerator.java | 55 +++++++--------- .../lflang/generator/docker/DockerData.java | 13 +++- .../generator/docker/DockerGenerator.java | 64 ++++++++++++++++++- .../generator/docker/TSDockerGenerator.java | 19 ++++-- .../target/property/DockerProperty.java | 26 ++++++-- .../DistributedCountContainerized.lf | 4 +- 6 files changed, 128 insertions(+), 53 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index 8440f83e59..f7c51c21a4 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -1,8 +1,7 @@ package org.lflang.generator.docker; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; +import org.lflang.LocalStrings; import org.lflang.generator.LFGeneratorContext; import org.lflang.generator.c.CCompiler; import org.lflang.target.Target; @@ -37,7 +36,9 @@ protected String generateDockerFileContent() { var baseImage = baseImage(); return String.join( "\n", - "# For instructions, see: https://www.lf-lang.org/docs/handbook/containerized-execution", + "# Generated by the Lingua Franca compiler version " + LocalStrings.VERSION, + "# - Docs: https://www.lf-lang.org/docs/handbook/containerized-execution", + "", "FROM " + baseImage + " AS builder", "WORKDIR /lingua-franca/" + lfModuleName, generateRunForInstallingDeps(), @@ -46,33 +47,27 @@ protected String generateDockerFileContent() { "", "FROM " + baseImage, "WORKDIR /lingua-franca", + "RUN mkdir scripts", + generateCopyOfScript(), "RUN mkdir bin", - "COPY --from=builder /lingua-franca/" - + lfModuleName - + "/bin/" - + lfModuleName - + " ./bin/" - + lfModuleName, - "", - "# Use ENTRYPOINT not CMD so that command-line arguments go through", - "ENTRYPOINT [\"./bin/" + lfModuleName + "\"]", + generateCopyOfExecutable(), + generateEntryPoint(), ""); } + @Override + public List defaultEntryPoint() { + return List.of("./bin/" + context.getFileConfig().name); + } + @Override protected String generateRunForInstallingDeps() { var config = context.getTargetConfig(); var compiler = config.target == Target.CCPP ? "g++" : "gcc"; if (baseImage().equals(defaultImage())) { - return """ - # Install build dependencies - RUN set -ex && apk add --no-cache %s musl-dev cmake make - """ - .formatted(compiler); + return "RUN set -ex && apk add --no-cache %s musl-dev cmake make".formatted(compiler); } else { - return """ - # Skipping installation of build dependencies (custom base image) - """; + return "# Skipping installation of build dependencies (custom base image)"; } } @@ -83,18 +78,14 @@ protected List defaultBuildCommands() { context.getTargetConfig(), context.getFileConfig(), context.getErrorReporter(), - context.getTargetConfig().target == Target.C); + context.getTargetConfig().target == Target.CCPP); - return Stream.of( - List.of("set -ex"), - getPreBuildCommand(), - List.of( - "mkdir -p bin", - String.format("%s -DCMAKE_INSTALL_BINDIR=./bin -S src-gen -B bin", - ccompile.compileCmakeCommand()), - "cd bin", - "make all")) - .flatMap(java.util.Collection::stream) - .collect(Collectors.toList()); + return List.of( + "mkdir -p bin", + String.format( + "%s -DCMAKE_INSTALL_BINDIR=./bin -S src-gen -B bin", ccompile.compileCmakeCommand()), + "cd bin", + "make all", + "cd .."); } } diff --git a/core/src/main/java/org/lflang/generator/docker/DockerData.java b/core/src/main/java/org/lflang/generator/docker/DockerData.java index 5c51158e1b..27d126665e 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerData.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerData.java @@ -3,10 +3,10 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import org.lflang.FileConfig; import org.lflang.generator.LFGeneratorContext; import org.lflang.target.property.DockerProperty; import org.lflang.util.FileUtil; -import org.lflang.FileConfig; /** * Build configuration of a docker service. @@ -48,8 +48,15 @@ public void writeDockerFile() throws IOException { } public void copyScripts(LFGeneratorContext context) throws IOException { - copyScript(context.getFileConfig(), context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildScript()); - copyScript(context.getFileConfig(), context.getTargetConfig().get(DockerProperty.INSTANCE).runScript()); + copyScript( + context.getFileConfig(), + context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildScript()); + copyScript( + context.getFileConfig(), + context.getTargetConfig().get(DockerProperty.INSTANCE).postBuildScript()); + copyScript( + context.getFileConfig(), + context.getTargetConfig().get(DockerProperty.INSTANCE).preRunScript()); } private void copyScript(FileConfig fileConfig, String script) throws IOException { diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index b8bdd8a996..2ed09b4768 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -2,9 +2,12 @@ import java.nio.file.Path; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.lflang.generator.LFGeneratorContext; import org.lflang.target.property.BuildCommandsProperty; import org.lflang.target.property.DockerProperty; +import org.lflang.target.property.DockerProperty.DockerOptions; import org.lflang.util.StringUtil; /** @@ -36,15 +39,16 @@ public DockerGenerator(LFGeneratorContext context) { /** Return the default compile commands for the C docker container. */ protected abstract List defaultBuildCommands(); + /** Return the commands used to build */ protected List getBuildCommands() { var customBuildCommands = context.getTargetConfig().get(BuildCommandsProperty.INSTANCE); - // FIXME: also use pre-build-script in case of custom build commands. if (customBuildCommands != null && !customBuildCommands.isEmpty()) { return customBuildCommands; } return defaultBuildCommands(); } + /** Return the command that sources the pre-build script, if there is one. */ protected List getPreBuildCommand() { var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildScript(); if (!script.isEmpty()) { @@ -53,11 +57,65 @@ protected List getPreBuildCommand() { return List.of(); } - /** Return the default compile command for the C docker container. */ + /** Return the command that sources the post-build script, if there is one. */ + protected List getPostBuildCommand() { + var script = context.getTargetConfig().get(DockerProperty.INSTANCE).postBuildScript(); + if (!script.isEmpty()) { + return List.of("source src-gen/" + script); + } + return List.of(); + } + + /** Return the Docker RUN command used for building. */ protected String generateRunForBuild() { - return "RUN " + StringUtil.joinObjects(getBuildCommands(), " && "); + return "RUN " + + StringUtil.joinObjects( + Stream.of( + List.of("set -ex"), + getPreBuildCommand(), + getBuildCommands(), + getPostBuildCommand()) + .flatMap(java.util.Collection::stream) + .collect(Collectors.toList()), + " \\\n\t&& "); + } + + protected String generateEntryPoint() { + return "ENTRYPOINT [" + + getEntryPointCommands().stream() + .map(cmd -> "\"" + cmd + "\"") + .collect(Collectors.joining(",")) + + "]"; + } + + protected String generateCopyOfExecutable() { + var lfModuleName = context.getFileConfig().name; + return "COPY --from=builder /lingua-franca/%s/bin/%s ./bin/%s" + .formatted(lfModuleName, lfModuleName, lfModuleName); } + protected String generateCopyOfScript() { + var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preRunScript(); + if (!script.isEmpty()) { + return "COPY --from=builder /lingua-franca/%s/src-gen/%s ./scripts/%s" + .formatted(context.getFileConfig().name, script, script); + } + return "# (No pre-run script provided.)"; + } + + protected List getEntryPointCommands() { + var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preRunScript(); + if (!script.isEmpty()) { + return Stream.concat( + List.of(DockerOptions.DEFAULT_SHELL, "-c", "source scripts/" + script).stream(), + defaultEntryPoint().stream()) + .toList(); + } + return defaultEntryPoint(); + } + + public abstract List defaultEntryPoint(); + /** Return the default base image. */ public abstract String defaultImage(); diff --git a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java index cadae9f1e5..6ac2cf97c7 100644 --- a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java @@ -18,13 +18,18 @@ public TSDockerGenerator(LFGeneratorContext context) { /** Return the content of the docker file for [tsFileName]. */ public String generateDockerFileContent() { return """ - FROM %s - WORKDIR /linguafranca/$name - %s - COPY . . - ENTRYPOINT ["node", "dist/%s.js"] - """ - .formatted(baseImage(), generateRunForInstallingDeps(), context.getFileConfig().name); + FROM %s + WORKDIR /linguafranca/$name + %s + COPY . . + %s + """ + .formatted(baseImage(), generateRunForInstallingDeps(), generateEntryPoint()); + } + + @Override + public List defaultEntryPoint() { + return List.of("node", "dist/%s.js".formatted(context.getFileConfig().name)); } @Override diff --git a/core/src/main/java/org/lflang/target/property/DockerProperty.java b/core/src/main/java/org/lflang/target/property/DockerProperty.java index 3e4f29266d..0a927655b6 100644 --- a/core/src/main/java/org/lflang/target/property/DockerProperty.java +++ b/core/src/main/java/org/lflang/target/property/DockerProperty.java @@ -36,7 +36,9 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { var enabled = false; var from = ""; var rti = DockerOptions.DOCKERHUB_RTI_IMAGE; + var shell = DockerOptions.DEFAULT_SHELL; var preBuildScript = ""; + var postBuildScript = ""; var runScript = ""; if (node.getLiteral() != null) { @@ -51,12 +53,13 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { switch (option) { case FROM -> from = str; case PRE_BUILD_SCRIPT -> preBuildScript = str; + case PRE_RUN_SCRIPT -> runScript = str; + case POST_BUILD_SCRIPT -> postBuildScript = str; case RTI_IMAGE -> rti = str; - case RUN_SCRIPT -> runScript = str; } } } - return new DockerOptions(enabled, from, rti, preBuildScript, runScript); + return new DockerOptions(enabled, from, rti, shell, preBuildScript, postBuildScript, runScript); } @Override @@ -87,8 +90,9 @@ public Element toAstElement(DockerOptions value) { switch (opt) { case FROM -> pair.setValue(ASTUtils.toElement(value.from)); case PRE_BUILD_SCRIPT -> pair.setValue(ASTUtils.toElement(value.preBuildScript)); + case PRE_RUN_SCRIPT -> pair.setValue(ASTUtils.toElement(value.preRunScript)); + case POST_BUILD_SCRIPT -> pair.setValue(ASTUtils.toElement(value.postBuildScript)); case RTI_IMAGE -> pair.setValue(ASTUtils.toElement(value.rti)); - case RUN_SCRIPT -> pair.setValue(ASTUtils.toElement(value.runScript)); } kvp.getPairs().add(pair); } @@ -106,18 +110,25 @@ public String name() { } /** Settings related to Docker options. */ - public record DockerOptions( - boolean enabled, String from, String rti, String preBuildScript, String runScript) { + boolean enabled, + String from, + String rti, + String shell, + String preBuildScript, + String postBuildScript, + String preRunScript) { /** Default location to pull the rti from. */ public static final String DOCKERHUB_RTI_IMAGE = "lflang/rti:rti"; + public static final String DEFAULT_SHELL = "/bin/sh"; + /** String to indicate a local build of the rti. */ public static final String LOCAL_RTI_IMAGE = "rti:local"; public DockerOptions(boolean enabled) { - this(enabled, "", DOCKERHUB_RTI_IMAGE, "", ""); + this(enabled, "", DOCKERHUB_RTI_IMAGE, DEFAULT_SHELL, "", "", ""); } } @@ -130,7 +141,8 @@ public enum DockerOption implements DictionaryElement { FROM("FROM", PrimitiveType.STRING), RTI_IMAGE("rti-image", PrimitiveType.STRING), PRE_BUILD_SCRIPT("pre-build-script", PrimitiveType.STRING), - RUN_SCRIPT("run-script", PrimitiveType.STRING); + PRE_RUN_SCRIPT("pre-run-script", PrimitiveType.STRING), + POST_BUILD_SCRIPT("post-build-script", PrimitiveType.STRING); public final PrimitiveType type; diff --git a/test/C/src/docker/federated/DistributedCountContainerized.lf b/test/C/src/docker/federated/DistributedCountContainerized.lf index 2dc87df455..679e9a0572 100644 --- a/test/C/src/docker/federated/DistributedCountContainerized.lf +++ b/test/C/src/docker/federated/DistributedCountContainerized.lf @@ -10,7 +10,9 @@ target C { coordination: centralized, docker: { rti-image: "rti:local", - pre-build-script: "foo.sh" + pre-build-script: "foo.sh", + pre-run-script: "bar.sh", + post-build-script: "baz.sh" } } From 7e9dc70036abc6cba3bf1905b30ef0af29a3c5e5 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Wed, 12 Jun 2024 23:01:55 -0700 Subject: [PATCH 09/48] Distinguish between builder and runner --- .../federated/generator/FedGenerator.java | 2 +- .../generator/docker/CDockerGenerator.java | 9 +++--- .../lflang/generator/docker/DockerData.java | 32 +++++++++---------- .../generator/docker/DockerGenerator.java | 15 ++++++--- .../docker/PythonDockerGenerator.java | 14 +++----- .../generator/docker/RtiDockerGenerator.java | 2 +- .../generator/docker/TSDockerGenerator.java | 2 +- .../target/property/DockerProperty.java | 19 +++++++---- test/C/src/docker/federated/bar.sh | 2 ++ test/C/src/docker/federated/baz.sh | 2 ++ test/C/src/docker/federated/foo.sh | 2 ++ 11 files changed, 56 insertions(+), 45 deletions(-) create mode 100644 test/C/src/docker/federated/bar.sh create mode 100644 test/C/src/docker/federated/baz.sh create mode 100644 test/C/src/docker/federated/foo.sh diff --git a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java index 7456b6010f..f0e62d9f87 100644 --- a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java +++ b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java @@ -279,7 +279,7 @@ private List createDockerFiles( var dockerData = dockerGenerator.generateDockerData(); try { dockerData.writeDockerFile(); - dockerData.copyScripts(context); + dockerData.copyScripts(); } catch (IOException e) { throw new RuntimeIOException(e); } diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index f7c51c21a4..5d9fc4df66 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -33,19 +33,18 @@ public String defaultImage() { @Override protected String generateDockerFileContent() { var lfModuleName = context.getFileConfig().name; - var baseImage = baseImage(); return String.join( "\n", "# Generated by the Lingua Franca compiler version " + LocalStrings.VERSION, "# - Docs: https://www.lf-lang.org/docs/handbook/containerized-execution", "", - "FROM " + baseImage + " AS builder", + "FROM " + builderBase() + " AS builder", "WORKDIR /lingua-franca/" + lfModuleName, generateRunForInstallingDeps(), "COPY . src-gen", generateRunForBuild(), "", - "FROM " + baseImage, + "FROM " + runnerBase(), "WORKDIR /lingua-franca", "RUN mkdir scripts", generateCopyOfScript(), @@ -64,10 +63,10 @@ public List defaultEntryPoint() { protected String generateRunForInstallingDeps() { var config = context.getTargetConfig(); var compiler = config.target == Target.CCPP ? "g++" : "gcc"; - if (baseImage().equals(defaultImage())) { + if (builderBase().equals(defaultImage())) { return "RUN set -ex && apk add --no-cache %s musl-dev cmake make".formatted(compiler); } else { - return "# Skipping installation of build dependencies (custom base image)"; + return "# (Skipping installation of build dependencies; custom base image.)"; } } diff --git a/core/src/main/java/org/lflang/generator/docker/DockerData.java b/core/src/main/java/org/lflang/generator/docker/DockerData.java index 27d126665e..46eb9b3da5 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerData.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerData.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import org.lflang.FileConfig; import org.lflang.generator.LFGeneratorContext; import org.lflang.target.property.DockerProperty; @@ -47,25 +48,24 @@ public void writeDockerFile() throws IOException { context.getErrorReporter().nowhere().info("Dockerfile written to " + dockerFilePath); } - public void copyScripts(LFGeneratorContext context) throws IOException { - copyScript( + /** Copy the pre-build, post-build, and pre-run scripts, if specified */ + public void copyScripts() throws IOException { + var prop = context.getTargetConfig().get(DockerProperty.INSTANCE); + copyScripts( context.getFileConfig(), - context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildScript()); - copyScript( - context.getFileConfig(), - context.getTargetConfig().get(DockerProperty.INSTANCE).postBuildScript()); - copyScript( - context.getFileConfig(), - context.getTargetConfig().get(DockerProperty.INSTANCE).preRunScript()); + List.of(prop.preBuildScript(), prop.postBuildScript(), prop.preRunScript())); } - private void copyScript(FileConfig fileConfig, String script) throws IOException { - if (!script.isEmpty()) { - var found = FileUtil.findInPackage(Path.of(script), fileConfig); - if (found != null) { - var destination = dockerFilePath.getParent().resolve(found.getFileName()); - FileUtil.copyFile(found, destination); - this.context.getErrorReporter().nowhere().info("Script written to " + destination); + /** Copy the given list of scripts */ + private void copyScripts(FileConfig fileConfig, List scripts) throws IOException { + for (var script : scripts) { + if (!script.isEmpty()) { + var found = FileUtil.findInPackage(Path.of(script), fileConfig); + if (found != null) { + var destination = dockerFilePath.getParent().resolve(found.getFileName()); + FileUtil.copyFile(found, destination); + this.context.getErrorReporter().nowhere().info("Script written to " + destination); + } } } } diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index 2ed09b4768..75dd92d4c6 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -119,11 +119,18 @@ protected List getEntryPointCommands() { /** Return the default base image. */ public abstract String defaultImage(); + protected String builderBase() { + return baseImage(context.getTargetConfig().get(DockerProperty.INSTANCE).builderBase()); + } + + protected String runnerBase() { + return baseImage(context.getTargetConfig().get(DockerProperty.INSTANCE).runnerBase()); + } + /** Return the selected base image, or the default one if none was selected. */ - public String baseImage() { - var baseImage = context.getTargetConfig().get(DockerProperty.INSTANCE).from(); - if (baseImage != null && !baseImage.isEmpty()) { - return baseImage; + private String baseImage(String name) { + if (name != null && !name.isEmpty()) { + return name; } return defaultImage(); } diff --git a/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java index c341e24245..10eb85b50f 100644 --- a/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java @@ -21,16 +21,10 @@ public String defaultImage() { @Override protected String generateRunForInstallingDeps() { - if (baseImage().equals(defaultImage())) { - return """ - # Install build dependencies - RUN set -ex && apt-get update && apt-get install -y python3-pip && pip install cmake - """; + if (builderBase().equals(defaultImage())) { + return "RUN set -ex && apt-get update && apt-get install -y python3-pip && pip install cmake"; } else { - return """ - # Check for build dependencies - RUN which make && which cmake && which gcc - """; + return "# (Skipping installation of build dependencies; custom base image.)"; } } @@ -41,7 +35,7 @@ protected String generateDockerFileContent() { "\n", "# For instructions, see:" + " https://www.lf-lang.org/docs/handbook/containerized-execution?target=py", - "FROM " + baseImage(), + "FROM " + builderBase(), // FIXME: create stages "WORKDIR /lingua-franca/" + context.getFileConfig().name, generateRunForInstallingDeps(), "COPY . src-gen", diff --git a/core/src/main/java/org/lflang/generator/docker/RtiDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/RtiDockerGenerator.java index d281cc896c..c936e529ae 100644 --- a/core/src/main/java/org/lflang/generator/docker/RtiDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/RtiDockerGenerator.java @@ -28,7 +28,7 @@ protected String generateDockerFileContent() { } @Override - public String baseImage() { + public String builderBase() { return defaultImage(); } } diff --git a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java index 6ac2cf97c7..bf0530e4c0 100644 --- a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java @@ -24,7 +24,7 @@ public String generateDockerFileContent() { COPY . . %s """ - .formatted(baseImage(), generateRunForInstallingDeps(), generateEntryPoint()); + .formatted(builderBase(), generateRunForInstallingDeps(), generateEntryPoint()); } @Override diff --git a/core/src/main/java/org/lflang/target/property/DockerProperty.java b/core/src/main/java/org/lflang/target/property/DockerProperty.java index 0a927655b6..8c007e0982 100644 --- a/core/src/main/java/org/lflang/target/property/DockerProperty.java +++ b/core/src/main/java/org/lflang/target/property/DockerProperty.java @@ -34,7 +34,8 @@ public DockerOptions initialValue() { @Override public DockerOptions fromAst(Element node, MessageReporter reporter) { var enabled = false; - var from = ""; + var builderBase = ""; + var runnerBase = ""; var rti = DockerOptions.DOCKERHUB_RTI_IMAGE; var shell = DockerOptions.DEFAULT_SHELL; var preBuildScript = ""; @@ -51,7 +52,8 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { DockerOption option = (DockerOption) DictionaryType.DOCKER_DICT.forName(entry.getName()); var str = ASTUtils.elementToSingleString(entry.getValue()); switch (option) { - case FROM -> from = str; + case BUILDER_BASE -> builderBase = str; + case RUNNER_BASE -> runnerBase = str; case PRE_BUILD_SCRIPT -> preBuildScript = str; case PRE_RUN_SCRIPT -> runScript = str; case POST_BUILD_SCRIPT -> postBuildScript = str; @@ -59,7 +61,7 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { } } } - return new DockerOptions(enabled, from, rti, shell, preBuildScript, postBuildScript, runScript); + return new DockerOptions(enabled, builderBase, runnerBase, rti, shell, preBuildScript, postBuildScript, runScript); } @Override @@ -88,7 +90,8 @@ public Element toAstElement(DockerOptions value) { KeyValuePair pair = LfFactory.eINSTANCE.createKeyValuePair(); pair.setName(opt.toString()); switch (opt) { - case FROM -> pair.setValue(ASTUtils.toElement(value.from)); + case BUILDER_BASE -> pair.setValue(ASTUtils.toElement(value.builderBase)); + case RUNNER_BASE -> pair.setValue(ASTUtils.toElement(value.runnerBase)); case PRE_BUILD_SCRIPT -> pair.setValue(ASTUtils.toElement(value.preBuildScript)); case PRE_RUN_SCRIPT -> pair.setValue(ASTUtils.toElement(value.preRunScript)); case POST_BUILD_SCRIPT -> pair.setValue(ASTUtils.toElement(value.postBuildScript)); @@ -112,7 +115,8 @@ public String name() { /** Settings related to Docker options. */ public record DockerOptions( boolean enabled, - String from, + String builderBase, + String runnerBase, String rti, String shell, String preBuildScript, @@ -128,7 +132,7 @@ public record DockerOptions( public static final String LOCAL_RTI_IMAGE = "rti:local"; public DockerOptions(boolean enabled) { - this(enabled, "", DOCKERHUB_RTI_IMAGE, DEFAULT_SHELL, "", "", ""); + this(enabled, "", "", DOCKERHUB_RTI_IMAGE, DEFAULT_SHELL, "", "", ""); } } @@ -138,7 +142,8 @@ public DockerOptions(boolean enabled) { * @author Edward A. Lee */ public enum DockerOption implements DictionaryElement { - FROM("FROM", PrimitiveType.STRING), + BUILDER_BASE("builder-base", PrimitiveType.STRING), + RUNNER_BASE("runner-base", PrimitiveType.STRING), RTI_IMAGE("rti-image", PrimitiveType.STRING), PRE_BUILD_SCRIPT("pre-build-script", PrimitiveType.STRING), PRE_RUN_SCRIPT("pre-run-script", PrimitiveType.STRING), diff --git a/test/C/src/docker/federated/bar.sh b/test/C/src/docker/federated/bar.sh new file mode 100644 index 0000000000..60f774ffa7 --- /dev/null +++ b/test/C/src/docker/federated/bar.sh @@ -0,0 +1,2 @@ +#!/bin/bash +echo "Hello from pre-build script!" diff --git a/test/C/src/docker/federated/baz.sh b/test/C/src/docker/federated/baz.sh new file mode 100644 index 0000000000..f640992ee4 --- /dev/null +++ b/test/C/src/docker/federated/baz.sh @@ -0,0 +1,2 @@ +#!/bin/bash +echo "Hello from post-build script!" diff --git a/test/C/src/docker/federated/foo.sh b/test/C/src/docker/federated/foo.sh new file mode 100644 index 0000000000..60f774ffa7 --- /dev/null +++ b/test/C/src/docker/federated/foo.sh @@ -0,0 +1,2 @@ +#!/bin/bash +echo "Hello from pre-build script!" From c85a2eb9b265851a6773a834231df140723af3ca Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Wed, 12 Jun 2024 23:15:36 -0700 Subject: [PATCH 10/48] Use correct context --- .../main/java/org/lflang/federated/generator/FedGenerator.java | 2 +- core/src/main/java/org/lflang/generator/docker/DockerData.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java index f0e62d9f87..7456b6010f 100644 --- a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java +++ b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java @@ -279,7 +279,7 @@ private List createDockerFiles( var dockerData = dockerGenerator.generateDockerData(); try { dockerData.writeDockerFile(); - dockerData.copyScripts(); + dockerData.copyScripts(context); } catch (IOException e) { throw new RuntimeIOException(e); } diff --git a/core/src/main/java/org/lflang/generator/docker/DockerData.java b/core/src/main/java/org/lflang/generator/docker/DockerData.java index 46eb9b3da5..75338630b5 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerData.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerData.java @@ -49,7 +49,7 @@ public void writeDockerFile() throws IOException { } /** Copy the pre-build, post-build, and pre-run scripts, if specified */ - public void copyScripts() throws IOException { + public void copyScripts(LFGeneratorContext context) throws IOException { var prop = context.getTargetConfig().get(DockerProperty.INSTANCE); copyScripts( context.getFileConfig(), From 844d82bfd7c1b824fefc7498226c0dba078caee2 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Wed, 12 Jun 2024 23:28:21 -0700 Subject: [PATCH 11/48] Formatting --- .../main/java/org/lflang/target/property/DockerProperty.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/lflang/target/property/DockerProperty.java b/core/src/main/java/org/lflang/target/property/DockerProperty.java index 8c007e0982..69960949f8 100644 --- a/core/src/main/java/org/lflang/target/property/DockerProperty.java +++ b/core/src/main/java/org/lflang/target/property/DockerProperty.java @@ -61,7 +61,8 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { } } } - return new DockerOptions(enabled, builderBase, runnerBase, rti, shell, preBuildScript, postBuildScript, runScript); + return new DockerOptions( + enabled, builderBase, runnerBase, rti, shell, preBuildScript, postBuildScript, runScript); } @Override From 999b1a5de3477d8215bc33b35c0bcc21a578d422 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Thu, 13 Jun 2024 18:03:32 -0700 Subject: [PATCH 12/48] Make it possible to skip building using docker --- .../federated/generator/FedGenerator.java | 11 ++++--- .../org/lflang/generator/c/CGenerator.java | 20 +++++++----- .../target/property/DockerProperty.java | 32 +++++++++++++------ .../DistributedCountContainerized.lf | 3 +- 4 files changed, 44 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java index 7456b6010f..bd81717c40 100644 --- a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java +++ b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java @@ -221,10 +221,13 @@ private void buildUsingDocker(LFGeneratorContext context, List subCo try { var dockerGen = new FedDockerComposeGenerator(context, rtiConfig.getHost()); dockerGen.writeDockerComposeFile(createDockerFiles(context, subContexts)); - if (dockerGen.build()) { - dockerGen.createLauncher(); - } else { - context.getErrorReporter().nowhere().error("Docker build failed."); + // Only build if requested + if (context.getTargetConfig().get(DockerProperty.INSTANCE).build()) { + if (dockerGen.build()) { + dockerGen.createLauncher(); + } else { + context.getErrorReporter().nowhere().error("Docker build failed."); + } } } catch (IOException e) { context diff --git a/core/src/main/java/org/lflang/generator/c/CGenerator.java b/core/src/main/java/org/lflang/generator/c/CGenerator.java index 298a637098..bb31c8bfad 100644 --- a/core/src/main/java/org/lflang/generator/c/CGenerator.java +++ b/core/src/main/java/org/lflang/generator/c/CGenerator.java @@ -586,16 +586,20 @@ private boolean buildUsingDocker() { dockerData.writeDockerFile(); dockerCompose.writeDockerComposeFile(List.of(dockerData)); } catch (IOException e) { - throw new RuntimeException("Error while writing Docker files", e); + context.getErrorReporter().nowhere().error("Error while writing Docker files"); + return false; } - var success = dockerCompose.build(); - if (!success) { - messageReporter.nowhere().error("Docker-compose build failed."); - } - if (success && mainDef != null) { - dockerCompose.createLauncher(); + // Only build if requested + if (context.getTargetConfig().get(DockerProperty.INSTANCE).build()) { + var success = dockerCompose.build(); + if (success) { + dockerCompose.createLauncher(); + } else { + context.getErrorReporter().nowhere().error("Docker build failed."); + } + return success; } - return success; + return true; } private void generateCodeFor(String lfModuleName) throws IOException { diff --git a/core/src/main/java/org/lflang/target/property/DockerProperty.java b/core/src/main/java/org/lflang/target/property/DockerProperty.java index 69960949f8..28ee18b126 100644 --- a/core/src/main/java/org/lflang/target/property/DockerProperty.java +++ b/core/src/main/java/org/lflang/target/property/DockerProperty.java @@ -34,6 +34,7 @@ public DockerOptions initialValue() { @Override public DockerOptions fromAst(Element node, MessageReporter reporter) { var enabled = false; + var build = true; var builderBase = ""; var runnerBase = ""; var rti = DockerOptions.DOCKERHUB_RTI_IMAGE; @@ -50,19 +51,29 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { enabled = true; for (KeyValuePair entry : node.getKeyvalue().getPairs()) { DockerOption option = (DockerOption) DictionaryType.DOCKER_DICT.forName(entry.getName()); - var str = ASTUtils.elementToSingleString(entry.getValue()); switch (option) { - case BUILDER_BASE -> builderBase = str; - case RUNNER_BASE -> runnerBase = str; - case PRE_BUILD_SCRIPT -> preBuildScript = str; - case PRE_RUN_SCRIPT -> runScript = str; - case POST_BUILD_SCRIPT -> postBuildScript = str; - case RTI_IMAGE -> rti = str; + case BUILD -> build = ASTUtils.toBoolean(entry.getValue()); + case BUILDER_BASE -> builderBase = ASTUtils.elementToSingleString(entry.getValue()); + case RUNNER_BASE -> runnerBase = ASTUtils.elementToSingleString(entry.getValue()); + case PRE_BUILD_SCRIPT -> + preBuildScript = ASTUtils.elementToSingleString(entry.getValue()); + case PRE_RUN_SCRIPT -> runScript = ASTUtils.elementToSingleString(entry.getValue()); + case POST_BUILD_SCRIPT -> + postBuildScript = ASTUtils.elementToSingleString(entry.getValue()); + case RTI_IMAGE -> rti = ASTUtils.elementToSingleString(entry.getValue()); } } } return new DockerOptions( - enabled, builderBase, runnerBase, rti, shell, preBuildScript, postBuildScript, runScript); + enabled, + build, + builderBase, + runnerBase, + rti, + shell, + preBuildScript, + postBuildScript, + runScript); } @Override @@ -91,6 +102,7 @@ public Element toAstElement(DockerOptions value) { KeyValuePair pair = LfFactory.eINSTANCE.createKeyValuePair(); pair.setName(opt.toString()); switch (opt) { + case BUILD -> pair.setValue(ASTUtils.toElement(value.build)); case BUILDER_BASE -> pair.setValue(ASTUtils.toElement(value.builderBase)); case RUNNER_BASE -> pair.setValue(ASTUtils.toElement(value.runnerBase)); case PRE_BUILD_SCRIPT -> pair.setValue(ASTUtils.toElement(value.preBuildScript)); @@ -116,6 +128,7 @@ public String name() { /** Settings related to Docker options. */ public record DockerOptions( boolean enabled, + boolean build, String builderBase, String runnerBase, String rti, @@ -133,7 +146,7 @@ public record DockerOptions( public static final String LOCAL_RTI_IMAGE = "rti:local"; public DockerOptions(boolean enabled) { - this(enabled, "", "", DOCKERHUB_RTI_IMAGE, DEFAULT_SHELL, "", "", ""); + this(enabled, true, "", "", DOCKERHUB_RTI_IMAGE, DEFAULT_SHELL, "", "", ""); } } @@ -143,6 +156,7 @@ public DockerOptions(boolean enabled) { * @author Edward A. Lee */ public enum DockerOption implements DictionaryElement { + BUILD("build", PrimitiveType.BOOLEAN), BUILDER_BASE("builder-base", PrimitiveType.STRING), RUNNER_BASE("runner-base", PrimitiveType.STRING), RTI_IMAGE("rti-image", PrimitiveType.STRING), diff --git a/test/C/src/docker/federated/DistributedCountContainerized.lf b/test/C/src/docker/federated/DistributedCountContainerized.lf index 679e9a0572..d2e3903b5d 100644 --- a/test/C/src/docker/federated/DistributedCountContainerized.lf +++ b/test/C/src/docker/federated/DistributedCountContainerized.lf @@ -12,7 +12,8 @@ target C { rti-image: "rti:local", pre-build-script: "foo.sh", pre-run-script: "bar.sh", - post-build-script: "baz.sh" + post-build-script: "baz.sh", + build: false } } From c87d82e1344df56ffb4adeba85cd30dc974156fd Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Thu, 13 Jun 2024 18:09:18 -0700 Subject: [PATCH 13/48] Change from build to no-build and change polarity --- .../org/lflang/target/property/DockerProperty.java | 12 ++++++------ .../federated/DistributedCountContainerized.lf | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/lflang/target/property/DockerProperty.java b/core/src/main/java/org/lflang/target/property/DockerProperty.java index 28ee18b126..bef0689fe9 100644 --- a/core/src/main/java/org/lflang/target/property/DockerProperty.java +++ b/core/src/main/java/org/lflang/target/property/DockerProperty.java @@ -34,7 +34,7 @@ public DockerOptions initialValue() { @Override public DockerOptions fromAst(Element node, MessageReporter reporter) { var enabled = false; - var build = true; + var noBuild = false; var builderBase = ""; var runnerBase = ""; var rti = DockerOptions.DOCKERHUB_RTI_IMAGE; @@ -52,7 +52,7 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { for (KeyValuePair entry : node.getKeyvalue().getPairs()) { DockerOption option = (DockerOption) DictionaryType.DOCKER_DICT.forName(entry.getName()); switch (option) { - case BUILD -> build = ASTUtils.toBoolean(entry.getValue()); + case NO_BUILD -> noBuild = ASTUtils.toBoolean(entry.getValue()); case BUILDER_BASE -> builderBase = ASTUtils.elementToSingleString(entry.getValue()); case RUNNER_BASE -> runnerBase = ASTUtils.elementToSingleString(entry.getValue()); case PRE_BUILD_SCRIPT -> @@ -66,7 +66,7 @@ public DockerOptions fromAst(Element node, MessageReporter reporter) { } return new DockerOptions( enabled, - build, + noBuild, builderBase, runnerBase, rti, @@ -102,7 +102,7 @@ public Element toAstElement(DockerOptions value) { KeyValuePair pair = LfFactory.eINSTANCE.createKeyValuePair(); pair.setName(opt.toString()); switch (opt) { - case BUILD -> pair.setValue(ASTUtils.toElement(value.build)); + case NO_BUILD -> pair.setValue(ASTUtils.toElement(value.build)); case BUILDER_BASE -> pair.setValue(ASTUtils.toElement(value.builderBase)); case RUNNER_BASE -> pair.setValue(ASTUtils.toElement(value.runnerBase)); case PRE_BUILD_SCRIPT -> pair.setValue(ASTUtils.toElement(value.preBuildScript)); @@ -146,7 +146,7 @@ public record DockerOptions( public static final String LOCAL_RTI_IMAGE = "rti:local"; public DockerOptions(boolean enabled) { - this(enabled, true, "", "", DOCKERHUB_RTI_IMAGE, DEFAULT_SHELL, "", "", ""); + this(enabled, false, "", "", DOCKERHUB_RTI_IMAGE, DEFAULT_SHELL, "", "", ""); } } @@ -156,7 +156,7 @@ public DockerOptions(boolean enabled) { * @author Edward A. Lee */ public enum DockerOption implements DictionaryElement { - BUILD("build", PrimitiveType.BOOLEAN), + NO_BUILD("no-build", PrimitiveType.BOOLEAN), BUILDER_BASE("builder-base", PrimitiveType.STRING), RUNNER_BASE("runner-base", PrimitiveType.STRING), RTI_IMAGE("rti-image", PrimitiveType.STRING), diff --git a/test/C/src/docker/federated/DistributedCountContainerized.lf b/test/C/src/docker/federated/DistributedCountContainerized.lf index d2e3903b5d..7b6de5b2f2 100644 --- a/test/C/src/docker/federated/DistributedCountContainerized.lf +++ b/test/C/src/docker/federated/DistributedCountContainerized.lf @@ -13,7 +13,7 @@ target C { pre-build-script: "foo.sh", pre-run-script: "bar.sh", post-build-script: "baz.sh", - build: false + no-build: false } } From 753352da55e098906b2a2e6d909abba5680a3ff8 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Thu, 13 Jun 2024 20:18:13 -0700 Subject: [PATCH 14/48] Bugfix and added test --- .../java/org/lflang/federated/generator/FedGenerator.java | 2 +- core/src/main/java/org/lflang/generator/c/CGenerator.java | 2 +- .../java/org/lflang/target/property/DockerProperty.java | 4 ++-- .../C/src/docker/federated/DistributedCountContainerized.lf | 6 +----- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java index bd81717c40..b89d453557 100644 --- a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java +++ b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java @@ -222,7 +222,7 @@ private void buildUsingDocker(LFGeneratorContext context, List subCo var dockerGen = new FedDockerComposeGenerator(context, rtiConfig.getHost()); dockerGen.writeDockerComposeFile(createDockerFiles(context, subContexts)); // Only build if requested - if (context.getTargetConfig().get(DockerProperty.INSTANCE).build()) { + if (!context.getTargetConfig().get(DockerProperty.INSTANCE).noBuild()) { if (dockerGen.build()) { dockerGen.createLauncher(); } else { diff --git a/core/src/main/java/org/lflang/generator/c/CGenerator.java b/core/src/main/java/org/lflang/generator/c/CGenerator.java index bb31c8bfad..8fff49d1ef 100644 --- a/core/src/main/java/org/lflang/generator/c/CGenerator.java +++ b/core/src/main/java/org/lflang/generator/c/CGenerator.java @@ -590,7 +590,7 @@ private boolean buildUsingDocker() { return false; } // Only build if requested - if (context.getTargetConfig().get(DockerProperty.INSTANCE).build()) { + if (!context.getTargetConfig().get(DockerProperty.INSTANCE).noBuild()) { var success = dockerCompose.build(); if (success) { dockerCompose.createLauncher(); diff --git a/core/src/main/java/org/lflang/target/property/DockerProperty.java b/core/src/main/java/org/lflang/target/property/DockerProperty.java index bef0689fe9..2aa539207e 100644 --- a/core/src/main/java/org/lflang/target/property/DockerProperty.java +++ b/core/src/main/java/org/lflang/target/property/DockerProperty.java @@ -102,7 +102,7 @@ public Element toAstElement(DockerOptions value) { KeyValuePair pair = LfFactory.eINSTANCE.createKeyValuePair(); pair.setName(opt.toString()); switch (opt) { - case NO_BUILD -> pair.setValue(ASTUtils.toElement(value.build)); + case NO_BUILD -> pair.setValue(ASTUtils.toElement(value.noBuild)); case BUILDER_BASE -> pair.setValue(ASTUtils.toElement(value.builderBase)); case RUNNER_BASE -> pair.setValue(ASTUtils.toElement(value.runnerBase)); case PRE_BUILD_SCRIPT -> pair.setValue(ASTUtils.toElement(value.preBuildScript)); @@ -128,7 +128,7 @@ public String name() { /** Settings related to Docker options. */ public record DockerOptions( boolean enabled, - boolean build, + boolean noBuild, String builderBase, String runnerBase, String rti, diff --git a/test/C/src/docker/federated/DistributedCountContainerized.lf b/test/C/src/docker/federated/DistributedCountContainerized.lf index 7b6de5b2f2..dfcf9472cf 100644 --- a/test/C/src/docker/federated/DistributedCountContainerized.lf +++ b/test/C/src/docker/federated/DistributedCountContainerized.lf @@ -9,11 +9,7 @@ target C { logging: DEBUG, coordination: centralized, docker: { - rti-image: "rti:local", - pre-build-script: "foo.sh", - pre-run-script: "bar.sh", - post-build-script: "baz.sh", - no-build: false + rti-image: "rti:local" } } From 73c24f7e7a7a99921576cd4c6b2e571fd63c6a37 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Thu, 13 Jun 2024 20:24:11 -0700 Subject: [PATCH 15/48] Fix test --- .../compiler/LinguaFrancaValidationTest.java | 4 ++- test/C/src/docker/federated/DockerOptions.lf | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 test/C/src/docker/federated/DockerOptions.lf diff --git a/core/src/test/java/org/lflang/tests/compiler/LinguaFrancaValidationTest.java b/core/src/test/java/org/lflang/tests/compiler/LinguaFrancaValidationTest.java index 6132e7d0d1..8a51d4a027 100644 --- a/core/src/test/java/org/lflang/tests/compiler/LinguaFrancaValidationTest.java +++ b/core/src/test/java/org/lflang/tests/compiler/LinguaFrancaValidationTest.java @@ -1457,7 +1457,9 @@ public void recognizeHostNames() throws Exception { LfPackage.eINSTANCE.getKeyValuePair(), DictionaryType.DOCKER_DICT), List.of( - "{FROM: [1, 2, 3]}", LfPackage.eINSTANCE.getElement(), PrimitiveType.STRING)), + "{builder-base: [1, 2, 3]}", + LfPackage.eINSTANCE.getElement(), + PrimitiveType.STRING)), UnionType.TRACING_UNION, List.of( List.of("foo", LfPackage.eINSTANCE.getKeyValuePair(), UnionType.TRACING_UNION), diff --git a/test/C/src/docker/federated/DockerOptions.lf b/test/C/src/docker/federated/DockerOptions.lf new file mode 100644 index 0000000000..cd3a7bd382 --- /dev/null +++ b/test/C/src/docker/federated/DockerOptions.lf @@ -0,0 +1,29 @@ +/** + * Test a particularly simple form of a distributed deterministic system where a federation that + * receives timestamped messages has only those messages as triggers. Therefore, no additional + * coordination of the advancement of time (HLA or Ptides) is needed. + * @author Edward A. Lee + */ +target C { + timeout: 5 sec, + logging: DEBUG, + coordination: centralized, + docker: { + rti-image: "rti:local", + pre-build-script: "foo.sh", + pre-run-script: "bar.sh", + post-build-script: "baz.sh", + no-build: false + } +} + +reactor Hello { + reaction(startup) {= + printf("Hello World!"); + =} +} + +federated reactor { + a = new Hello() + b = new Hello() +} From c8cb5030c9165d97f9c43ab6ad3bec2a3c5e913a Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Fri, 14 Jun 2024 00:15:09 -0700 Subject: [PATCH 16/48] Handle federation id differently --- .../org/lflang/generator/docker/CDockerGenerator.java | 2 +- .../org/lflang/generator/docker/DockerGenerator.java | 11 +++++++---- .../generator/docker/FedDockerComposeGenerator.java | 1 - .../lflang/generator/docker/TSDockerGenerator.java | 2 +- test/C/src/docker/federated/DockerOptions.lf | 10 ++-------- test/C/src/docker/federated/bar.sh | 2 +- 6 files changed, 12 insertions(+), 16 deletions(-) mode change 100644 => 100755 test/C/src/docker/federated/bar.sh diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index 5d9fc4df66..f154f1a1a3 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -56,7 +56,7 @@ protected String generateDockerFileContent() { @Override public List defaultEntryPoint() { - return List.of("./bin/" + context.getFileConfig().name); + return List.of("./bin/" + context.getFileConfig().name, "-i", "1"); } @Override diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index 75dd92d4c6..a7cd618c62 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -106,10 +106,13 @@ protected String generateCopyOfScript() { protected List getEntryPointCommands() { var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preRunScript(); if (!script.isEmpty()) { - return Stream.concat( - List.of(DockerOptions.DEFAULT_SHELL, "-c", "source scripts/" + script).stream(), - defaultEntryPoint().stream()) - .toList(); + return List.of( + DockerOptions.DEFAULT_SHELL, + "-c", + "source scripts/" + + script + + " && " + + defaultEntryPoint().stream().collect(Collectors.joining(" "))); } return defaultEntryPoint(); } diff --git a/core/src/main/java/org/lflang/generator/docker/FedDockerComposeGenerator.java b/core/src/main/java/org/lflang/generator/docker/FedDockerComposeGenerator.java index 1e5deb3860..2eb298b2de 100644 --- a/core/src/main/java/org/lflang/generator/docker/FedDockerComposeGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/FedDockerComposeGenerator.java @@ -65,7 +65,6 @@ protected String generateDockerServices(List services) { protected String getServiceDescription(DockerData data) { return """ %s\ - command: "-i 1" depends_on: - rti """ diff --git a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java index bf0530e4c0..40333b31a4 100644 --- a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java @@ -29,7 +29,7 @@ public String generateDockerFileContent() { @Override public List defaultEntryPoint() { - return List.of("node", "dist/%s.js".formatted(context.getFileConfig().name)); + return List.of("node", "dist/%s.js".formatted(context.getFileConfig().name), "-i", "1"); } @Override diff --git a/test/C/src/docker/federated/DockerOptions.lf b/test/C/src/docker/federated/DockerOptions.lf index cd3a7bd382..8c74248597 100644 --- a/test/C/src/docker/federated/DockerOptions.lf +++ b/test/C/src/docker/federated/DockerOptions.lf @@ -1,12 +1,6 @@ -/** - * Test a particularly simple form of a distributed deterministic system where a federation that - * receives timestamped messages has only those messages as triggers. Therefore, no additional - * coordination of the advancement of time (HLA or Ptides) is needed. - * @author Edward A. Lee - */ target C { - timeout: 5 sec, logging: DEBUG, + timeout: 1 s, coordination: centralized, docker: { rti-image: "rti:local", @@ -19,7 +13,7 @@ target C { reactor Hello { reaction(startup) {= - printf("Hello World!"); + printf("Hello World!\n"); =} } diff --git a/test/C/src/docker/federated/bar.sh b/test/C/src/docker/federated/bar.sh old mode 100644 new mode 100755 index 60f774ffa7..e55421afd2 --- a/test/C/src/docker/federated/bar.sh +++ b/test/C/src/docker/federated/bar.sh @@ -1,2 +1,2 @@ #!/bin/bash -echo "Hello from pre-build script!" +echo "Hello from pre-run script!" From 73d69157762f47dd5001392cd89d6cbc34ad8865 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Fri, 14 Jun 2024 14:46:22 -0700 Subject: [PATCH 17/48] Do not add -i argument when unfederated --- .../java/org/lflang/generator/docker/CDockerGenerator.java | 7 ++++++- .../lflang/generator/docker/DockerComposeGenerator.java | 1 - .../org/lflang/generator/docker/TSDockerGenerator.java | 7 ++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index f154f1a1a3..03220d08ae 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -3,6 +3,7 @@ import java.util.List; import org.lflang.LocalStrings; import org.lflang.generator.LFGeneratorContext; +import org.lflang.generator.SubContext; import org.lflang.generator.c.CCompiler; import org.lflang.target.Target; @@ -56,7 +57,11 @@ protected String generateDockerFileContent() { @Override public List defaultEntryPoint() { - return List.of("./bin/" + context.getFileConfig().name, "-i", "1"); + if (context instanceof SubContext) { + return List.of("./bin/" + context.getFileConfig().name, "-i", "1"); + } else { + return List.of("./bin/" + context.getFileConfig().name); + } } @Override diff --git a/core/src/main/java/org/lflang/generator/docker/DockerComposeGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerComposeGenerator.java index 9b6f684f75..7fbdae0f3c 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerComposeGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerComposeGenerator.java @@ -46,7 +46,6 @@ protected String generateDockerNetwork(String networkName) { */ protected String generateDockerServices(List services) { return """ - version: "3.9" services: %s """ diff --git a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java index 40333b31a4..18e29a9b39 100644 --- a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java @@ -2,6 +2,7 @@ import java.util.List; import org.lflang.generator.LFGeneratorContext; +import org.lflang.generator.SubContext; /** * Generates the docker file related code for the Typescript target. @@ -29,7 +30,11 @@ public String generateDockerFileContent() { @Override public List defaultEntryPoint() { - return List.of("node", "dist/%s.js".formatted(context.getFileConfig().name), "-i", "1"); + if (context instanceof SubContext) { + return List.of("node", "dist/%s.js".formatted(context.getFileConfig().name), "-i", "1"); + } else { + return List.of("node", "dist/%s.js".formatted(context.getFileConfig().name)); + } } @Override From 3b0cd7eacdc552aeb2c88173145f3d054bcae3f6 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Sat, 15 Jun 2024 22:32:12 -0700 Subject: [PATCH 18/48] Fix paths in generated Dockerfile --- .../generator/docker/CDockerGenerator.java | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index 03220d08ae..14f2c78713 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -1,16 +1,20 @@ package org.lflang.generator.docker; +import java.io.IOException; +import java.nio.file.Path; import java.util.List; import org.lflang.LocalStrings; import org.lflang.generator.LFGeneratorContext; import org.lflang.generator.SubContext; import org.lflang.generator.c.CCompiler; +import org.lflang.generator.c.CFileConfig; import org.lflang.target.Target; /** * Generate the docker file related code for the C and CCpp target. * * @author Hou Seng Wong + * @author Marten Lohstroh */ public class CDockerGenerator extends DockerGenerator { @@ -77,19 +81,29 @@ protected String generateRunForInstallingDeps() { @Override protected List defaultBuildCommands() { - var ccompile = - new CCompiler( - context.getTargetConfig(), - context.getFileConfig(), - context.getErrorReporter(), - context.getTargetConfig().target == Target.CCPP); - - return List.of( - "mkdir -p bin", - String.format( - "%s -DCMAKE_INSTALL_BINDIR=./bin -S src-gen -B bin", ccompile.compileCmakeCommand()), - "cd bin", - "make all", - "cd .."); + try { + var ccompile = + new CCompiler( + context.getTargetConfig(), + new CFileConfig( + context.getFileConfig().resource, + Path.of("/lingua-franca", context.getFileConfig().name), + false), + context.getErrorReporter(), + context.getTargetConfig().target == Target.CCPP); + return List.of( + "mkdir -p bin", + String.format( + "%s -DCMAKE_INSTALL_BINDIR=./bin -S src-gen -B bin", ccompile.compileCmakeCommand()), + "cd bin", + "make all", + "cd .."); + } catch (IOException e) { + context + .getErrorReporter() + .nowhere() + .error("Unable to create file configuration for Docker container"); + throw new RuntimeException(e); + } } } From 4c667161fe46b3e7c4f03b741a96824cb36c9d23 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Sat, 15 Jun 2024 22:50:27 -0700 Subject: [PATCH 19/48] Got rid of some code duplication --- .../lflang/federated/generator/FedGenerator.java | 9 +-------- .../java/org/lflang/generator/c/CGenerator.java | 11 +---------- .../generator/docker/DockerComposeGenerator.java | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java index b89d453557..5ca1b22574 100644 --- a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java +++ b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java @@ -221,14 +221,7 @@ private void buildUsingDocker(LFGeneratorContext context, List subCo try { var dockerGen = new FedDockerComposeGenerator(context, rtiConfig.getHost()); dockerGen.writeDockerComposeFile(createDockerFiles(context, subContexts)); - // Only build if requested - if (!context.getTargetConfig().get(DockerProperty.INSTANCE).noBuild()) { - if (dockerGen.build()) { - dockerGen.createLauncher(); - } else { - context.getErrorReporter().nowhere().error("Docker build failed."); - } - } + dockerGen.buildIfRequested(); } catch (IOException e) { context .getErrorReporter() diff --git a/core/src/main/java/org/lflang/generator/c/CGenerator.java b/core/src/main/java/org/lflang/generator/c/CGenerator.java index 8fff49d1ef..5af154e12a 100644 --- a/core/src/main/java/org/lflang/generator/c/CGenerator.java +++ b/core/src/main/java/org/lflang/generator/c/CGenerator.java @@ -590,16 +590,7 @@ private boolean buildUsingDocker() { return false; } // Only build if requested - if (!context.getTargetConfig().get(DockerProperty.INSTANCE).noBuild()) { - var success = dockerCompose.build(); - if (success) { - dockerCompose.createLauncher(); - } else { - context.getErrorReporter().nowhere().error("Docker build failed."); - } - return success; - } - return true; + return dockerCompose.buildIfRequested(); } private void generateCodeFor(String lfModuleName) throws IOException { diff --git a/core/src/main/java/org/lflang/generator/docker/DockerComposeGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerComposeGenerator.java index 7fbdae0f3c..2255f3ce3b 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerComposeGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerComposeGenerator.java @@ -7,6 +7,7 @@ import java.util.Objects; import java.util.stream.Collectors; import org.lflang.generator.LFGeneratorContext; +import org.lflang.target.property.DockerProperty; import org.lflang.util.FileUtil; import org.lflang.util.LFCommand; @@ -152,4 +153,19 @@ public void createLauncher() { messageReporter.nowhere().warning("Unable to make launcher script executable."); } } + + /** + * Build, unless building was disabled. + * + * @return {@code false} if building failed, {@code true} otherwise + */ + public boolean buildIfRequested() { + if (!context.getTargetConfig().get(DockerProperty.INSTANCE).noBuild()) { + if (build()) { + createLauncher(); + } else context.getErrorReporter().nowhere().error("Docker build failed."); + return false; + } + return true; + } } From 8de5de4db208bb62e16acbad996bb06e92402cc5 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Sat, 15 Jun 2024 23:46:32 -0700 Subject: [PATCH 20/48] Various cleanups --- .../org/lflang/generator/c/CGenerator.java | 1 - .../generator/docker/CDockerGenerator.java | 12 ++-------- .../generator/docker/DockerGenerator.java | 23 +++++++++++++++++-- .../docker/PythonDockerGenerator.java | 21 +++++++++-------- .../generator/docker/TSDockerGenerator.java | 7 +----- 5 files changed, 35 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/c/CGenerator.java b/core/src/main/java/org/lflang/generator/c/CGenerator.java index 5af154e12a..9b0ca2e870 100644 --- a/core/src/main/java/org/lflang/generator/c/CGenerator.java +++ b/core/src/main/java/org/lflang/generator/c/CGenerator.java @@ -589,7 +589,6 @@ private boolean buildUsingDocker() { context.getErrorReporter().nowhere().error("Error while writing Docker files"); return false; } - // Only build if requested return dockerCompose.buildIfRequested(); } diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index 14f2c78713..3be09ad263 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -3,9 +3,7 @@ import java.io.IOException; import java.nio.file.Path; import java.util.List; -import org.lflang.LocalStrings; import org.lflang.generator.LFGeneratorContext; -import org.lflang.generator.SubContext; import org.lflang.generator.c.CCompiler; import org.lflang.generator.c.CFileConfig; import org.lflang.target.Target; @@ -40,9 +38,7 @@ protected String generateDockerFileContent() { var lfModuleName = context.getFileConfig().name; return String.join( "\n", - "# Generated by the Lingua Franca compiler version " + LocalStrings.VERSION, - "# - Docs: https://www.lf-lang.org/docs/handbook/containerized-execution", - "", + generateHeader(), "FROM " + builderBase() + " AS builder", "WORKDIR /lingua-franca/" + lfModuleName, generateRunForInstallingDeps(), @@ -61,11 +57,7 @@ protected String generateDockerFileContent() { @Override public List defaultEntryPoint() { - if (context instanceof SubContext) { - return List.of("./bin/" + context.getFileConfig().name, "-i", "1"); - } else { - return List.of("./bin/" + context.getFileConfig().name); - } + return List.of("./bin/" + context.getFileConfig().name); } @Override diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index a7cd618c62..6f227ddc05 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -4,7 +4,9 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.lflang.LocalStrings; import org.lflang.generator.LFGeneratorContext; +import org.lflang.generator.SubContext; import org.lflang.target.property.BuildCommandsProperty; import org.lflang.target.property.DockerProperty; import org.lflang.target.property.DockerProperty.DockerOptions; @@ -66,6 +68,15 @@ protected List getPostBuildCommand() { return List.of(); } + /** Generate a header to print at the top of the Dockerfile. */ + protected String generateHeader() { + return """ + # Generated by the Lingua Franca compiler version %s + # - Docs: https://www.lf-lang.org/docs/handbook/containerized-execution" + """ + .formatted(LocalStrings.VERSION); + } + /** Return the Docker RUN command used for building. */ protected String generateRunForBuild() { return "RUN " @@ -103,6 +114,14 @@ protected String generateCopyOfScript() { return "# (No pre-run script provided.)"; } + protected List entryPoint() { + if (context instanceof SubContext) { + return Stream.concat(defaultEntryPoint().stream(), List.of("-i", "1").stream()).toList(); + } else { + return defaultEntryPoint(); + } + } + protected List getEntryPointCommands() { var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preRunScript(); if (!script.isEmpty()) { @@ -112,9 +131,9 @@ protected List getEntryPointCommands() { "source scripts/" + script + " && " - + defaultEntryPoint().stream().collect(Collectors.joining(" "))); + + entryPoint().stream().collect(Collectors.joining(" "))); } - return defaultEntryPoint(); + return entryPoint(); } public abstract List defaultEntryPoint(); diff --git a/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java index 10eb85b50f..9ad1081bca 100644 --- a/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java @@ -1,5 +1,6 @@ package org.lflang.generator.docker; +import java.util.List; import org.lflang.generator.LFGeneratorContext; /** @@ -28,18 +29,18 @@ protected String generateRunForInstallingDeps() { } } - /** Generates the contents of the docker file. */ @Override - protected String generateDockerFileContent() { + protected String generateCopyOfExecutable() { + var lfModuleName = context.getFileConfig().name; return String.join( "\n", - "# For instructions, see:" - + " https://www.lf-lang.org/docs/handbook/containerized-execution?target=py", - "FROM " + builderBase(), // FIXME: create stages - "WORKDIR /lingua-franca/" + context.getFileConfig().name, - generateRunForInstallingDeps(), - "COPY . src-gen", - super.generateRunForBuild(), - "ENTRYPOINT [\"python3\", \"-u\", \"src-gen/" + context.getFileConfig().name + ".py\"]"); + super.generateCopyOfExecutable(), + "COPY --from=builder /lingua-franca/%s/src-gen ./src-gen" + .formatted(lfModuleName, lfModuleName, lfModuleName)); + } + + @Override + public List defaultEntryPoint() { + return List.of("python3", "-u", "src-gen/" + context.getFileConfig().name + ".py"); } } diff --git a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java index 18e29a9b39..bf0530e4c0 100644 --- a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java @@ -2,7 +2,6 @@ import java.util.List; import org.lflang.generator.LFGeneratorContext; -import org.lflang.generator.SubContext; /** * Generates the docker file related code for the Typescript target. @@ -30,11 +29,7 @@ public String generateDockerFileContent() { @Override public List defaultEntryPoint() { - if (context instanceof SubContext) { - return List.of("node", "dist/%s.js".formatted(context.getFileConfig().name), "-i", "1"); - } else { - return List.of("node", "dist/%s.js".formatted(context.getFileConfig().name)); - } + return List.of("node", "dist/%s.js".formatted(context.getFileConfig().name)); } @Override From 91e8d22373920cf0f6e36a2f2ba538dbb434bcbf Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Sun, 16 Jun 2024 16:00:57 -0700 Subject: [PATCH 21/48] Fix Docker support for TypeScript --- .../lflang/analyses/uclid/UclidGenerator.java | 6 +++ .../federated/generator/FedGenerator.java | 4 +- .../org/lflang/generator/GeneratorBase.java | 25 ++++++++++++ .../org/lflang/generator/c/CGenerator.java | 17 +------- .../generator/docker/CDockerGenerator.java | 23 ----------- .../generator/docker/DockerGenerator.java | 39 ++++++++++++++++--- .../generator/docker/TSDockerGenerator.java | 28 +++++++------ .../main/java/org/lflang/target/Target.java | 8 ---- .../java/org/lflang/target/TargetConfig.java | 5 +++ .../org/lflang/generator/cpp/CppGenerator.kt | 4 ++ .../lflang/generator/rust/RustGenerator.kt | 4 ++ .../org/lflang/generator/ts/TSGenerator.kt | 26 +++++++++---- 12 files changed, 114 insertions(+), 75 deletions(-) diff --git a/core/src/main/java/org/lflang/analyses/uclid/UclidGenerator.java b/core/src/main/java/org/lflang/analyses/uclid/UclidGenerator.java index 540a2b5c81..9253856fcc 100644 --- a/core/src/main/java/org/lflang/analyses/uclid/UclidGenerator.java +++ b/core/src/main/java/org/lflang/analyses/uclid/UclidGenerator.java @@ -70,6 +70,7 @@ import org.lflang.generator.TargetTypes; import org.lflang.generator.TimerInstance; import org.lflang.generator.TriggerInstance; +import org.lflang.generator.docker.DockerGenerator; import org.lflang.lf.AttrParm; import org.lflang.lf.Attribute; import org.lflang.lf.Connection; @@ -1744,4 +1745,9 @@ public TargetTypes getTargetTypes() { throw new UnsupportedOperationException( "This method is not applicable for this generator since Uclid5 is not an LF target."); } + + @Override + protected DockerGenerator getDockerGenerator(LFGeneratorContext context) { + return null; + } } diff --git a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java index 5ca1b22574..e1f36a9ad1 100644 --- a/core/src/main/java/org/lflang/federated/generator/FedGenerator.java +++ b/core/src/main/java/org/lflang/federated/generator/FedGenerator.java @@ -363,9 +363,7 @@ time allowed for the test expires (currently two hours). new TargetConfig( subFileConfig.resource, GeneratorArguments.none(), subContextMessageReporter); - if (targetConfig.get(DockerProperty.INSTANCE).enabled() - && targetConfig.target.buildsUsingDocker() - || fed.isRemote) { + if (targetConfig.get(DockerProperty.INSTANCE).enabled() || fed.isRemote) { NoCompileProperty.INSTANCE.override(subConfig, true); } // Disabled Docker for the federate and put federation in charge. diff --git a/core/src/main/java/org/lflang/generator/GeneratorBase.java b/core/src/main/java/org/lflang/generator/GeneratorBase.java index ea99722a9a..b585bb9233 100644 --- a/core/src/main/java/org/lflang/generator/GeneratorBase.java +++ b/core/src/main/java/org/lflang/generator/GeneratorBase.java @@ -51,6 +51,8 @@ import org.lflang.analyses.uclid.UclidGenerator; import org.lflang.ast.ASTUtils; import org.lflang.ast.AstTransformation; +import org.lflang.generator.docker.DockerComposeGenerator; +import org.lflang.generator.docker.DockerGenerator; import org.lflang.graph.InstantiationGraph; import org.lflang.lf.Attribute; import org.lflang.lf.Connection; @@ -627,6 +629,29 @@ protected void cleanIfNeeded(LFGeneratorContext context) { } } + /** + * Get the Docker generator. + * + * @param context + * @return + */ + protected abstract DockerGenerator getDockerGenerator(LFGeneratorContext context); + + /** Create Dockerfiles and docker-compose.yml, build, and create a launcher. */ + protected boolean buildUsingDocker() { + // Create docker file. + var dockerCompose = new DockerComposeGenerator(context); + var dockerData = getDockerGenerator(context).generateDockerData(); + try { + dockerData.writeDockerFile(); + dockerCompose.writeDockerComposeFile(List.of(dockerData)); + } catch (IOException e) { + context.getErrorReporter().nowhere().error("Error while writing Docker files"); + return false; + } + return dockerCompose.buildIfRequested(); + } + /** * Check if @property is used. If so, instantiate a UclidGenerator. The verification model needs * to be generated before the target code since code generation changes LF program (desugar diff --git a/core/src/main/java/org/lflang/generator/c/CGenerator.java b/core/src/main/java/org/lflang/generator/c/CGenerator.java index 9b0ca2e870..58624b353d 100644 --- a/core/src/main/java/org/lflang/generator/c/CGenerator.java +++ b/core/src/main/java/org/lflang/generator/c/CGenerator.java @@ -69,7 +69,6 @@ import org.lflang.generator.TimerInstance; import org.lflang.generator.TriggerInstance; import org.lflang.generator.docker.CDockerGenerator; -import org.lflang.generator.docker.DockerComposeGenerator; import org.lflang.generator.docker.DockerGenerator; import org.lflang.generator.python.PythonGenerator; import org.lflang.lf.Action; @@ -577,21 +576,6 @@ public void doGenerate(Resource resource, LFGeneratorContext context) { GeneratorUtils.refreshProject(resource, context.getMode()); } - /** Create Dockerfiles and docker-compose.yml, build, and create a launcher. */ - private boolean buildUsingDocker() { - // Create docker file. - var dockerCompose = new DockerComposeGenerator(context); - var dockerData = getDockerGenerator(context).generateDockerData(); - try { - dockerData.writeDockerFile(); - dockerCompose.writeDockerComposeFile(List.of(dockerData)); - } catch (IOException e) { - context.getErrorReporter().nowhere().error("Error while writing Docker files"); - return false; - } - return dockerCompose.buildIfRequested(); - } - private void generateCodeFor(String lfModuleName) throws IOException { code.pr(generateDirectives()); code.pr(new CMainFunctionGenerator(targetConfig).generateCode()); @@ -1960,6 +1944,7 @@ public TargetTypes getTargetTypes() { * @param context * @return */ + @Override protected DockerGenerator getDockerGenerator(LFGeneratorContext context) { return new CDockerGenerator(context); } diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index 3be09ad263..afe67d8147 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -32,29 +32,6 @@ public String defaultImage() { return DEFAULT_BASE_IMAGE; } - /** Generate the contents of the docker file. */ - @Override - protected String generateDockerFileContent() { - var lfModuleName = context.getFileConfig().name; - return String.join( - "\n", - generateHeader(), - "FROM " + builderBase() + " AS builder", - "WORKDIR /lingua-franca/" + lfModuleName, - generateRunForInstallingDeps(), - "COPY . src-gen", - generateRunForBuild(), - "", - "FROM " + runnerBase(), - "WORKDIR /lingua-franca", - "RUN mkdir scripts", - generateCopyOfScript(), - "RUN mkdir bin", - generateCopyOfExecutable(), - generateEntryPoint(), - ""); - } - @Override public List defaultEntryPoint() { return List.of("./bin/" + context.getFileConfig().name); diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index 6f227ddc05..e32f100cbf 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -32,8 +32,35 @@ public DockerGenerator(LFGeneratorContext context) { this.context = context; } - /** Generate the contents of a Dockerfile. */ - protected abstract String generateDockerFileContent(); + /** Generate the contents of the docker file. */ + protected String generateDockerFileContent() { + var lfModuleName = context.getFileConfig().name; + return String.join( + "\n", + generateHeader(), + "FROM " + builderBase() + " AS builder", + "WORKDIR /lingua-franca/" + lfModuleName, + generateRunForInstallingDeps(), + generateCopyForSources(), + generateRunForBuild(), + "", + "FROM " + runnerBase(), + "WORKDIR /lingua-franca", + "RUN mkdir scripts", + generateCopyOfScript(), + generateRunForMkdir(), + generateCopyOfExecutable(), + generateEntryPoint(), + ""); + } + + protected String generateRunForMkdir() { + return "RUN mkdir bin"; + } + + protected String generateCopyForSources() { + return "COPY . src-gen"; + } /** Return a RUN command for installing/checking build dependencies. */ protected abstract String generateRunForInstallingDeps(); @@ -43,9 +70,11 @@ public DockerGenerator(LFGeneratorContext context) { /** Return the commands used to build */ protected List getBuildCommands() { - var customBuildCommands = context.getTargetConfig().get(BuildCommandsProperty.INSTANCE); - if (customBuildCommands != null && !customBuildCommands.isEmpty()) { - return customBuildCommands; + if (context.getTargetConfig().isSupported(BuildCommandsProperty.INSTANCE)) { + var customBuildCommands = context.getTargetConfig().get(BuildCommandsProperty.INSTANCE); + if (customBuildCommands != null && !customBuildCommands.isEmpty()) { + return customBuildCommands; + } } return defaultBuildCommands(); } diff --git a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java index bf0530e4c0..0769aa4571 100644 --- a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java @@ -15,16 +15,20 @@ public TSDockerGenerator(LFGeneratorContext context) { super(context); } - /** Return the content of the docker file for [tsFileName]. */ - public String generateDockerFileContent() { - return """ - FROM %s - WORKDIR /linguafranca/$name - %s - COPY . . - %s - """ - .formatted(builderBase(), generateRunForInstallingDeps(), generateEntryPoint()); + @Override + protected String generateCopyOfExecutable() { + var lfModuleName = context.getFileConfig().name; + return "COPY --from=builder /lingua-franca/%s .".formatted(lfModuleName); + } + + @Override + protected String generateRunForMkdir() { + return "RUN mkdir dist"; + } + + @Override + protected String generateCopyForSources() { + return "COPY . ."; } @Override @@ -34,12 +38,12 @@ public List defaultEntryPoint() { @Override protected String generateRunForInstallingDeps() { - return "RUN which node && node --version"; + return "RUN apk add git && npm install -g pnpm"; } @Override protected List defaultBuildCommands() { - return List.of("pnpm install"); // FIXME: actually build using docker, not natively. + return List.of("pnpm install && pnpm run build"); } @Override diff --git a/core/src/main/java/org/lflang/target/Target.java b/core/src/main/java/org/lflang/target/Target.java index 6467023621..d30e7ff982 100644 --- a/core/src/main/java/org/lflang/target/Target.java +++ b/core/src/main/java/org/lflang/target/Target.java @@ -497,14 +497,6 @@ public boolean supportsReactionDeclarations() { return this.equals(Target.C) || this.equals(Target.CPP); } - /** Return true if this code for this target should be built using Docker if Docker is used. */ - public boolean buildsUsingDocker() { - return switch (this) { - case TS -> false; - case C, CCPP, CPP, Python, Rust -> true; - }; - } - /** * Whether the target requires using an equal sign to assign a default value to a parameter, or * initialize a state variable. All targets mandate an equal sign when passing arguments to a diff --git a/core/src/main/java/org/lflang/target/TargetConfig.java b/core/src/main/java/org/lflang/target/TargetConfig.java index f3dc1695f6..a5ce42c6cc 100644 --- a/core/src/main/java/org/lflang/target/TargetConfig.java +++ b/core/src/main/java/org/lflang/target/TargetConfig.java @@ -170,6 +170,11 @@ protected void load(JsonObject jsonObject, MessageReporter messageReporter) { } } + /** Return {@code true} if the given target property is supported, {@code false} otherwise. */ + public boolean isSupported(TargetProperty p) { + return properties.containsKey(p); + } + /** * Report that a target property is not supported by the current target. * diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt index 204b6e9324..0f1645b856 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt @@ -31,6 +31,7 @@ import org.lflang.target.Target import org.lflang.generator.* import org.lflang.generator.GeneratorUtils.canGenerate import org.lflang.generator.LFGeneratorContext.Mode +import org.lflang.generator.docker.DockerGenerator import org.lflang.isGeneric import org.lflang.scoping.LFGlobalScopeProvider import org.lflang.target.property.* @@ -186,5 +187,8 @@ class CppGenerator( override fun getTarget() = Target.CPP override fun getTargetTypes(): TargetTypes = CppTypes + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator { + TODO("Not yet implemented") + } } diff --git a/core/src/main/kotlin/org/lflang/generator/rust/RustGenerator.kt b/core/src/main/kotlin/org/lflang/generator/rust/RustGenerator.kt index b64d849945..b0d275d48a 100644 --- a/core/src/main/kotlin/org/lflang/generator/rust/RustGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/rust/RustGenerator.kt @@ -28,6 +28,7 @@ import org.eclipse.emf.ecore.resource.Resource import org.lflang.target.Target import org.lflang.generator.* import org.lflang.generator.GeneratorUtils.canGenerate +import org.lflang.generator.docker.DockerGenerator import org.lflang.joinWithCommas import org.lflang.scoping.LFGlobalScopeProvider import org.lflang.target.property.* @@ -148,5 +149,8 @@ class RustGenerator( override fun getTarget(): Target = Target.Rust override fun getTargetTypes(): TargetTypes = RustTypes + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator { + TODO("Not yet implemented") + } } diff --git a/core/src/main/kotlin/org/lflang/generator/ts/TSGenerator.kt b/core/src/main/kotlin/org/lflang/generator/ts/TSGenerator.kt index 0404f5818b..91db8810e1 100644 --- a/core/src/main/kotlin/org/lflang/generator/ts/TSGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/ts/TSGenerator.kt @@ -28,15 +28,17 @@ package org.lflang.generator.ts import com.google.common.base.Strings import org.eclipse.emf.ecore.resource.Resource import org.eclipse.xtext.util.CancelIndicator -import org.lflang.target.Target import org.lflang.TimeValue import org.lflang.ast.DelayedConnectionTransformation import org.lflang.generator.* import org.lflang.generator.GeneratorUtils.canGenerate +import org.lflang.generator.docker.CDockerGenerator import org.lflang.generator.docker.DockerComposeGenerator +import org.lflang.generator.docker.DockerGenerator import org.lflang.generator.docker.TSDockerGenerator import org.lflang.lf.Preamble import org.lflang.model +import org.lflang.target.Target import org.lflang.target.property.DockerProperty import org.lflang.target.property.NoCompileProperty import org.lflang.target.property.ProtobufsProperty @@ -44,8 +46,7 @@ import org.lflang.target.property.RuntimeVersionProperty import org.lflang.util.FileUtil import java.nio.file.Files import java.nio.file.Path -import java.util.LinkedList -import kotlin.collections.HashMap +import java.util.* private const val NO_NPM_MESSAGE = "The TypeScript target requires npm >= 6.14.4. " + "For installation instructions, see: https://www.npmjs.com/get-npm. \n" + @@ -98,6 +99,10 @@ class TSGenerator( } + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator { + return TSDockerGenerator(context) + } + /** Generate TypeScript code from the Lingua Franca model contained by the * specified resource. This is the main entry point for code * generation. @@ -156,6 +161,8 @@ class TSGenerator( return; } context.finish(GeneratorResult.GENERATED_NO_EXECUTABLE.apply(context, codeMaps)) + } else if (targetConfig.get(DockerProperty.INSTANCE).enabled) { + buildUsingDocker() } else { compile(validator, resource, parsingContext) concludeCompilation(context, codeMaps) @@ -279,10 +286,13 @@ class TSGenerator( /** * Return whether it is advisable to install dependencies. */ - private fun shouldCollectDependencies(context: LFGeneratorContext): Boolean = - (context.mode != LFGeneratorContext.Mode.LSP_MEDIUM - && !targetConfig.get(NoCompileProperty.INSTANCE)) - || !fileConfig.srcGenPkgPath.resolve("node_modules").toFile().exists() + private fun shouldCollectDependencies(context: LFGeneratorContext): Boolean { + if (targetConfig.get(NoCompileProperty.INSTANCE) || targetConfig.get(DockerProperty.INSTANCE).enabled) { + return false; + } + return ((context.mode != LFGeneratorContext.Mode.LSP_MEDIUM && !targetConfig.get(NoCompileProperty.INSTANCE) + || !fileConfig.srcGenPkgPath.resolve("node_modules").toFile().exists())) + } /** * Collect the dependencies in package.json and their @@ -445,7 +455,7 @@ class TSGenerator( val jsPath = fileConfig.srcGenPath.resolve("dist").resolve("${fileConfig.name}.js") FileUtil.writeToFile("#!/bin/sh\nnode $jsPath", shScriptPath) shScriptPath.toFile().setExecutable(true) - messageReporter.nowhere().info("Script for executing the compiled program: $shScriptPath.") + messageReporter.nowhere().info("Script for running the program: $shScriptPath.") } } From 070ea6fd0c0321a4556d750fa36ebc5563f27234 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Sun, 16 Jun 2024 16:42:34 -0700 Subject: [PATCH 22/48] Comments --- .../org/lflang/generator/GeneratorBase.java | 7 +----- .../generator/docker/DockerGenerator.java | 22 ++++++++++++++++--- .../generator/docker/TSDockerGenerator.java | 4 ++-- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/GeneratorBase.java b/core/src/main/java/org/lflang/generator/GeneratorBase.java index b585bb9233..91917c2345 100644 --- a/core/src/main/java/org/lflang/generator/GeneratorBase.java +++ b/core/src/main/java/org/lflang/generator/GeneratorBase.java @@ -629,12 +629,7 @@ protected void cleanIfNeeded(LFGeneratorContext context) { } } - /** - * Get the Docker generator. - * - * @param context - * @return - */ + /** Return a {@code DockerGenerator} instance suitable for the target. */ protected abstract DockerGenerator getDockerGenerator(LFGeneratorContext context); /** Create Dockerfiles and docker-compose.yml, build, and create a launcher. */ diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index e32f100cbf..6ee2847ead 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -48,16 +48,18 @@ protected String generateDockerFileContent() { "WORKDIR /lingua-franca", "RUN mkdir scripts", generateCopyOfScript(), - generateRunForMkdir(), + generateRunForMakingExecutableDir(), generateCopyOfExecutable(), generateEntryPoint(), ""); } - protected String generateRunForMkdir() { + /** Return a RUN command for making a directory to place executables in. */ + protected String generateRunForMakingExecutableDir() { return "RUN mkdir bin"; } + /** Return a COPY command for copying sources from host into container. */ protected String generateCopyForSources() { return "COPY . src-gen"; } @@ -120,6 +122,7 @@ protected String generateRunForBuild() { " \\\n\t&& "); } + /** Return the ENTRYPOINT command. */ protected String generateEntryPoint() { return "ENTRYPOINT [" + getEntryPointCommands().stream() @@ -128,12 +131,14 @@ protected String generateEntryPoint() { + "]"; } + /** Return a COPY command to copy the executable from the builder to the runner. */ protected String generateCopyOfExecutable() { var lfModuleName = context.getFileConfig().name; return "COPY --from=builder /lingua-franca/%s/bin/%s ./bin/%s" .formatted(lfModuleName, lfModuleName, lfModuleName); } + /** Return a COPY command to copy the scripts from the builder to the runner. */ protected String generateCopyOfScript() { var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preRunScript(); if (!script.isEmpty()) { @@ -143,6 +148,10 @@ protected String generateCopyOfScript() { return "# (No pre-run script provided.)"; } + /** + * Return a list of strings used to construct and entrypoint. If this is done for a federate, then + * also include additional parameters to pass in the federate ID. + */ protected List entryPoint() { if (context instanceof SubContext) { return Stream.concat(defaultEntryPoint().stream(), List.of("-i", "1").stream()).toList(); @@ -151,7 +160,11 @@ protected List entryPoint() { } } - protected List getEntryPointCommands() { + /** + * Return a list of commands to be used to construct an ENTRYPOINT, taking into account the + * existence of a possible pre-run script. + */ + protected final List getEntryPointCommands() { var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preRunScript(); if (!script.isEmpty()) { return List.of( @@ -165,15 +178,18 @@ protected List getEntryPointCommands() { return entryPoint(); } + /** The default list of commands to construct an ENTRYPOINT out of. Different for each target. */ public abstract List defaultEntryPoint(); /** Return the default base image. */ public abstract String defaultImage(); + /** Return the base image to be used during the building stage. */ protected String builderBase() { return baseImage(context.getTargetConfig().get(DockerProperty.INSTANCE).builderBase()); } + /** Return the base image to be used during the running stage. */ protected String runnerBase() { return baseImage(context.getTargetConfig().get(DockerProperty.INSTANCE).runnerBase()); } diff --git a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java index 0769aa4571..d369634fa0 100644 --- a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java @@ -22,7 +22,7 @@ protected String generateCopyOfExecutable() { } @Override - protected String generateRunForMkdir() { + protected String generateRunForMakingExecutableDir() { return "RUN mkdir dist"; } @@ -43,7 +43,7 @@ protected String generateRunForInstallingDeps() { @Override protected List defaultBuildCommands() { - return List.of("pnpm install && pnpm run build"); + return List.of("pnpm install", "pnpm run build"); } @Override From 0ca3d1bbca065d4497a1ce4e7fa7acbd8413faf5 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Sun, 16 Jun 2024 17:52:05 -0700 Subject: [PATCH 23/48] First steps towards Docker support for C++ --- .../generator/docker/CDockerGenerator.java | 2 +- .../generator/docker/CppDockerGenerator.java | 48 +++++++++++++++++++ .../generator/docker/TSDockerGenerator.java | 1 + .../main/java/org/lflang/target/Target.java | 1 + .../org/lflang/generator/cpp/CppGenerator.kt | 25 ++++++---- .../Cpp/src/docker/HelloWorldContainerized.lf | 12 +++++ 6 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java create mode 100644 test/Cpp/src/docker/HelloWorldContainerized.lf diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index afe67d8147..e8c114a98c 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -40,7 +40,7 @@ public List defaultEntryPoint() { @Override protected String generateRunForInstallingDeps() { var config = context.getTargetConfig(); - var compiler = config.target == Target.CCPP ? "g++" : "gcc"; + var compiler = config.target == Target.CCPP || config.target == Target.CPP ? "g++" : "gcc"; if (builderBase().equals(defaultImage())) { return "RUN set -ex && apk add --no-cache %s musl-dev cmake make".formatted(compiler); } else { diff --git a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java new file mode 100644 index 0000000000..efe89a78e8 --- /dev/null +++ b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java @@ -0,0 +1,48 @@ +package org.lflang.generator.docker; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +import org.lflang.generator.LFGeneratorContext; +import org.lflang.generator.c.CCompiler; +import org.lflang.generator.c.CFileConfig; +import org.lflang.generator.cpp.CppGenerator; +import org.lflang.generator.cpp.CppPlatformGenerator; +import org.lflang.target.Target; + +/** + * Generates the docker file related code for the Typescript target. + * + * @author Marten Lohstroh + */ +public class CppDockerGenerator extends CDockerGenerator { + + private CppPlatformGenerator platformGenerator; + + /** Construct a new Docker generator. */ + public CppDockerGenerator(LFGeneratorContext context, CppPlatformGenerator platformGenerator) { + super(context); + this.platformGenerator = platformGenerator; + } + + @Override + protected List defaultBuildCommands() { + try { + return List.of( + "mkdir -p bin", + String.format( + "%s -DCMAKE_INSTALL_BINDIR=./bin -S src-gen -B bin", platformGenerator), + "cd bin", + "make all", + "cd .."); + } catch (IOException e) { + context + .getErrorReporter() + .nowhere() + .error("Unable to create file configuration for Docker container"); + throw new RuntimeException(e); + } + } + +} diff --git a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java index d369634fa0..1e81439886 100644 --- a/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/TSDockerGenerator.java @@ -7,6 +7,7 @@ * Generates the docker file related code for the Typescript target. * * @author Hou Seng Wong + * @author Marten Lohstroh */ public class TSDockerGenerator extends DockerGenerator { diff --git a/core/src/main/java/org/lflang/target/Target.java b/core/src/main/java/org/lflang/target/Target.java index d30e7ff982..30048f5a58 100644 --- a/core/src/main/java/org/lflang/target/Target.java +++ b/core/src/main/java/org/lflang/target/Target.java @@ -610,6 +610,7 @@ public void initialize(TargetConfig config) { BuildTypeProperty.INSTANCE, CmakeIncludeProperty.INSTANCE, CompilerProperty.INSTANCE, + DockerProperty.INSTANCE, ExportDependencyGraphProperty.INSTANCE, ExportToYamlProperty.INSTANCE, ExternalRuntimePathProperty.INSTANCE, diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt index 0f1645b856..afa377f76c 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt @@ -27,13 +27,14 @@ package org.lflang.generator.cpp import org.eclipse.emf.ecore.resource.Resource -import org.lflang.target.Target import org.lflang.generator.* import org.lflang.generator.GeneratorUtils.canGenerate import org.lflang.generator.LFGeneratorContext.Mode +import org.lflang.generator.docker.CppDockerGenerator import org.lflang.generator.docker.DockerGenerator import org.lflang.isGeneric import org.lflang.scoping.LFGlobalScopeProvider +import org.lflang.target.Target import org.lflang.target.property.* import org.lflang.util.FileUtil import java.nio.file.Files @@ -59,6 +60,7 @@ class CppGenerator( const val MINIMUM_CMAKE_VERSION = "3.5" const val CPP_VERSION = "20" + } override fun doGenerate(resource: Resource, context: LFGeneratorContext) { @@ -67,8 +69,7 @@ class CppGenerator( if (!canGenerate(errorsOccurred(), mainDef, messageReporter, context)) return // create a platform-specific generator - val platformGenerator: CppPlatformGenerator = - if (targetConfig.get(Ros2Property.INSTANCE)) CppRos2Generator(this) else CppStandaloneGenerator(this) + val platformGenerator: CppPlatformGenerator = getPlatformGenerator() // generate all core files generateFiles(platformGenerator.srcGenPath, getAllImportedResources(resource)) @@ -83,7 +84,6 @@ class CppGenerator( context.reportProgress( "Code generation complete. Validating generated code...", IntegratedBuilder.GENERATED_PERCENT_PROGRESS ) - if (platformGenerator.doCompile(context)) { CppValidator(fileConfig, messageReporter, codeMaps).doValidate(context) context.finish(GeneratorResult.GENERATED_NO_EXECUTABLE.apply(context, codeMaps)) @@ -94,10 +94,14 @@ class CppGenerator( context.reportProgress( "Code generation complete. Compiling...", IntegratedBuilder.GENERATED_PERCENT_PROGRESS ) - if (platformGenerator.doCompile(context)) { - context.finish(GeneratorResult.Status.COMPILED, codeMaps) + if (targetConfig.get(DockerProperty.INSTANCE).enabled) { + buildUsingDocker(); } else { - context.unsuccessfulFinish() + if (platformGenerator.doCompile(context)) { + context.finish(GeneratorResult.Status.COMPILED, codeMaps) + } else { + context.unsuccessfulFinish() + } } } } @@ -184,11 +188,12 @@ class CppGenerator( } } + private fun getPlatformGenerator() = if (targetConfig.get(Ros2Property.INSTANCE)) CppRos2Generator(this) else CppStandaloneGenerator(this) + override fun getTarget() = Target.CPP override fun getTargetTypes(): TargetTypes = CppTypes - override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator { - TODO("Not yet implemented") - } + + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator = CppDockerGenerator(context, getPlatformGenerator()) } diff --git a/test/Cpp/src/docker/HelloWorldContainerized.lf b/test/Cpp/src/docker/HelloWorldContainerized.lf new file mode 100644 index 0000000000..85cdcc195e --- /dev/null +++ b/test/Cpp/src/docker/HelloWorldContainerized.lf @@ -0,0 +1,12 @@ +target Cpp { + // To test generating a custom trace file name. + logging: error, + docker: true, + build-type: Debug +} + +import HelloWorld2 from "../HelloWorld.lf" + +main reactor { + a = new HelloWorld2() +} From b58903ffe5d8353d72f66906a12ecbdbd73a2d7c Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Sun, 16 Jun 2024 18:08:44 -0700 Subject: [PATCH 24/48] Added FIXME --- .../generator/docker/CppDockerGenerator.java | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java index efe89a78e8..c445765048 100644 --- a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java @@ -1,15 +1,8 @@ package org.lflang.generator.docker; -import java.io.IOException; -import java.nio.file.Path; import java.util.List; - import org.lflang.generator.LFGeneratorContext; -import org.lflang.generator.c.CCompiler; -import org.lflang.generator.c.CFileConfig; -import org.lflang.generator.cpp.CppGenerator; import org.lflang.generator.cpp.CppPlatformGenerator; -import org.lflang.target.Target; /** * Generates the docker file related code for the Typescript target. @@ -28,21 +21,7 @@ public CppDockerGenerator(LFGeneratorContext context, CppPlatformGenerator platf @Override protected List defaultBuildCommands() { - try { - return List.of( - "mkdir -p bin", - String.format( - "%s -DCMAKE_INSTALL_BINDIR=./bin -S src-gen -B bin", platformGenerator), - "cd bin", - "make all", - "cd .."); - } catch (IOException e) { - context - .getErrorReporter() - .nowhere() - .error("Unable to create file configuration for Docker container"); - throw new RuntimeException(e); - } + // FIXME + return List.of(); } - } From de08638cae9564090b4c961106b7e8579d162065 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Mon, 17 Jun 2024 15:41:20 -0700 Subject: [PATCH 25/48] Apply suggestions from code review Co-authored-by: Peter Donovan <33707478+petervdonovan@users.noreply.github.com> --- core/src/main/java/org/lflang/generator/GeneratorBase.java | 2 +- .../java/org/lflang/generator/docker/DockerGenerator.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/GeneratorBase.java b/core/src/main/java/org/lflang/generator/GeneratorBase.java index 91917c2345..4f2c79506c 100644 --- a/core/src/main/java/org/lflang/generator/GeneratorBase.java +++ b/core/src/main/java/org/lflang/generator/GeneratorBase.java @@ -641,7 +641,7 @@ protected boolean buildUsingDocker() { dockerData.writeDockerFile(); dockerCompose.writeDockerComposeFile(List.of(dockerData)); } catch (IOException e) { - context.getErrorReporter().nowhere().error("Error while writing Docker files"); + context.getErrorReporter().nowhere().error("Error while writing Docker files: " + (e.getMessage() == null ? "No cause given" : e.getMessage())); return false; } return dockerCompose.buildIfRequested(); diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index 6ee2847ead..d3fa1ad3fd 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -134,7 +134,7 @@ protected String generateEntryPoint() { /** Return a COPY command to copy the executable from the builder to the runner. */ protected String generateCopyOfExecutable() { var lfModuleName = context.getFileConfig().name; - return "COPY --from=builder /lingua-franca/%s/bin/%s ./bin/%s" + return "COPY --from=builder /lingua-franca/%s/bin/%s ./bin/%s" // safe because context.getFileConfig().name never contains spaces .formatted(lfModuleName, lfModuleName, lfModuleName); } @@ -150,7 +150,7 @@ protected String generateCopyOfScript() { /** * Return a list of strings used to construct and entrypoint. If this is done for a federate, then - * also include additional parameters to pass in the federate ID. + * also include additional parameters to pass in the federation ID. */ protected List entryPoint() { if (context instanceof SubContext) { From e911799cd97f742c16b2bd0311a9c4c0ce746aa3 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Mon, 17 Jun 2024 16:08:58 -0700 Subject: [PATCH 26/48] Ensure that scripts are also copied for unfederated dockerized programs --- core/src/main/java/org/lflang/generator/GeneratorBase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/org/lflang/generator/GeneratorBase.java b/core/src/main/java/org/lflang/generator/GeneratorBase.java index 4f2c79506c..25e8d1efdb 100644 --- a/core/src/main/java/org/lflang/generator/GeneratorBase.java +++ b/core/src/main/java/org/lflang/generator/GeneratorBase.java @@ -639,6 +639,7 @@ protected boolean buildUsingDocker() { var dockerData = getDockerGenerator(context).generateDockerData(); try { dockerData.writeDockerFile(); + dockerData.copyScripts(context); dockerCompose.writeDockerComposeFile(List.of(dockerData)); } catch (IOException e) { context.getErrorReporter().nowhere().error("Error while writing Docker files: " + (e.getMessage() == null ? "No cause given" : e.getMessage())); From fdc08548e3abb1f04c1c4fbbdf7006c9a6a33c0b Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Mon, 17 Jun 2024 16:20:51 -0700 Subject: [PATCH 27/48] Let runner-base default to builder-base --- .../java/org/lflang/generator/GeneratorBase.java | 7 ++++++- .../lflang/generator/docker/DockerGenerator.java | 15 ++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/GeneratorBase.java b/core/src/main/java/org/lflang/generator/GeneratorBase.java index 25e8d1efdb..6911f5a4ed 100644 --- a/core/src/main/java/org/lflang/generator/GeneratorBase.java +++ b/core/src/main/java/org/lflang/generator/GeneratorBase.java @@ -642,7 +642,12 @@ protected boolean buildUsingDocker() { dockerData.copyScripts(context); dockerCompose.writeDockerComposeFile(List.of(dockerData)); } catch (IOException e) { - context.getErrorReporter().nowhere().error("Error while writing Docker files: " + (e.getMessage() == null ? "No cause given" : e.getMessage())); + context + .getErrorReporter() + .nowhere() + .error( + "Error while writing Docker files: " + + (e.getMessage() == null ? "No cause given" : e.getMessage())); return false; } return dockerCompose.buildIfRequested(); diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index d3fa1ad3fd..4ff92f2ba1 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -134,7 +134,8 @@ protected String generateEntryPoint() { /** Return a COPY command to copy the executable from the builder to the runner. */ protected String generateCopyOfExecutable() { var lfModuleName = context.getFileConfig().name; - return "COPY --from=builder /lingua-franca/%s/bin/%s ./bin/%s" // safe because context.getFileConfig().name never contains spaces + // safe becaused context.getFileConfig().name never contains spaces + return "COPY --from=builder /lingua-franca/%s/bin/%s ./bin/%s" .formatted(lfModuleName, lfModuleName, lfModuleName); } @@ -186,20 +187,24 @@ protected final List getEntryPointCommands() { /** Return the base image to be used during the building stage. */ protected String builderBase() { - return baseImage(context.getTargetConfig().get(DockerProperty.INSTANCE).builderBase()); + return baseImage( + context.getTargetConfig().get(DockerProperty.INSTANCE).builderBase(), defaultImage()); } /** Return the base image to be used during the running stage. */ protected String runnerBase() { - return baseImage(context.getTargetConfig().get(DockerProperty.INSTANCE).runnerBase()); + return baseImage( + context.getTargetConfig().get(DockerProperty.INSTANCE).runnerBase(), + baseImage( + context.getTargetConfig().get(DockerProperty.INSTANCE).builderBase(), defaultImage())); } /** Return the selected base image, or the default one if none was selected. */ - private String baseImage(String name) { + private String baseImage(String name, String defaultImage) { if (name != null && !name.isEmpty()) { return name; } - return defaultImage(); + return defaultImage; } /** From 151d540279ffe8d2980936c780dec9f223c8f0f8 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Mon, 17 Jun 2024 17:39:31 -0700 Subject: [PATCH 28/48] Escape filenames referenced in Dockerfile --- core/build.gradle | 1 + .../generator/docker/DockerGenerator.java | 15 ++++++++------- gradle.properties | 1 + .../src/docker/federated/EscapedScriptName.lf | 19 +++++++++++++++++++ test/C/src/docker/federated/foo ish.sh | 2 ++ 5 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 test/C/src/docker/federated/EscapedScriptName.lf create mode 100644 test/C/src/docker/federated/foo ish.sh diff --git a/core/build.gradle b/core/build.gradle index 2979ab494b..f35dcff7f8 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -23,6 +23,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-core:$fasterxmlVersion" implementation "com.fasterxml.jackson.core:jackson-annotations:$fasterxmlVersion" implementation "com.fasterxml.jackson.core:jackson-databind:$fasterxmlVersion" + implementation "org.apache.commons:commons-text:$commonsTextVersion" implementation ("de.cau.cs.kieler.klighd:de.cau.cs.kieler.klighd.lsp:$klighdVersion") { exclude group: 'org.eclipse.platform', module: 'org.eclipse.swt.*' diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index 4ff92f2ba1..f3001ef882 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.commons.text.StringEscapeUtils; import org.lflang.LocalStrings; import org.lflang.generator.LFGeneratorContext; import org.lflang.generator.SubContext; @@ -85,7 +86,7 @@ protected List getBuildCommands() { protected List getPreBuildCommand() { var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preBuildScript(); if (!script.isEmpty()) { - return List.of("source src-gen/" + script); + return List.of("source src-gen/" + StringEscapeUtils.escapeXSI(script)); } return List.of(); } @@ -94,7 +95,7 @@ protected List getPreBuildCommand() { protected List getPostBuildCommand() { var script = context.getTargetConfig().get(DockerProperty.INSTANCE).postBuildScript(); if (!script.isEmpty()) { - return List.of("source src-gen/" + script); + return List.of("source src-gen/" + StringEscapeUtils.escapeXSI(script)); } return List.of(); } @@ -126,7 +127,7 @@ protected String generateRunForBuild() { protected String generateEntryPoint() { return "ENTRYPOINT [" + getEntryPointCommands().stream() - .map(cmd -> "\"" + cmd + "\"") + .map(cmd -> "\"" + StringEscapeUtils.escapeXSI(cmd) + "\"") .collect(Collectors.joining(",")) + "]"; } @@ -134,7 +135,7 @@ protected String generateEntryPoint() { /** Return a COPY command to copy the executable from the builder to the runner. */ protected String generateCopyOfExecutable() { var lfModuleName = context.getFileConfig().name; - // safe becaused context.getFileConfig().name never contains spaces + // safe because context.getFileConfig().name never contains spaces return "COPY --from=builder /lingua-franca/%s/bin/%s ./bin/%s" .formatted(lfModuleName, lfModuleName, lfModuleName); } @@ -143,8 +144,8 @@ protected String generateCopyOfExecutable() { protected String generateCopyOfScript() { var script = context.getTargetConfig().get(DockerProperty.INSTANCE).preRunScript(); if (!script.isEmpty()) { - return "COPY --from=builder /lingua-franca/%s/src-gen/%s ./scripts/%s" - .formatted(context.getFileConfig().name, script, script); + return "COPY --from=builder /lingua-franca/%s/src-gen/%s ./scripts/" + .formatted(context.getFileConfig().name, StringEscapeUtils.escapeXSI(script)); } return "# (No pre-run script provided.)"; } @@ -172,7 +173,7 @@ protected final List getEntryPointCommands() { DockerOptions.DEFAULT_SHELL, "-c", "source scripts/" - + script + + StringEscapeUtils.escapeXSI(script) + " && " + entryPoint().stream().collect(Collectors.joining(" "))); } diff --git a/gradle.properties b/gradle.properties index 307b046644..38a21f42dc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -22,6 +22,7 @@ freehepVersion=2.4 swtVersion=3.124.0 spotbugsToolVersion=4.7.3 jcipVersion=1.0 +commonsTextVersion=1.12.0 [manifestPropertyNames] org.eclipse.xtext=xtextVersion diff --git a/test/C/src/docker/federated/EscapedScriptName.lf b/test/C/src/docker/federated/EscapedScriptName.lf new file mode 100644 index 0000000000..6aea96fc44 --- /dev/null +++ b/test/C/src/docker/federated/EscapedScriptName.lf @@ -0,0 +1,19 @@ +target C { + logging: DEBUG, + timeout: 1 s, + coordination: centralized, + docker: { + rti-image: "rti:local", + pre-build-script: "foo ish.sh", + } +} + +reactor Hello { + reaction(startup) {= + printf("Hello World!\n"); + =} +} + +federated reactor { + a = new Hello() +} diff --git a/test/C/src/docker/federated/foo ish.sh b/test/C/src/docker/federated/foo ish.sh new file mode 100644 index 0000000000..c070ce28b4 --- /dev/null +++ b/test/C/src/docker/federated/foo ish.sh @@ -0,0 +1,2 @@ +#!/bin/bash +echo "Hello from pre-build script with a foolish name!" From 00e4151b04bee0e74861cda1e3944da89a722b98 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Mon, 17 Jun 2024 18:04:27 -0700 Subject: [PATCH 29/48] CppDockerGenerator.java passes smoke test --- .../generator/docker/CppDockerGenerator.java | 79 ++++++++++++++++++- .../org/lflang/generator/cpp/CppGenerator.kt | 25 +++++- .../generator/cpp/CppPlatformGenerator.kt | 2 +- .../cpp/CppStandaloneCmakeGenerator.kt | 6 ++ .../generator/cpp/CppStandaloneGenerator.kt | 12 +-- .../Cpp/src/docker/HelloWorldContainerized.lf | 3 +- 6 files changed, 113 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java index c445765048..5bc8f2c31c 100644 --- a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java @@ -1,15 +1,20 @@ package org.lflang.generator.docker; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.lflang.generator.LFGeneratorContext; import org.lflang.generator.cpp.CppPlatformGenerator; +import org.lflang.generator.cpp.CppStandaloneGenerator; +import org.lflang.target.property.BuildTypeProperty; /** - * Generates the docker file related code for the Typescript target. + * Generates the docker file related code for the C++ target. * * @author Marten Lohstroh */ -public class CppDockerGenerator extends CDockerGenerator { +public class CppDockerGenerator extends DockerGenerator { private CppPlatformGenerator platformGenerator; @@ -19,9 +24,75 @@ public CppDockerGenerator(LFGeneratorContext context, CppPlatformGenerator platf this.platformGenerator = platformGenerator; } + @Override + protected String generateCopyForSources() { + return "COPY src-gen src-gen"; + } + + public static final String DEFAULT_BASE_IMAGE = "alpine:latest"; + + @Override + public String defaultImage() { + return DEFAULT_BASE_IMAGE; + } + + @Override + protected String generateRunForInstallingDeps() { + if (builderBase().equals(defaultImage())) { + return "RUN set -ex && apk add --no-cache g++ musl-dev cmake make && apk add --no-cache" + + " --update --repository=https://dl-cdn.alpinelinux.org/alpine/v3.16/main/" + + " libexecinfo-dev"; + } else { + return "# (Skipping installation of build dependencies; custom base image.)"; + } + } + + @Override + public List defaultEntryPoint() { + return List.of("./bin/" + context.getFileConfig().name); + } + @Override protected List defaultBuildCommands() { - // FIXME - return List.of(); + var mkdirCommand = List.of("mkdir", "-p", "build", "&&", "mkdir", "-p", "bin"); + var cmakeCommand = new ArrayList(); + cmakeCommand.add("cmake"); + cmakeCommand.addAll(platformGenerator.getCmakeArgs()); + cmakeCommand.addAll( + List.of( + "-DCMAKE_INSTALL_BINDIR=bin", + "-DCMAKE_INSTALL_PREFIX=.", + "-DREACTOR_CPP_LINK_EXECINFO=ON", + "-S", + "src-gen", + "-B", + "build")); + var makeCommand = + List.of( + "cmake", + "--build", + "build", + "--target", + context.getFileConfig().name, + "--config", + CppStandaloneGenerator.Companion.buildTypeToCmakeConfig( + context.getTargetConfig().get(BuildTypeProperty.INSTANCE))); + var installCommand = + List.of( + "cmake", + "--build", + "build", + "--target", + "install", + "--config", + CppStandaloneGenerator.Companion.buildTypeToCmakeConfig( + context.getTargetConfig().get(BuildTypeProperty.INSTANCE))); + return Stream.of(mkdirCommand, cmakeCommand, makeCommand, installCommand) + .map(CppDockerGenerator::argListToCommand) + .toList(); + } + + static String argListToCommand(List args) { + return args.stream().map(it -> "\"" + it + "\"").collect(Collectors.joining(" ")); } } diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt index afa377f76c..9c63a713f6 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt @@ -37,6 +37,8 @@ import org.lflang.scoping.LFGlobalScopeProvider import org.lflang.target.Target import org.lflang.target.property.* import org.lflang.util.FileUtil +import java.io.File +import java.io.IOException import java.nio.file.Files import java.nio.file.Path @@ -95,7 +97,27 @@ class CppGenerator( "Code generation complete. Compiling...", IntegratedBuilder.GENERATED_PERCENT_PROGRESS ) if (targetConfig.get(DockerProperty.INSTANCE).enabled) { - buildUsingDocker(); + FileUtil.deleteDirectory(context.fileConfig.srcGenPath.resolve("src-gen")) + try { + val tempDir = Files.createTempDirectory(context.fileConfig.outPath, "src-gen-directory") + try { + FileUtil.copyDirectoryContents(context.fileConfig.srcGenBasePath, tempDir, false) + FileUtil.copyDirectoryContents(tempDir, context.fileConfig.srcGenPath.resolve("src-gen"), false) + } catch (e: IOException) { + context.errorReporter.nowhere() + .error("Failed to copy sources to make them accessible to Docker: " + if (e.message == null) "No cause given" else e.message) + e.printStackTrace() + } finally { + FileUtil.deleteDirectory(tempDir) + } + if (errorsOccurred()) { + return + } + } catch (e: IOException) { + context.errorReporter.nowhere().error("Failed to create temporary directory.") + e.printStackTrace() + } + buildUsingDocker() } else { if (platformGenerator.doCompile(context)) { context.finish(GeneratorResult.Status.COMPILED, codeMaps) @@ -196,4 +218,3 @@ class CppGenerator( override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator = CppDockerGenerator(context, getPlatformGenerator()) } - diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt index 220770b6a9..6f0f295897 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt @@ -28,7 +28,7 @@ abstract class CppPlatformGenerator(protected val generator: CppGenerator) { abstract fun doCompile(context: LFGeneratorContext, onlyGenerateBuildFiles: Boolean = false): Boolean - protected val cmakeArgs: List + val cmakeArgs: List get() = listOf( "-DCMAKE_BUILD_TYPE=${targetConfig.get(BuildTypeProperty.INSTANCE)}", "-DREACTOR_CPP_VALIDATE=${if (targetConfig.get(NoRuntimeValidationProperty.INSTANCE)) "OFF" else "ON"}", diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneCmakeGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneCmakeGenerator.kt index 2c041fe6d2..5d36e8e327 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneCmakeGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneCmakeGenerator.kt @@ -153,6 +153,8 @@ class CppStandaloneCmakeGenerator(private val targetConfig: TargetConfig, privat |cmake_minimum_required(VERSION 3.5) |project(${fileConfig.name} VERSION 0.0.0 LANGUAGES CXX) | + |option(REACTOR_CPP_LINK_EXECINFO "Link against execinfo" OFF) + | |${if (targetConfig.get(ExternalRuntimePathProperty.INSTANCE) != null) "find_package(reactor-cpp PATHS ${targetConfig.get(ExternalRuntimePathProperty.INSTANCE)})" else ""} | |set(LF_MAIN_TARGET ${fileConfig.name}) @@ -167,6 +169,10 @@ class CppStandaloneCmakeGenerator(private val targetConfig: TargetConfig, privat |) |target_link_libraries($S{LF_MAIN_TARGET} $reactorCppTarget) | + |if(REACTOR_CPP_LINK_EXECINFO) + | target_link_libraries($S{LF_MAIN_TARGET} execinfo) + |endif() + | |if(MSVC) | target_compile_options($S{LF_MAIN_TARGET} PRIVATE /W4) |else() diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt index dd03db376d..1640432fd4 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt @@ -16,6 +16,13 @@ import java.nio.file.Paths class CppStandaloneGenerator(generator: CppGenerator) : CppPlatformGenerator(generator) { + companion object { + fun buildTypeToCmakeConfig(type: BuildType) = when (type) { + BuildType.TEST -> "Debug" + else -> type.toString() + } + } + override fun generatePlatformFiles() { // generate the main source file (containing main()) @@ -132,11 +139,6 @@ class CppStandaloneGenerator(generator: CppGenerator) : return 0 } - private fun buildTypeToCmakeConfig(type: BuildType) = when (type) { - BuildType.TEST -> "Debug" - else -> type.toString() - } - private fun createMakeCommand(buildPath: Path, version: String, target: String): LFCommand { val makeArgs: List if (version.compareVersion("3.12.0") < 0) { diff --git a/test/Cpp/src/docker/HelloWorldContainerized.lf b/test/Cpp/src/docker/HelloWorldContainerized.lf index 85cdcc195e..a7ddb6205c 100644 --- a/test/Cpp/src/docker/HelloWorldContainerized.lf +++ b/test/Cpp/src/docker/HelloWorldContainerized.lf @@ -1,6 +1,5 @@ target Cpp { - // To test generating a custom trace file name. - logging: error, + logging: error, // To test generating a custom trace file name. docker: true, build-type: Debug } From 7f58039bc7e7899d43a0ae667ecb7e99235865f3 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Mon, 17 Jun 2024 18:35:26 -0700 Subject: [PATCH 30/48] Remove escaping from ENTRYPOINT command --- .../java/org/lflang/generator/docker/DockerGenerator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index f3001ef882..2b7e192d03 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -127,7 +127,7 @@ protected String generateRunForBuild() { protected String generateEntryPoint() { return "ENTRYPOINT [" + getEntryPointCommands().stream() - .map(cmd -> "\"" + StringEscapeUtils.escapeXSI(cmd) + "\"") + .map(cmd -> "\"" + cmd + "\"") .collect(Collectors.joining(",")) + "]"; } @@ -173,7 +173,7 @@ protected final List getEntryPointCommands() { DockerOptions.DEFAULT_SHELL, "-c", "source scripts/" - + StringEscapeUtils.escapeXSI(script) + + script + " && " + entryPoint().stream().collect(Collectors.joining(" "))); } From 279b00ea468dd214212aaf86d66fdee728820f74 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Mon, 17 Jun 2024 18:50:45 -0700 Subject: [PATCH 31/48] Make test more thorough --- test/C/src/docker/federated/DockerOptions.lf | 11 +++++++++-- test/C/src/docker/federated/bar.sh | 1 + .../federated/cmake-check-environment-variable.cmake | 6 ++++++ test/C/src/docker/federated/foo.sh | 1 + 4 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 test/C/src/docker/federated/cmake-check-environment-variable.cmake diff --git a/test/C/src/docker/federated/DockerOptions.lf b/test/C/src/docker/federated/DockerOptions.lf index 8c74248597..a63608191f 100644 --- a/test/C/src/docker/federated/DockerOptions.lf +++ b/test/C/src/docker/federated/DockerOptions.lf @@ -1,5 +1,5 @@ target C { - logging: DEBUG, + logging: WARN, timeout: 1 s, coordination: centralized, docker: { @@ -8,12 +8,19 @@ target C { pre-run-script: "bar.sh", post-build-script: "baz.sh", no-build: false - } + }, + cmake-include: "cmake-check-environment-variable.cmake" } reactor Hello { reaction(startup) {= printf("Hello World!\n"); + // crash if env var "bar" is not set + if (getenv("BAR") == NULL) { + printf("bar is not set\n"); + exit(1); + } + printf("success: bar = %s\n", getenv("BAR")); =} } diff --git a/test/C/src/docker/federated/bar.sh b/test/C/src/docker/federated/bar.sh index e55421afd2..fcd6eed11a 100755 --- a/test/C/src/docker/federated/bar.sh +++ b/test/C/src/docker/federated/bar.sh @@ -1,2 +1,3 @@ #!/bin/bash echo "Hello from pre-run script!" +export BAR=bar diff --git a/test/C/src/docker/federated/cmake-check-environment-variable.cmake b/test/C/src/docker/federated/cmake-check-environment-variable.cmake new file mode 100644 index 0000000000..72c9aff868 --- /dev/null +++ b/test/C/src/docker/federated/cmake-check-environment-variable.cmake @@ -0,0 +1,6 @@ +# fail if the environment variable FOO is not set +if(DEFINED ENV{FOO}) + message("FOO is set to $ENV{FOO}") +else() + message(FATAL_ERROR "FOO is not set") +endif() diff --git a/test/C/src/docker/federated/foo.sh b/test/C/src/docker/federated/foo.sh index 60f774ffa7..9e8fa941ef 100644 --- a/test/C/src/docker/federated/foo.sh +++ b/test/C/src/docker/federated/foo.sh @@ -1,2 +1,3 @@ #!/bin/bash echo "Hello from pre-build script!" +export FOO=foo From b04092f1c9f8769721cd5a79c94ed78f715a388d Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Mon, 17 Jun 2024 18:57:54 -0700 Subject: [PATCH 32/48] Add back part of the escaping within ENTRYPOINT --- .../main/java/org/lflang/generator/docker/DockerGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index 2b7e192d03..15f4a5911c 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -173,7 +173,7 @@ protected final List getEntryPointCommands() { DockerOptions.DEFAULT_SHELL, "-c", "source scripts/" - + script + + StringEscapeUtils.escapeXSI(script) + " && " + entryPoint().stream().collect(Collectors.joining(" "))); } From 3eba547ec6d8b138239c21c21dcb6aaad8be08ab Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Mon, 17 Jun 2024 21:06:02 -0700 Subject: [PATCH 33/48] Format --- test/C/src/docker/federated/EscapedScriptName.lf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/C/src/docker/federated/EscapedScriptName.lf b/test/C/src/docker/federated/EscapedScriptName.lf index 6aea96fc44..25b44164b3 100644 --- a/test/C/src/docker/federated/EscapedScriptName.lf +++ b/test/C/src/docker/federated/EscapedScriptName.lf @@ -4,7 +4,7 @@ target C { coordination: centralized, docker: { rti-image: "rti:local", - pre-build-script: "foo ish.sh", + pre-build-script: "foo ish.sh" } } From 0fbde62bcc24baf2dc75bfd2caf55f9e5b7f53c4 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Mon, 17 Jun 2024 22:16:37 -0700 Subject: [PATCH 34/48] Update build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index da1990ca3f..fce0ae6ea9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,6 @@ jobs: epoch: uses: lf-lang/epoch/.github/workflows/build.yml@main with: - lingua-franca-ref: ${{ github.head_ref || github.ref_name }} + lingua-franca-ref: pre-build-cmd # ${{ github.head_ref || github.ref_name }} lingua-franca-repo: ${{ github.event.pull_request.head.repo.full_name }} upload-artifacts: false From f3f37a82de67eeaf91286829439694c1ca2ad940 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Mon, 17 Jun 2024 22:43:39 -0700 Subject: [PATCH 35/48] Address comments from code review --- .../generator/docker/CppDockerGenerator.java | 7 +-- .../generator/docker/DockerGenerator.java | 8 ++++ .../org/lflang/generator/cpp/CppGenerator.kt | 44 ++++++++++--------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java index 5bc8f2c31c..db56a682cd 100644 --- a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java @@ -2,7 +2,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.lflang.generator.LFGeneratorContext; import org.lflang.generator.cpp.CppPlatformGenerator; @@ -88,11 +87,7 @@ protected List defaultBuildCommands() { CppStandaloneGenerator.Companion.buildTypeToCmakeConfig( context.getTargetConfig().get(BuildTypeProperty.INSTANCE))); return Stream.of(mkdirCommand, cmakeCommand, makeCommand, installCommand) - .map(CppDockerGenerator::argListToCommand) + .map(DockerGenerator::argListToCommand) .toList(); } - - static String argListToCommand(List args) { - return args.stream().map(it -> "\"" + it + "\"").collect(Collectors.joining(" ")); - } } diff --git a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java index 6ee2847ead..59bc91b9e8 100644 --- a/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/DockerGenerator.java @@ -233,4 +233,12 @@ public static DockerGenerator dockerGeneratorFactory(LFGeneratorContext context) throw new IllegalArgumentException("No Docker support for " + target + " yet."); }; } + + /** + * Convert an argument list, starting with the command to execute, into a string that can be + * executed by a POSIX-compliant shell. + */ + public static String argListToCommand(List args) { + return args.stream().map(it -> "\"" + it + "\"").collect(Collectors.joining(" ")); + } } diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt index 9c63a713f6..4fd410d73d 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt @@ -97,26 +97,7 @@ class CppGenerator( "Code generation complete. Compiling...", IntegratedBuilder.GENERATED_PERCENT_PROGRESS ) if (targetConfig.get(DockerProperty.INSTANCE).enabled) { - FileUtil.deleteDirectory(context.fileConfig.srcGenPath.resolve("src-gen")) - try { - val tempDir = Files.createTempDirectory(context.fileConfig.outPath, "src-gen-directory") - try { - FileUtil.copyDirectoryContents(context.fileConfig.srcGenBasePath, tempDir, false) - FileUtil.copyDirectoryContents(tempDir, context.fileConfig.srcGenPath.resolve("src-gen"), false) - } catch (e: IOException) { - context.errorReporter.nowhere() - .error("Failed to copy sources to make them accessible to Docker: " + if (e.message == null) "No cause given" else e.message) - e.printStackTrace() - } finally { - FileUtil.deleteDirectory(tempDir) - } - if (errorsOccurred()) { - return - } - } catch (e: IOException) { - context.errorReporter.nowhere().error("Failed to create temporary directory.") - e.printStackTrace() - } + copySrcGenBaseDirIntoSrcGenDir() buildUsingDocker() } else { if (platformGenerator.doCompile(context)) { @@ -128,6 +109,29 @@ class CppGenerator( } } + private fun copySrcGenBaseDirIntoSrcGenDir() { + FileUtil.deleteDirectory(context.fileConfig.srcGenPath.resolve("src-gen")) + try { + val tempDir = Files.createTempDirectory(context.fileConfig.outPath, "src-gen-directory") + try { + FileUtil.copyDirectoryContents(context.fileConfig.srcGenBasePath, tempDir, false) + FileUtil.copyDirectoryContents(tempDir, context.fileConfig.srcGenPath.resolve("src-gen"), false) + } catch (e: IOException) { + context.errorReporter.nowhere() + .error("Failed to copy sources to make them accessible to Docker: " + if (e.message == null) "No cause given" else e.message) + e.printStackTrace() + } finally { + FileUtil.deleteDirectory(tempDir) + } + if (errorsOccurred()) { + return + } + } catch (e: IOException) { + context.errorReporter.nowhere().error("Failed to create temporary directory.") + e.printStackTrace() + } + } + private fun fetchReactorCpp(version: String) { val libPath = fileConfig.srcGenBasePath.resolve("reactor-cpp-$version") // abort if the directory already exists From e7d510ec05b07dac411f2e68a7c6d783a9b8018f Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Mon, 17 Jun 2024 22:51:00 -0700 Subject: [PATCH 36/48] Update core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java --- .../main/java/org/lflang/generator/docker/CDockerGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java index e8c114a98c..afe67d8147 100644 --- a/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CDockerGenerator.java @@ -40,7 +40,7 @@ public List defaultEntryPoint() { @Override protected String generateRunForInstallingDeps() { var config = context.getTargetConfig(); - var compiler = config.target == Target.CCPP || config.target == Target.CPP ? "g++" : "gcc"; + var compiler = config.target == Target.CCPP ? "g++" : "gcc"; if (builderBase().equals(defaultImage())) { return "RUN set -ex && apk add --no-cache %s musl-dev cmake make".formatted(compiler); } else { From 0ddd12149f81b7546186f9a941750652371c48fc Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Tue, 18 Jun 2024 08:36:07 -0700 Subject: [PATCH 37/48] Update gradle.properties --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 38a21f42dc..75e2c2dfd9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -22,7 +22,7 @@ freehepVersion=2.4 swtVersion=3.124.0 spotbugsToolVersion=4.7.3 jcipVersion=1.0 -commonsTextVersion=1.12.0 +commonsTextVersion=1.11.0 [manifestPropertyNames] org.eclipse.xtext=xtextVersion From 7c98324450bb1555ae86e4802757be52e1a7f26c Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Tue, 18 Jun 2024 22:14:58 -0700 Subject: [PATCH 38/48] Address failure to load dynamic libraries Observation: this will not work well with cross-compilation. It will not really make sense for the two base images to differ. --- .../lflang/generator/docker/CppDockerGenerator.java | 11 +++++++++++ .../generator/docker/PythonDockerGenerator.java | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java index db56a682cd..471f6f9ce6 100644 --- a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java @@ -51,6 +51,17 @@ public List defaultEntryPoint() { return List.of("./bin/" + context.getFileConfig().name); } + + @Override + protected String generateCopyOfExecutable() { + return String.join( + "\n", + super.generateCopyOfExecutable(), + "COPY --from=builder /usr/local/lib /usr/local/lib", + "COPY --from=builder /usr/lib /usr/lib", + "COPY --from=builder /lingua-franca ."); + } + @Override protected List defaultBuildCommands() { var mkdirCommand = List.of("mkdir", "-p", "build", "&&", "mkdir", "-p", "bin"); diff --git a/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java index 9ad1081bca..046b60fbdb 100644 --- a/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java @@ -36,7 +36,7 @@ protected String generateCopyOfExecutable() { "\n", super.generateCopyOfExecutable(), "COPY --from=builder /lingua-franca/%s/src-gen ./src-gen" - .formatted(lfModuleName, lfModuleName, lfModuleName)); + .formatted(lfModuleName)); } @Override From 227c29f5565738687e14b80b825ccfa4922c1d21 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Tue, 18 Jun 2024 23:44:38 -0700 Subject: [PATCH 39/48] Refactor according to code review --- .../org/lflang/tests/runtime/CppRos2Test.java | 2 + .../generator/docker/CppDockerGenerator.java | 44 +++--------------- .../docker/PythonDockerGenerator.java | 3 +- .../generator/cpp/CppPlatformGenerator.kt | 2 + .../lflang/generator/cpp/CppRos2Generator.kt | 30 +++++++----- .../generator/cpp/CppStandaloneGenerator.kt | 46 ++++++++++++++----- 6 files changed, 65 insertions(+), 62 deletions(-) diff --git a/core/src/integrationTest/java/org/lflang/tests/runtime/CppRos2Test.java b/core/src/integrationTest/java/org/lflang/tests/runtime/CppRos2Test.java index 334f9ef7aa..9ddae14d32 100644 --- a/core/src/integrationTest/java/org/lflang/tests/runtime/CppRos2Test.java +++ b/core/src/integrationTest/java/org/lflang/tests/runtime/CppRos2Test.java @@ -5,6 +5,7 @@ import org.lflang.lf.Element; import org.lflang.lf.LfFactory; import org.lflang.target.Target; +import org.lflang.target.property.DockerProperty; import org.lflang.target.property.Ros2Property; import org.lflang.tests.TestBase; import org.lflang.tests.Transformers; @@ -33,6 +34,7 @@ public void runWithRos2() { it -> true, Transformers::noChanges, config -> { + if (config.get(DockerProperty.INSTANCE) != null) return false; Ros2Property.INSTANCE.override(config, true); return true; }, diff --git a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java index 471f6f9ce6..149d9e1c76 100644 --- a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java @@ -1,12 +1,9 @@ package org.lflang.generator.docker; -import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; import org.lflang.generator.LFGeneratorContext; import org.lflang.generator.cpp.CppPlatformGenerator; -import org.lflang.generator.cpp.CppStandaloneGenerator; -import org.lflang.target.property.BuildTypeProperty; /** * Generates the docker file related code for the C++ target. @@ -51,7 +48,6 @@ public List defaultEntryPoint() { return List.of("./bin/" + context.getFileConfig().name); } - @Override protected String generateCopyOfExecutable() { return String.join( @@ -64,40 +60,12 @@ protected String generateCopyOfExecutable() { @Override protected List defaultBuildCommands() { - var mkdirCommand = List.of("mkdir", "-p", "build", "&&", "mkdir", "-p", "bin"); - var cmakeCommand = new ArrayList(); - cmakeCommand.add("cmake"); - cmakeCommand.addAll(platformGenerator.getCmakeArgs()); - cmakeCommand.addAll( - List.of( - "-DCMAKE_INSTALL_BINDIR=bin", - "-DCMAKE_INSTALL_PREFIX=.", - "-DREACTOR_CPP_LINK_EXECINFO=ON", - "-S", - "src-gen", - "-B", - "build")); - var makeCommand = - List.of( - "cmake", - "--build", - "build", - "--target", - context.getFileConfig().name, - "--config", - CppStandaloneGenerator.Companion.buildTypeToCmakeConfig( - context.getTargetConfig().get(BuildTypeProperty.INSTANCE))); - var installCommand = - List.of( - "cmake", - "--build", - "build", - "--target", - "install", - "--config", - CppStandaloneGenerator.Companion.buildTypeToCmakeConfig( - context.getTargetConfig().get(BuildTypeProperty.INSTANCE))); - return Stream.of(mkdirCommand, cmakeCommand, makeCommand, installCommand) + var mkdirCommand = List.of("mkdir", "-p", "build"); + return Stream.concat( + Stream.of(mkdirCommand), + platformGenerator + .getBuildCommands(List.of("-DREACTOR_CPP_LINK_EXECINFO=ON"), false) + .stream()) .map(DockerGenerator::argListToCommand) .toList(); } diff --git a/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java index 046b60fbdb..6a55669752 100644 --- a/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java +++ b/core/src/main/java/org/lflang/generator/docker/PythonDockerGenerator.java @@ -35,8 +35,7 @@ protected String generateCopyOfExecutable() { return String.join( "\n", super.generateCopyOfExecutable(), - "COPY --from=builder /lingua-franca/%s/src-gen ./src-gen" - .formatted(lfModuleName)); + "COPY --from=builder /lingua-franca/%s/src-gen ./src-gen".formatted(lfModuleName)); } @Override diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt index 6f0f295897..e193b45deb 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt @@ -37,4 +37,6 @@ abstract class CppPlatformGenerator(protected val generator: CppGenerator) { "-DREACTOR_CPP_LOG_LEVEL=${targetConfig.get(LoggingProperty.INSTANCE).severity}", "-DLF_SRC_PKG_PATH=${fileConfig.srcPkgPath}", ) + + abstract fun getBuildCommands(additionalCmakeArgs: List, parallelize: Boolean = true): List> } diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt index 76969269c0..a2a75471b7 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt @@ -1,7 +1,9 @@ package org.lflang.generator.cpp +import jakarta.ws.rs.NotSupportedException import org.lflang.generator.LFGeneratorContext import org.lflang.util.FileUtil +import org.lflang.util.LFCommand import java.nio.file.Path /** C++ platform generator for the ROS2 platform.*/ @@ -46,24 +48,30 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator ) return false } - val colconCommand = commandFactory.createCommand( - "colcon", listOf( + "colcon", colconArgs(), fileConfig.outPath) + val returnCode = colconCommand?.run(context.cancelIndicator) + if (returnCode != 0 && !messageReporter.errorsOccurred) { + // If errors occurred but none were reported, then the following message is the best we can do. + messageReporter.nowhere().error("colcon failed with error code $returnCode") + } + + return !messageReporter.errorsOccurred + } + + private fun colconArgs(additionalCmakeArgs: List = listOf()): List { + return listOf( "build", "--packages-select", fileConfig.name, packageGenerator.reactorCppName, "--cmake-args", "-DLF_REACTOR_CPP_SUFFIX=${packageGenerator.reactorCppSuffix}", - ) + cmakeArgs, - fileConfig.outPath - ) - val returnCode = colconCommand?.run(context.cancelIndicator); - if (returnCode != 0 && !messageReporter.errorsOccurred) { - // If errors occurred but none were reported, then the following message is the best we can do. - messageReporter.nowhere().error("colcon failed with error code $returnCode") - } + ) + cmakeArgs + additionalCmakeArgs + } - return !messageReporter.errorsOccurred + override fun getBuildCommands(additionalCmakeArgs: List, parallelize: Boolean): List> { +// return listOf(listOf("colcon") + colconArgs(additionalCmakeArgs)) + throw NotImplementedError("Docker file generation for ROS 2 interoperability is not supported") } } diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt index 1640432fd4..d27763bde4 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt @@ -2,8 +2,10 @@ package org.lflang.generator.cpp import org.lflang.generator.CodeMap import org.lflang.generator.LFGeneratorContext +import org.lflang.generator.docker.DockerGenerator import org.lflang.target.property.BuildTypeProperty import org.lflang.target.property.CompilerProperty +import org.lflang.target.property.type.BuildTypeType import org.lflang.target.property.type.BuildTypeType.BuildType import org.lflang.toUnixString import org.lflang.util.FileUtil @@ -11,6 +13,8 @@ import org.lflang.util.LFCommand import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths +import java.util.stream.Stream +import kotlin.io.path.name /** C++ platform generator for the default native platform without additional dependencies.*/ class CppStandaloneGenerator(generator: CppGenerator) : @@ -98,6 +102,13 @@ class CppStandaloneGenerator(generator: CppGenerator) : return !messageReporter.errorsOccurred } + override fun getBuildCommands(additionalCmakeArgs: List, parallelize: Boolean): List> { + val cmakeCommand = createCmakeCommand(Path.of("./build"), Path.of("."), additionalCmakeArgs, "src-gen") + val makeCommand = createMakeCommand(fileConfig.buildPath, "3.12.0", fileConfig.name) + val installCommand = createMakeCommand(Path.of("./build"), "3.12.0", "install") + return listOf(cmakeCommand, makeCommand, installCommand).map { it.command() } + } + private fun checkCmakeVersion(): String? { // get the installed cmake version and make sure it is at least 3.5 val cmd = commandFactory.createCommand("cmake", listOf("--version"), fileConfig.buildPath) @@ -139,38 +150,51 @@ class CppStandaloneGenerator(generator: CppGenerator) : return 0 } - private fun createMakeCommand(buildPath: Path, version: String, target: String): LFCommand { + private fun createMakeCommand(buildPath: Path, version: String, target: String, parallelize: Boolean = false): LFCommand { val makeArgs: List + val cmakeConfig = buildTypeToCmakeConfig(targetConfig.get(BuildTypeProperty.INSTANCE)) if (version.compareVersion("3.12.0") < 0) { messageReporter.nowhere().warning("CMAKE is older than version 3.12. Parallel building is not supported.") makeArgs = - listOf("--build", ".", "--target", target, "--config", buildTypeToCmakeConfig(targetConfig.get(BuildTypeProperty.INSTANCE))) - } else { + listOf("--build", buildPath.name, "--target", target, "--config", cmakeConfig) + } else if (parallelize) { val cores = Runtime.getRuntime().availableProcessors() makeArgs = listOf( "--build", - ".", + buildPath.name, "--target", target, "--parallel", cores.toString(), "--config", - buildTypeToCmakeConfig(targetConfig.get(BuildTypeProperty.INSTANCE)) + cmakeConfig + ) + } else { + makeArgs = listOf( + "--build", + buildPath.name, + "--target", + target, + "--config", + cmakeConfig ) } - return commandFactory.createCommand("cmake", makeArgs, buildPath) + return commandFactory.createCommand("cmake", makeArgs, buildPath.parent) } - private fun createCmakeCommand(buildPath: Path, outPath: Path): LFCommand { + private fun createCmakeCommand(buildPath: Path, outPath: Path, additionalCmakeArgs: List = listOf(), sourcesRoot: String? = null): LFCommand { val cmd = commandFactory.createCommand( "cmake", - cmakeArgs + listOf( + cmakeArgs + additionalCmakeArgs + listOf( "-DCMAKE_INSTALL_PREFIX=${outPath.toUnixString()}", - "-DCMAKE_INSTALL_BINDIR=${outPath.relativize(fileConfig.binPath).toUnixString()}", - fileConfig.srcGenBasePath.toUnixString() + "-DCMAKE_INSTALL_BINDIR=${if (outPath.isAbsolute) outPath.relativize(fileConfig.binPath).toUnixString() else fileConfig.binPath.name}", + "-S", + sourcesRoot ?: fileConfig.srcGenBasePath.toUnixString(), + "-B", + buildPath.name ), - buildPath + buildPath.parent ) // prepare cmake From 75cfbaf2542f79252bdabf9102ea33a7cdcaacd7 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 19 Jun 2024 13:07:01 +0200 Subject: [PATCH 40/48] clean up code for parallel builds --- .../generator/cpp/CppStandaloneGenerator.kt | 56 ++++++++----------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt index d27763bde4..2e97831d5e 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt @@ -2,10 +2,8 @@ package org.lflang.generator.cpp import org.lflang.generator.CodeMap import org.lflang.generator.LFGeneratorContext -import org.lflang.generator.docker.DockerGenerator import org.lflang.target.property.BuildTypeProperty import org.lflang.target.property.CompilerProperty -import org.lflang.target.property.type.BuildTypeType import org.lflang.target.property.type.BuildTypeType.BuildType import org.lflang.toUnixString import org.lflang.util.FileUtil @@ -13,7 +11,6 @@ import org.lflang.util.LFCommand import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -import java.util.stream.Stream import kotlin.io.path.name /** C++ platform generator for the default native platform without additional dependencies.*/ @@ -73,16 +70,22 @@ class CppStandaloneGenerator(generator: CppGenerator) : Files.createDirectories(fileConfig.buildPath) val version = checkCmakeVersion() + var parallelize = true + if (version != null && version.compareVersion("3.12.0") < 0) { + messageReporter.nowhere().warning("CMAKE is older than version 3.12. Parallel building is not supported.") + parallelize = false + } + if (version != null) { val cmakeReturnCode = runCmake(context) if (cmakeReturnCode == 0 && runMake) { // If cmake succeeded, run make - val makeCommand = createMakeCommand(fileConfig.buildPath, version, fileConfig.name) + val makeCommand = createMakeCommand(fileConfig.buildPath, parallelize, fileConfig.name) val makeReturnCode = CppValidator(fileConfig, messageReporter, codeMaps).run(makeCommand, context.cancelIndicator) var installReturnCode = 0 if (makeReturnCode == 0) { - val installCommand = createMakeCommand(fileConfig.buildPath, version, "install") + val installCommand = createMakeCommand(fileConfig.buildPath, parallelize, "install") installReturnCode = installCommand.run(context.cancelIndicator) if (installReturnCode == 0) { println("SUCCESS (compiling generated C++ code)") @@ -104,8 +107,8 @@ class CppStandaloneGenerator(generator: CppGenerator) : override fun getBuildCommands(additionalCmakeArgs: List, parallelize: Boolean): List> { val cmakeCommand = createCmakeCommand(Path.of("./build"), Path.of("."), additionalCmakeArgs, "src-gen") - val makeCommand = createMakeCommand(fileConfig.buildPath, "3.12.0", fileConfig.name) - val installCommand = createMakeCommand(Path.of("./build"), "3.12.0", "install") + val makeCommand = createMakeCommand(fileConfig.buildPath, true, fileConfig.name) + val installCommand = createMakeCommand(Path.of("./build"), true, "install") return listOf(cmakeCommand, makeCommand, installCommand).map { it.command() } } @@ -150,34 +153,19 @@ class CppStandaloneGenerator(generator: CppGenerator) : return 0 } - private fun createMakeCommand(buildPath: Path, version: String, target: String, parallelize: Boolean = false): LFCommand { - val makeArgs: List + private fun createMakeCommand(buildPath: Path, parallelize: Boolean, target: String, ): LFCommand { val cmakeConfig = buildTypeToCmakeConfig(targetConfig.get(BuildTypeProperty.INSTANCE)) - if (version.compareVersion("3.12.0") < 0) { - messageReporter.nowhere().warning("CMAKE is older than version 3.12. Parallel building is not supported.") - makeArgs = - listOf("--build", buildPath.name, "--target", target, "--config", cmakeConfig) - } else if (parallelize) { - val cores = Runtime.getRuntime().availableProcessors() - makeArgs = listOf( - "--build", - buildPath.name, - "--target", - target, - "--parallel", - cores.toString(), - "--config", - cmakeConfig - ) - } else { - makeArgs = listOf( - "--build", - buildPath.name, - "--target", - target, - "--config", - cmakeConfig - ) + val makeArgs: MutableList = listOf( + "--build", + buildPath.name, + "--target", + target, + "--config", + cmakeConfig + ).toMutableList() + + if (parallelize) { + makeArgs.addAll(listOf("--parallel", Runtime.getRuntime().availableProcessors().toString())) } return commandFactory.createCommand("cmake", makeArgs, buildPath.parent) From 4ba586988bf3baac8722ad58e60566b1131d334a Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 19 Jun 2024 13:29:27 +0200 Subject: [PATCH 41/48] keep all build information in a single place --- .../generator/docker/CppDockerGenerator.java | 72 ----------------- .../org/lflang/generator/cpp/CppGenerator.kt | 4 +- .../generator/cpp/CppPlatformGenerator.kt | 3 +- .../lflang/generator/cpp/CppRos2Generator.kt | 10 ++- .../generator/cpp/CppStandaloneGenerator.kt | 77 ++++++++++++++++--- 5 files changed, 76 insertions(+), 90 deletions(-) delete mode 100644 core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java diff --git a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java b/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java deleted file mode 100644 index 149d9e1c76..0000000000 --- a/core/src/main/java/org/lflang/generator/docker/CppDockerGenerator.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.lflang.generator.docker; - -import java.util.List; -import java.util.stream.Stream; -import org.lflang.generator.LFGeneratorContext; -import org.lflang.generator.cpp.CppPlatformGenerator; - -/** - * Generates the docker file related code for the C++ target. - * - * @author Marten Lohstroh - */ -public class CppDockerGenerator extends DockerGenerator { - - private CppPlatformGenerator platformGenerator; - - /** Construct a new Docker generator. */ - public CppDockerGenerator(LFGeneratorContext context, CppPlatformGenerator platformGenerator) { - super(context); - this.platformGenerator = platformGenerator; - } - - @Override - protected String generateCopyForSources() { - return "COPY src-gen src-gen"; - } - - public static final String DEFAULT_BASE_IMAGE = "alpine:latest"; - - @Override - public String defaultImage() { - return DEFAULT_BASE_IMAGE; - } - - @Override - protected String generateRunForInstallingDeps() { - if (builderBase().equals(defaultImage())) { - return "RUN set -ex && apk add --no-cache g++ musl-dev cmake make && apk add --no-cache" - + " --update --repository=https://dl-cdn.alpinelinux.org/alpine/v3.16/main/" - + " libexecinfo-dev"; - } else { - return "# (Skipping installation of build dependencies; custom base image.)"; - } - } - - @Override - public List defaultEntryPoint() { - return List.of("./bin/" + context.getFileConfig().name); - } - - @Override - protected String generateCopyOfExecutable() { - return String.join( - "\n", - super.generateCopyOfExecutable(), - "COPY --from=builder /usr/local/lib /usr/local/lib", - "COPY --from=builder /usr/lib /usr/lib", - "COPY --from=builder /lingua-franca ."); - } - - @Override - protected List defaultBuildCommands() { - var mkdirCommand = List.of("mkdir", "-p", "build"); - return Stream.concat( - Stream.of(mkdirCommand), - platformGenerator - .getBuildCommands(List.of("-DREACTOR_CPP_LINK_EXECINFO=ON"), false) - .stream()) - .map(DockerGenerator::argListToCommand) - .toList(); - } -} diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt index 4fd410d73d..d64afb4d01 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt @@ -30,14 +30,12 @@ import org.eclipse.emf.ecore.resource.Resource import org.lflang.generator.* import org.lflang.generator.GeneratorUtils.canGenerate import org.lflang.generator.LFGeneratorContext.Mode -import org.lflang.generator.docker.CppDockerGenerator import org.lflang.generator.docker.DockerGenerator import org.lflang.isGeneric import org.lflang.scoping.LFGlobalScopeProvider import org.lflang.target.Target import org.lflang.target.property.* import org.lflang.util.FileUtil -import java.io.File import java.io.IOException import java.nio.file.Files import java.nio.file.Path @@ -220,5 +218,5 @@ class CppGenerator( override fun getTargetTypes(): TargetTypes = CppTypes - override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator = CppDockerGenerator(context, getPlatformGenerator()) + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator = getPlatformGenerator().getDockerGenerator(context) } diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt index e193b45deb..30b9a78d6e 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt @@ -4,6 +4,7 @@ import org.lflang.MessageReporter import org.lflang.target.TargetConfig import org.lflang.generator.GeneratorCommandFactory import org.lflang.generator.LFGeneratorContext +import org.lflang.generator.docker.DockerGenerator import org.lflang.target.property.BuildTypeProperty import org.lflang.target.property.LoggingProperty import org.lflang.target.property.NoRuntimeValidationProperty @@ -38,5 +39,5 @@ abstract class CppPlatformGenerator(protected val generator: CppGenerator) { "-DLF_SRC_PKG_PATH=${fileConfig.srcPkgPath}", ) - abstract fun getBuildCommands(additionalCmakeArgs: List, parallelize: Boolean = true): List> + abstract fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator } diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt index a2a75471b7..06bfadc98e 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt @@ -2,6 +2,7 @@ package org.lflang.generator.cpp import jakarta.ws.rs.NotSupportedException import org.lflang.generator.LFGeneratorContext +import org.lflang.generator.docker.DockerGenerator import org.lflang.util.FileUtil import org.lflang.util.LFCommand import java.nio.file.Path @@ -59,6 +60,10 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator return !messageReporter.errorsOccurred } + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator { + TODO("Not yet implemented") + } + private fun colconArgs(additionalCmakeArgs: List = listOf()): List { return listOf( "build", @@ -70,8 +75,5 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator ) + cmakeArgs + additionalCmakeArgs } - override fun getBuildCommands(additionalCmakeArgs: List, parallelize: Boolean): List> { -// return listOf(listOf("colcon") + colconArgs(additionalCmakeArgs)) - throw NotImplementedError("Docker file generation for ROS 2 interoperability is not supported") - } + } diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt index 2e97831d5e..fda17fb0c4 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt @@ -2,6 +2,7 @@ package org.lflang.generator.cpp import org.lflang.generator.CodeMap import org.lflang.generator.LFGeneratorContext +import org.lflang.generator.docker.DockerGenerator import org.lflang.target.property.BuildTypeProperty import org.lflang.target.property.CompilerProperty import org.lflang.target.property.type.BuildTypeType.BuildType @@ -22,6 +23,8 @@ class CppStandaloneGenerator(generator: CppGenerator) : BuildType.TEST -> "Debug" else -> type.toString() } + + const val DEFAULT_BASE_IMAGE: String = "alpine:latest" } override fun generatePlatformFiles() { @@ -105,13 +108,6 @@ class CppStandaloneGenerator(generator: CppGenerator) : return !messageReporter.errorsOccurred } - override fun getBuildCommands(additionalCmakeArgs: List, parallelize: Boolean): List> { - val cmakeCommand = createCmakeCommand(Path.of("./build"), Path.of("."), additionalCmakeArgs, "src-gen") - val makeCommand = createMakeCommand(fileConfig.buildPath, true, fileConfig.name) - val installCommand = createMakeCommand(Path.of("./build"), true, "install") - return listOf(cmakeCommand, makeCommand, installCommand).map { it.command() } - } - private fun checkCmakeVersion(): String? { // get the installed cmake version and make sure it is at least 3.5 val cmd = commandFactory.createCommand("cmake", listOf("--version"), fileConfig.buildPath) @@ -153,7 +149,7 @@ class CppStandaloneGenerator(generator: CppGenerator) : return 0 } - private fun createMakeCommand(buildPath: Path, parallelize: Boolean, target: String, ): LFCommand { + private fun createMakeCommand(buildPath: Path, parallelize: Boolean, target: String): LFCommand { val cmakeConfig = buildTypeToCmakeConfig(targetConfig.get(BuildTypeProperty.INSTANCE)) val makeArgs: MutableList = listOf( "--build", @@ -171,12 +167,19 @@ class CppStandaloneGenerator(generator: CppGenerator) : return commandFactory.createCommand("cmake", makeArgs, buildPath.parent) } - private fun createCmakeCommand(buildPath: Path, outPath: Path, additionalCmakeArgs: List = listOf(), sourcesRoot: String? = null): LFCommand { + private fun createCmakeCommand( + buildPath: Path, + outPath: Path, + additionalCmakeArgs: List = listOf(), + sourcesRoot: String? = null + ): LFCommand { val cmd = commandFactory.createCommand( "cmake", cmakeArgs + additionalCmakeArgs + listOf( "-DCMAKE_INSTALL_PREFIX=${outPath.toUnixString()}", - "-DCMAKE_INSTALL_BINDIR=${if (outPath.isAbsolute) outPath.relativize(fileConfig.binPath).toUnixString() else fileConfig.binPath.name}", + "-DCMAKE_INSTALL_BINDIR=${ + if (outPath.isAbsolute) outPath.relativize(fileConfig.binPath).toUnixString() else fileConfig.binPath.name + }", "-S", sourcesRoot ?: fileConfig.srcGenBasePath.toUnixString(), "-B", @@ -191,4 +194,58 @@ class CppStandaloneGenerator(generator: CppGenerator) : } return cmd } + + inner class CppDockerGenerator(context: LFGeneratorContext?) : DockerGenerator(context) { + override fun generateCopyForSources(): String { + return "COPY src-gen src-gen" + } + + override fun defaultImage(): String { + return DEFAULT_BASE_IMAGE + } + + override fun generateRunForInstallingDeps(): String { + return if (builderBase() == defaultImage()) { + ("RUN set -ex && apk add --no-cache g++ musl-dev cmake make && apk add --no-cache" + + " --update --repository=https://dl-cdn.alpinelinux.org/alpine/v3.16/main/" + + " libexecinfo-dev") + } else { + "# (Skipping installation of build dependencies; custom base image.)" + } + } + + override fun defaultEntryPoint(): List { + return listOf("./bin/" + context.fileConfig.name) + } + + override fun generateCopyOfExecutable(): String { + return java.lang.String.join( + "\n", + super.generateCopyOfExecutable(), + "COPY --from=builder /usr/local/lib /usr/local/lib", + "COPY --from=builder /usr/lib /usr/lib", + "COPY --from=builder /lingua-franca ." + ) + } + + override fun defaultBuildCommands(): List { + val mkdirCommand = listOf("mkdir", "-p", "build") + val commands = listOf( + mkdirCommand, + createCmakeCommand( + Path.of("./build"), + Path.of("."), + listOf("-DREACTOR_CPP_LINK_EXECINFO=ON"), + "src-gen" + ).command(), + createMakeCommand(fileConfig.buildPath, true, fileConfig.name).command(), + createMakeCommand(Path.of("./build"), true, "install").command() + ) + return commands.map { argListToCommand(it) } + } + } + + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator = CppDockerGenerator(context) } + + From 64d9be7ace72bd56b9aaa847028215b5ed18b452 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 19 Jun 2024 16:13:33 +0200 Subject: [PATCH 42/48] ros2 support for docker generation --- .../generator/cpp/CppPlatformGenerator.kt | 2 +- .../lflang/generator/cpp/CppRos2Generator.kt | 49 ++++++++++++++++--- .../generator/cpp/CppRos2PackageGenerator.kt | 2 +- .../generator/cpp/CppStandaloneGenerator.kt | 4 +- 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt index 30b9a78d6e..799d0125c2 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppPlatformGenerator.kt @@ -29,7 +29,7 @@ abstract class CppPlatformGenerator(protected val generator: CppGenerator) { abstract fun doCompile(context: LFGeneratorContext, onlyGenerateBuildFiles: Boolean = false): Boolean - val cmakeArgs: List + protected val cmakeArgs: List get() = listOf( "-DCMAKE_BUILD_TYPE=${targetConfig.get(BuildTypeProperty.INSTANCE)}", "-DREACTOR_CPP_VALIDATE=${if (targetConfig.get(NoRuntimeValidationProperty.INSTANCE)) "OFF" else "ON"}", diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt index 06bfadc98e..70aa9cdb4a 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt @@ -1,10 +1,9 @@ package org.lflang.generator.cpp -import jakarta.ws.rs.NotSupportedException import org.lflang.generator.LFGeneratorContext import org.lflang.generator.docker.DockerGenerator +import org.lflang.target.property.DockerProperty import org.lflang.util.FileUtil -import org.lflang.util.LFCommand import java.nio.file.Path /** C++ platform generator for the ROS2 platform.*/ @@ -15,6 +14,10 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator private val nodeGenerator = CppRos2NodeGenerator(mainReactor, targetConfig, fileConfig); private val packageGenerator = CppRos2PackageGenerator(generator, nodeGenerator.nodeName) + companion object { + const val DEFAULT_BASE_IMAGE: String = "ros:rolling-ros-base" + } + override fun generatePlatformFiles() { FileUtil.writeToFile( nodeGenerator.generateHeader(), @@ -33,7 +36,11 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator packagePath.resolve("CMakeLists.txt"), true ) - val scriptPath = fileConfig.binPath.resolve(fileConfig.name); + val scriptPath = + if (targetConfig.get(DockerProperty.INSTANCE).enabled) + fileConfig.srcGenPath.resolve("bin").resolve(fileConfig.name) + else + fileConfig.binPath.resolve(fileConfig.name) FileUtil.writeToFile(packageGenerator.generateBinScript(), scriptPath) scriptPath.toFile().setExecutable(true); } @@ -51,7 +58,7 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator } val colconCommand = commandFactory.createCommand( "colcon", colconArgs(), fileConfig.outPath) - val returnCode = colconCommand?.run(context.cancelIndicator) + val returnCode = colconCommand?.run(context.cancelIndicator) if (returnCode != 0 && !messageReporter.errorsOccurred) { // If errors occurred but none were reported, then the following message is the best we can do. messageReporter.nowhere().error("colcon failed with error code $returnCode") @@ -60,10 +67,6 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator return !messageReporter.errorsOccurred } - override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator { - TODO("Not yet implemented") - } - private fun colconArgs(additionalCmakeArgs: List = listOf()): List { return listOf( "build", @@ -75,5 +78,35 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator ) + cmakeArgs + additionalCmakeArgs } + inner class CppDockerGenerator(context: LFGeneratorContext?) : DockerGenerator(context) { + override fun generateCopyForSources() = + """ + COPY src-gen src-gen + COPY bin bin + """.trimIndent() + + override fun defaultImage(): String = DEFAULT_BASE_IMAGE + + override fun generateRunForInstallingDeps(): String = "" + + override fun defaultEntryPoint(): List = listOf("./bin/" + fileConfig.name) + + override fun generateCopyOfExecutable(): String = + """ + ${super.generateCopyOfExecutable()} + COPY --from=builder lingua-franca/${fileConfig.name}/install install + """.trimIndent() + + override fun defaultBuildCommands(): List { + val commands = listOf( + listOf(".", "/opt/ros/rolling/setup.sh"), + listOf("mkdir", "-p", "build"), + listOf("colcon") + colconArgs(listOf("-DREACTOR_CPP_LINK_EXECINFO=ON")), + ) + return commands.map { argListToCommand(it) } + } + } + + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator = CppDockerGenerator(context) } diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2PackageGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2PackageGenerator.kt index ea7568e2c4..c3ce84c747 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2PackageGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2PackageGenerator.kt @@ -113,7 +113,7 @@ class CppRos2PackageGenerator(generator: CppGenerator, private val nodeName: Str return """ |#!/bin/bash |script_dir="$S(dirname -- "$S(readlink -f -- "${S}0")")" - |source "$S{script_dir}/$relPath/install/setup.sh" + |source "$S{script_dir}/$relPath/install/setup.bash" |ros2 run ${fileConfig.name} ${fileConfig.name}_exe """.trimMargin() } diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt index fda17fb0c4..1dddb69bd1 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt @@ -195,7 +195,7 @@ class CppStandaloneGenerator(generator: CppGenerator) : return cmd } - inner class CppDockerGenerator(context: LFGeneratorContext?) : DockerGenerator(context) { + inner class StandaloneDockerGenerator(context: LFGeneratorContext?) : DockerGenerator(context) { override fun generateCopyForSources(): String { return "COPY src-gen src-gen" } @@ -245,7 +245,7 @@ class CppStandaloneGenerator(generator: CppGenerator) : } } - override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator = CppDockerGenerator(context) + override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator = StandaloneDockerGenerator(context) } From 647c8ebb780d137a0aa902c25f9823da606e8b07 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 19 Jun 2024 16:17:24 +0200 Subject: [PATCH 43/48] use more Kotlin --- .../generator/cpp/CppStandaloneGenerator.kt | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt index 1dddb69bd1..3e52038193 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt @@ -196,13 +196,10 @@ class CppStandaloneGenerator(generator: CppGenerator) : } inner class StandaloneDockerGenerator(context: LFGeneratorContext?) : DockerGenerator(context) { - override fun generateCopyForSources(): String { - return "COPY src-gen src-gen" - } - override fun defaultImage(): String { - return DEFAULT_BASE_IMAGE - } + override fun generateCopyForSources(): String = "COPY src-gen src-gen" + + override fun defaultImage(): String = DEFAULT_BASE_IMAGE override fun generateRunForInstallingDeps(): String { return if (builderBase() == defaultImage()) { @@ -214,19 +211,15 @@ class CppStandaloneGenerator(generator: CppGenerator) : } } - override fun defaultEntryPoint(): List { - return listOf("./bin/" + context.fileConfig.name) - } + override fun defaultEntryPoint(): List = listOf("./bin/" + context.fileConfig.name) - override fun generateCopyOfExecutable(): String { - return java.lang.String.join( - "\n", - super.generateCopyOfExecutable(), - "COPY --from=builder /usr/local/lib /usr/local/lib", - "COPY --from=builder /usr/lib /usr/lib", - "COPY --from=builder /lingua-franca ." - ) - } + override fun generateCopyOfExecutable(): String = + """ + ${super.generateCopyOfExecutable()} + COPY --from=builder /usr/local/lib /usr/local/lib + COPY --from=builder /usr/lib /usr/lib + COPY --from=builder /lingua-franca . + """.trimIndent() override fun defaultBuildCommands(): List { val mkdirCommand = listOf("mkdir", "-p", "build") From c1f4502f92d583951b065c1d5a53f70bfdcbb063 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 19 Jun 2024 16:25:11 +0200 Subject: [PATCH 44/48] run ros2 tests also with docker --- .../java/org/lflang/tests/runtime/CppRos2Test.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/integrationTest/java/org/lflang/tests/runtime/CppRos2Test.java b/core/src/integrationTest/java/org/lflang/tests/runtime/CppRos2Test.java index 9ddae14d32..334f9ef7aa 100644 --- a/core/src/integrationTest/java/org/lflang/tests/runtime/CppRos2Test.java +++ b/core/src/integrationTest/java/org/lflang/tests/runtime/CppRos2Test.java @@ -5,7 +5,6 @@ import org.lflang.lf.Element; import org.lflang.lf.LfFactory; import org.lflang.target.Target; -import org.lflang.target.property.DockerProperty; import org.lflang.target.property.Ros2Property; import org.lflang.tests.TestBase; import org.lflang.tests.Transformers; @@ -34,7 +33,6 @@ public void runWithRos2() { it -> true, Transformers::noChanges, config -> { - if (config.get(DockerProperty.INSTANCE) != null) return false; Ros2Property.INSTANCE.override(config, true); return true; }, From e5cfc623f9ec81aa6b8a241147977fbf0adaa5d2 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 19 Jun 2024 16:28:39 +0200 Subject: [PATCH 45/48] no need to link to execinfo in ros2 --- .../kotlin/org/lflang/generator/cpp/CppRos2Generator.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt index 70aa9cdb4a..2df1fdaa92 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt @@ -67,7 +67,7 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator return !messageReporter.errorsOccurred } - private fun colconArgs(additionalCmakeArgs: List = listOf()): List { + private fun colconArgs(): List { return listOf( "build", "--packages-select", @@ -75,7 +75,7 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator packageGenerator.reactorCppName, "--cmake-args", "-DLF_REACTOR_CPP_SUFFIX=${packageGenerator.reactorCppSuffix}", - ) + cmakeArgs + additionalCmakeArgs + ) + cmakeArgs } inner class CppDockerGenerator(context: LFGeneratorContext?) : DockerGenerator(context) { @@ -101,7 +101,7 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator val commands = listOf( listOf(".", "/opt/ros/rolling/setup.sh"), listOf("mkdir", "-p", "build"), - listOf("colcon") + colconArgs(listOf("-DREACTOR_CPP_LINK_EXECINFO=ON")), + listOf("colcon") + colconArgs(), ) return commands.map { argListToCommand(it) } } From c5f50f310d14f4e772104c93749c519feb84f31a Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 19 Jun 2024 16:36:16 +0200 Subject: [PATCH 46/48] comment and function rename --- .../kotlin/org/lflang/generator/cpp/CppGenerator.kt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt index d64afb4d01..adfbbb9dda 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppGenerator.kt @@ -95,7 +95,7 @@ class CppGenerator( "Code generation complete. Compiling...", IntegratedBuilder.GENERATED_PERCENT_PROGRESS ) if (targetConfig.get(DockerProperty.INSTANCE).enabled) { - copySrcGenBaseDirIntoSrcGenDir() + copySrcGenBaseDirIntoDockerDir() buildUsingDocker() } else { if (platformGenerator.doCompile(context)) { @@ -107,9 +107,15 @@ class CppGenerator( } } - private fun copySrcGenBaseDirIntoSrcGenDir() { + /** + * Copy the contents of the entire src-gen directory to a nested src-gen directory next to the generated Dockerfile. + */ + private fun copySrcGenBaseDirIntoDockerDir() { FileUtil.deleteDirectory(context.fileConfig.srcGenPath.resolve("src-gen")) try { + // We need to copy in two steps via a temporary directory, as the target directory + // is located within the source directory. Without the temporary directory, copying + // fails as we modify the source while writing the target. val tempDir = Files.createTempDirectory(context.fileConfig.outPath, "src-gen-directory") try { FileUtil.copyDirectoryContents(context.fileConfig.srcGenBasePath, tempDir, false) From c834f2d403b198c79185fa7e31548ac3aa9aa210 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Wed, 19 Jun 2024 12:13:39 -0700 Subject: [PATCH 47/48] Address failing Epoch build --- .../org/lflang/generator/cpp/CppStandaloneGenerator.kt | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt index 3e52038193..64fdf32ee0 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppStandaloneGenerator.kt @@ -12,7 +12,6 @@ import org.lflang.util.LFCommand import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -import kotlin.io.path.name /** C++ platform generator for the default native platform without additional dependencies.*/ class CppStandaloneGenerator(generator: CppGenerator) : @@ -153,7 +152,7 @@ class CppStandaloneGenerator(generator: CppGenerator) : val cmakeConfig = buildTypeToCmakeConfig(targetConfig.get(BuildTypeProperty.INSTANCE)) val makeArgs: MutableList = listOf( "--build", - buildPath.name, + buildPath.fileName.toString(), "--target", target, "--config", @@ -178,12 +177,12 @@ class CppStandaloneGenerator(generator: CppGenerator) : cmakeArgs + additionalCmakeArgs + listOf( "-DCMAKE_INSTALL_PREFIX=${outPath.toUnixString()}", "-DCMAKE_INSTALL_BINDIR=${ - if (outPath.isAbsolute) outPath.relativize(fileConfig.binPath).toUnixString() else fileConfig.binPath.name + if (outPath.isAbsolute) outPath.relativize(fileConfig.binPath).toUnixString() else fileConfig.binPath.fileName.toString() }", "-S", sourcesRoot ?: fileConfig.srcGenBasePath.toUnixString(), "-B", - buildPath.name + buildPath.fileName.toString() ), buildPath.parent ) @@ -240,5 +239,3 @@ class CppStandaloneGenerator(generator: CppGenerator) : override fun getDockerGenerator(context: LFGeneratorContext?): DockerGenerator = StandaloneDockerGenerator(context) } - - From 6f421f273cbe22dd76a8243f3ef632fa4d2a1a68 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Wed, 19 Jun 2024 13:18:31 -0700 Subject: [PATCH 48/48] Address failing C++ ROS2 Docker test --- .../main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt index 2df1fdaa92..4be02961ef 100644 --- a/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt +++ b/core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt @@ -3,6 +3,7 @@ package org.lflang.generator.cpp import org.lflang.generator.LFGeneratorContext import org.lflang.generator.docker.DockerGenerator import org.lflang.target.property.DockerProperty +import org.lflang.toUnixString import org.lflang.util.FileUtil import java.nio.file.Path @@ -89,7 +90,7 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator override fun generateRunForInstallingDeps(): String = "" - override fun defaultEntryPoint(): List = listOf("./bin/" + fileConfig.name) + override fun defaultEntryPoint(): List = listOf(fileConfig.outPath.relativize(fileConfig.binPath).toUnixString() + "/" + fileConfig.name) override fun generateCopyOfExecutable(): String = """