Merge pull request #971 from lf-lang/cpp-submodule

Add reactor-cpp as a submodule and reorganize the C++ build process

.github/scripts/test-lfc.sh

@@ -63,7 +63,7 @@ bin/lfc --output-path . test/C/src/Minimal.lf
 #    --runtime-version <arg>         Specify the version of the runtime
 #                                    library used for compiling LF
 #                                    programs.
-bin/lfc --runtime-version 26e6e641916924eae2e83bbf40cbc9b933414310 test/Cpp/src/Minimal.lf
+bin/lfc --runtime-version ca216ccc3da5ecff0e8013f75e275d7acac099de test/Cpp/src/Minimal.lf
 
 # -t,--threads                       Specify the default number of threads.
 bin/lfc -t 2 test/C/src/Minimal.lf

.github/workflows/benchmark-tests.yml

@@ -33,14 +33,17 @@ jobs:
         run: |
           python3 benchmark/runner/run_benchmark.py -m test_mode=True iterations=1 benchmark="glob(*)" target=lf-c,lf-c-unthreaded
         if: ${{ inputs.target == 'C' }}
-      - name: Setup C++ build environment
+      - name: Compile reactor-cpp once and reuse for all benchmarks
         run: |
-          ./bin/lfc test/Cpp/src/Minimal.lf
-          echo "LD_LIBRARY_PATH=$GITHUB_WORKSPACE/test/Cpp/lib" >> $GITHUB_ENV
+          mkdir -p reactor-cpp/build
+          cd reactor-cpp/build
+          cmake -DCMAKE_INSTALL_PREFIX=../install ../../org.lflang/src/lib/cpp/reactor-cpp
+          make install
+          echo "LD_LIBRARY_PATH=$GITHUB_WORKSPACE/reactor-cpp/install/lib" >> $GITHUB_ENV
         if: ${{ inputs.target == 'Cpp' }}
       - name: Test C++ benchmarks
         run: |
-          python3 benchmark/runner/run_benchmark.py -m test_mode=True iterations=1 benchmark="glob(*)" target=lf-cpp iterations=1 target.params.extra_args="[--external-runtime-path, ${GITHUB_WORKSPACE}/test/Cpp]"
+          python3 benchmark/runner/run_benchmark.py -m test_mode=True iterations=1 benchmark="glob(*)" target=lf-cpp iterations=1 target.params.extra_args="[--external-runtime-path, ${GITHUB_WORKSPACE}/reactor-cpp/install]"
         if: ${{ inputs.target == 'Cpp' }}
       - name: Setup Rust
         uses: ATiltedTree/setup-rust@v1

.github/workflows/ci.yml

@@ -54,7 +54,7 @@ jobs:
 
   # Run the C++ benchmark tests.
   cpp-benchmark-tests:
-    uses: lf-lang/lingua-franca/.github/workflows/benchmark-tests.yml@master
+    uses: lf-lang/lingua-franca/.github/workflows/benchmark-tests.yml@cpp-submodule
     with:
       target: 'Cpp'
 

.github/workflows/cpp-tests.yml

@@ -29,10 +29,13 @@ jobs:
           repository: lf-lang/lingua-franca
           submodules: true
           ref: ${{ inputs.compiler-ref }}
-      - name: Set cpp runtime version
-        run: |
-            echo ${{ inputs.runtime-ref }} > org.lflang/src/org/lflang/generator/cpp/cpp-runtime-version.txt
-        if: ${{ inputs.runtime-ref }}  
+      - name: Check out specific ref of reactor-cpp
+        uses: actions/checkout@v2
+        with:
+          repository: lf-lang/reactor-cpp
+          path: org.lflang/src/lib/cpp/reactor-cpp
+          ref: ${{ inputs.runtime-ref }}
+        if: ${{ inputs.runtime-ref }} 
       - name: Run C++ tests;
         run: |
           ./gradlew test --tests org.lflang.tests.runtime.CppTest.* --tests org.lflang.tests.lsp.LspTests.lspWithDependenciesTestCpp
@@ -42,4 +45,4 @@ jobs:
           file: org.lflang.tests/build/reports/xml/jacoco
           fail_ci_if_error: false
           verbose: true
-        if: ${{ !inputs.compiler-ref }}  # i.e., if this is part of the main repo's CI
+        if: ${{ !inputs.runtime-ref }}  # i.e., if this is part of the main repo's CI

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "org.lflang/src/lib/py/reactor-c-py"]
 	path = org.lflang/src/lib/py/reactor-c-py
 	url = https://github.com/lf-lang/reactor-c-py.git
+[submodule "org.lflang/src/lib/cpp/reactor-cpp"]
+	path = org.lflang/src/lib/cpp/reactor-cpp
+	url = https://github.com/lf-lang/reactor-cpp

org.lflang/src/org/lflang/FileConfig.java

@@ -1,16 +1,26 @@
 package org.lflang;
 
+import java.io.BufferedInputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.JarURLConnection;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLConnection;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
+import java.util.Arrays;
 import java.util.Comparator;
+import java.util.Enumeration;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.function.Consumer;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -19,6 +29,7 @@
 import org.eclipse.core.resources.IResource;
 import org.eclipse.core.resources.IWorkspaceRoot;
 import org.eclipse.core.resources.ResourcesPlugin;
+import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.emf.common.util.URI;
 import org.eclipse.emf.ecore.resource.Resource;
@@ -394,9 +405,10 @@ protected static Path getSubPkgPath(Path pkgPath, Path srcPath) {
      *
      * @param src The source directory path.
      * @param dest The destination directory path.
+     * @param skipIfUnchanged If true, don't overwrite the destination file if its content would not be changed
      * @throws IOException if copy fails.
      */
-    public static void copyDirectory(Path src, Path dest) throws IOException {
+    public static void copyDirectory(final Path src, final Path dest, final boolean skipIfUnchanged) throws IOException {
         try (Stream<Path> stream = Files.walk(src)) {
             stream.forEach(source -> {
                 // Handling checked exceptions in lambda expressions is
@@ -408,7 +420,7 @@ public static void copyDirectory(Path src, Path dest) throws IOException {
                     try {
                         Path target = dest.resolve(src.relativize(source));
                         Files.createDirectories(target.getParent());
-                        copyFile(source, target);
+                        copyFile(source, target, skipIfUnchanged);
                     } catch (IOException e) {
                         throw new RuntimeIOException(e);
                     } catch (Exception e) {
@@ -419,16 +431,72 @@ public static void copyDirectory(Path src, Path dest) throws IOException {
         }
     }
 
+    /**
+     * Recursively copies the contents of the given 'src'
+     * directory to 'dest'. Existing files of the destination
+     * may be overwritten.
+     *
+     * @param src The source directory path.
+     * @param dest The destination directory path.
+     * @throws IOException if copy fails.
+     */
+    public static void copyDirectory(final Path src, final Path dest) throws IOException {
+        copyDirectory(src, dest, false);
+    }
+
     /**
      * Copy a given file from 'source' to 'destination'.
      *
+     * This also creates new directories for any directories on the destination
+     * path that do not yet exist.
+     *
+     * @param source The source file path.
+     * @param destination The destination file path.
+     * @param skipIfUnchanged If true, don't overwrite the destination file if its content would not be changed
+     * @throws IOException if copy fails.
+     */
+    public static void copyFile(Path source, Path destination, boolean skipIfUnchanged)  throws IOException {
+        BufferedInputStream stream = new BufferedInputStream(new FileInputStream(source.toFile()));
+        try (stream) {
+            copyInputStream(stream, destination, skipIfUnchanged);
+        }
+    }
+
+    /**
+     * Copy a given file from 'source' to 'destination'.
+     *
+     * This also creates new directories for any directories on the destination
+     * path that do not yet exist.
+     *
      * @param source The source file path.
      * @param destination The destination file path.
      * @throws IOException if copy fails.
      */
     public static void copyFile(Path source, Path destination)  throws IOException {
+        copyFile(source, destination, false);
+    }
+
+    /**
+     * Copy a given input stream to a destination file.
+     *
+     * This also creates new directories for any directories on the destination
+     * path that do not yet exist.
+     *
+     * @param source The source input stream.
+     * @param destination The destination file path.
+     * @param skipIfUnchanged If true, don't overwrite the destination file if its content would not be changed
+     * @throws IOException if copy fails.
+     */
+    private static void copyInputStream(InputStream source, Path destination, boolean skipIfUnchanged) throws IOException {
+        Files.createDirectories(destination.getParent());
+        if(skipIfUnchanged && Files.isRegularFile(destination)) {
+            if (Arrays.equals(source.readAllBytes(), Files.readAllBytes(destination))) {
+                return;
+            }
+        }
         Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING);
     }
+
 
     /**
      *  Lookup a file in the classpath and copy its contents to a destination path 
@@ -439,32 +507,105 @@ public static void copyFile(Path source, Path destination)  throws IOException {
      * 
      *  @param source The source file as a path relative to the classpath.
      *  @param destination The file system path that the source file is copied to.
+     *  @param skipIfUnchanged If true, don't overwrite the destination file if its content would not be changed
      * @throws IOException If the given source cannot be copied.
      */
-    public void copyFileFromClassPath(String source, Path destination) throws IOException {
+    public void copyFileFromClassPath(final String source, final Path destination, final boolean skipIfUnchanged) throws IOException {
         InputStream sourceStream = this.getClass().getResourceAsStream(source);
 
         // Copy the file.
-        try (sourceStream) {
-            if (sourceStream == null) {
-                throw new IOException(
-                    "A required target resource could not be found: " + source + "\n" +
-                        "Perhaps a git submodule is missing or not up to date.\n" +
-                        "See https://github.com/icyphy/lingua-franca/wiki/downloading-and-building#clone-the-lingua-franca-repository.\n"
-                        +
-                        "Also try to refresh and clean the project explorer if working from eclipse.");
+        if (sourceStream == null) {
+            throw new IOException(unableToLocateResourceMessage(source));
+        } else {
+            try (sourceStream) {
+                copyInputStream(sourceStream, destination, skipIfUnchanged);
+            }
+        }
+    }
+
+    /**
+     *  Lookup a file in the classpath and copy its contents to a destination path
+     *  in the filesystem.
+     *
+     *  This also creates new directories for any directories on the destination
+     *  path that do not yet exist.
+     *
+     *  @param source The source file as a path relative to the classpath.
+     *  @param destination The file system path that the source file is copied to.
+     * @throws IOException If the given source cannot be copied.
+     */
+    public void copyFileFromClassPath(final String source, final Path destination) throws IOException {
+        copyFileFromClassPath(source, destination, false);
+    }
+
+    /**
+     *  Lookup a directory in the classpath and copy its contents to a destination path
+     *  in the filesystem.
+     *
+     *  This also creates new directories for any directories on the destination
+     *  path that do not yet exist.
+     *
+     *  @param source The source directory as a path relative to the classpath.
+     *  @param destination The file system path that the source directory is copied to.
+     *  @param skipIfUnchanged If true, don't overwrite the file if its content would not be changed
+     *  @throws IOException If the given source cannot be copied.
+     */
+    public void copyDirectoryFromClassPath(final String source, final Path destination, final boolean skipIfUnchanged) throws IOException {
+        final URL resource = getClass().getResource(source);
+        if (resource == null) {
+            throw new IOException(unableToLocateResourceMessage(source));
+        }
+
+        final URLConnection connection = resource.openConnection();
+        if (connection instanceof JarURLConnection) {
+            copyDirectoryFromJar((JarURLConnection) connection, destination, skipIfUnchanged);
+        } else {
+            try {
+                Path dir = Paths.get(FileLocator.toFileURL(resource).toURI());
+                copyDirectory(dir, destination, skipIfUnchanged);
+            } catch(URISyntaxException e) {
+                // This should never happen as toFileURL should always return a valid URL
+                throw new IOException("Unexpected error while resolving " + source + " on the classpath");
+            }
+        }
+    }
+
+    /**
+     * Copy a directory from ta jar to a destination path in the filesystem.
+     *
+     * This method should only be used in standalone mode (lfc).
+     *
+     * This also creates new directories for any directories on the destination
+     * path that do not yet exist
+     *
+     * @param connection a URLConnection to the source directory within the jar
+     * @param destination The file system path that the source directory is copied to.
+     * @param skipIfUnchanged If true, don't overwrite the file if its content would not be changed
+     * @throws IOException If the given source cannot be copied.
+     */
+    private void copyDirectoryFromJar(JarURLConnection connection, final Path destination, final boolean skipIfUnchanged) throws IOException {
+        final JarFile jar = connection.getJarFile();
+        final String connectionEntryName = connection.getEntryName();
+
+        // Iterate all entries in the jar file.
+        for (Enumeration<JarEntry> e = jar.entries(); e.hasMoreElements(); ) {
+            final JarEntry entry = e.nextElement();
+            final String entryName = entry.getName();
+
+            // Extract files only if they match the given source path.
+            if (entryName.startsWith(connectionEntryName)) {
+                String filename = entry.getName().substring(connectionEntryName.length() + 1);
+                Path currentFile = destination.resolve(filename);
+
+                if (entry.isDirectory()) {
+                    Files.createDirectories(currentFile);
+                } else {
+                    InputStream is = jar.getInputStream(entry);
+                    try (is) {
+                        copyInputStream(is, currentFile, skipIfUnchanged);
+                    }
+                }
             }
-            // Make sure the directory exists
-            //noinspection ResultOfMethodCallIgnored
-            destination.toFile().getParentFile().mkdirs();
-
-            Files.copy(sourceStream, destination, StandardCopyOption.REPLACE_EXISTING);
-        } catch (IOException ex) {
-            throw new IOException(
-                "A required target resource could not be copied: " + source + "\n" +
-                    "Perhaps a git submodule is missing or not up to date.\n" +
-                    "See https://github.com/icyphy/lingua-franca/wiki/downloading-and-building#clone-the-lingua-franca-repository.",
-                ex);
         }
     }
 
@@ -774,4 +915,18 @@ public String getRTIDistributionScriptName() {
     public static String nameWithoutExtension(Resource r) throws IOException {
         return nameWithoutExtension(toPath(r));
     }
+
+    /**
+     * Return a string that explains why a resource may not have been found and
+     * what to do about it.
+     *
+     * @param fileName The Name of the file that could not be found.
+     * @return an explanation.
+     */
+    public static String unableToLocateResourceMessage(String fileName) {
+        return "A required target resource could not be found: " + fileName + "\n" +
+            "Perhaps a git submodule is not initialized and/or not up to date.\n" +
+            "To initialize and update, run `git submodule update --init`.\n" +
+            "You may also need to refresh and clean your build system or IDE.";
+    }
 }

org.lflang/src/org/lflang/generator/JavaGeneratorUtils.java

@@ -5,6 +5,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -320,19 +321,35 @@ public static LFResource getLFResource(
      * Write text to a file.
      * @param text The text to be written.
      * @param path The file to write the code to.
+     * @param skipIfUnchanged If true, don't overwrite the destination file if its content would not be changed
      */
-    public static void writeToFile(String text, Path path) throws IOException {
-        path.getParent().toFile().mkdirs();
+    public static void writeToFile(String text, Path path, boolean skipIfUnchanged) throws IOException {
+        Files.createDirectories(path.getParent());
+        final byte[] bytes = text.getBytes();
+        if (skipIfUnchanged && Files.isRegularFile(path)) {
+            if (Arrays.equals(bytes, Files.readAllBytes(path))) {
+                return;
+            }
+        }
         Files.write(path, text.getBytes());
     }
 
+    /**
+     * Write text to a file.
+     * @param text The text to be written.
+     * @param path The file to write the code to.
+     */
+    public static void writeToFile(String text, Path path) throws IOException {
+        writeToFile(text, path, false);
+    }
+
     /**
      * Write text to a file.
      * @param text The text to be written.
      * @param path The file to write the code to.
      */
     public static void writeToFile(CharSequence text, Path path) throws IOException {
-        writeToFile(text.toString(), path);
+        writeToFile(text.toString(), path, false);
     }
 
     /**