Implement a --compile command for the engine runner (#1998)

enso-org · Sep 24, 2021 · d6465e9 · d6465e9
1 parent 8d67625
commit d6465e9
Show file tree

Hide file tree

Showing 18 changed files with 440 additions and 147 deletions.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -241,6 +241,39 @@ jobs:
         run: |
           sleep 1
           sbt buildEngineDistribution
+
+      - name: Compile the Standard Libraries (Unix)
+        working-directory: repo
+        shell: bash
+        if: runner.os != 'Windows'
+        run: |
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Base/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Test/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Visualization/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Searcher/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Table/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Database/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Geo/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Google_Api/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Image/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Examples/$DIST_VERSION
+
+      - name: Compile the Standard Libraries (Windows)
+        working-directory: repo
+        shell: bash
+        if: runner.os == 'Windows'
+        run: |
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Base/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Database/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Examples/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Geo/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Google_Api/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Image/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Searcher/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Table/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Test/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Visualization/$DIST_VERSION
+
       - name: Prepare Project Manager Distribution
         working-directory: repo
         shell: bash

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -195,6 +195,39 @@ jobs:
         run: |
           sleep 1
           sbt buildEngineDistribution
+
+      - name: Compile the Standard Libraries (Unix)
+        working-directory: repo
+        shell: bash
+        if: runner.os != 'Windows'
+        run: |
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Base/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Test/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Visualization/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Searcher/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Table/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Database/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Geo/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Google_Api/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Image/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Examples/$DIST_VERSION
+
+      - name: Compile the Standard Libraries (Windows)
+        working-directory: repo
+        shell: bash
+        if: runner.os == 'Windows'
+        run: |
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Base/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Database/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Examples/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Geo/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Google_Api/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Image/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Searcher/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Table/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Test/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Visualization/$DIST_VERSION
+
       - name: Prepare Project Manager Distribution
         working-directory: repo
         shell: bash

diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml
@@ -22,7 +22,7 @@ jobs:
   test_and_publish:
     name: Build and Test
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 100
+    timeout-minutes: 120
     strategy:
       matrix:
         os: [macOS-latest, ubuntu-18.04, windows-latest]
@@ -279,12 +279,27 @@ jobs:
         shell: bash
         if: runner.os != 'Windows'
         run: |
-          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-read-ir-caches --run test/Tests
-          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-read-ir-caches --run test/Table_Tests
-          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-read-ir-caches --run test/Database_Tests
-          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-read-ir-caches --run test/Geo_Tests
-          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-read-ir-caches --run test/Visualization_Tests
-          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-read-ir-caches --run test/Image_Tests
+          $ENGINE_DIST_DIR/bin/enso --no-ir-caches --run test/Tests
+          $ENGINE_DIST_DIR/bin/enso --no-ir-caches --run test/Table_Tests
+          $ENGINE_DIST_DIR/bin/enso --no-ir-caches --run test/Database_Tests
+          $ENGINE_DIST_DIR/bin/enso --no-ir-caches --run test/Geo_Tests
+          $ENGINE_DIST_DIR/bin/enso --no-ir-caches --run test/Visualization_Tests
+          $ENGINE_DIST_DIR/bin/enso --no-ir-caches --run test/Image_Tests
+
+      - name: Compile the Standard Libraries (Unix)
+        shell: bash
+        if: runner.os != 'Windows'
+        run: |
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Base/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Test/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Visualization/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Searcher/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Table/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Database/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Geo/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Google_Api/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Image/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Examples/$DIST_VERSION
 
       - name: Test Engine Distribution With Caches (Unix)
         shell: bash
@@ -301,12 +316,27 @@ jobs:
         shell: bash
         if: runner.os == 'Windows'
         run: |
-          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-read-ir-caches --run test/Tests
-          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-read-ir-caches --run test/Table_Tests
-          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-read-ir-caches --run test/Database_Tests
-          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-read-ir-caches --run test/Geo_Tests
-          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-read-ir-caches --run test/Visualization_Tests
-          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-read-ir-caches --run test/Image_Tests
+          $ENGINE_DIST_DIR/bin/enso.bat --no-ir-caches --run test/Tests
+          $ENGINE_DIST_DIR/bin/enso.bat --no-ir-caches --run test/Table_Tests
+          $ENGINE_DIST_DIR/bin/enso.bat --no-ir-caches --run test/Database_Tests
+          $ENGINE_DIST_DIR/bin/enso.bat --no-ir-caches --run test/Geo_Tests
+          $ENGINE_DIST_DIR/bin/enso.bat --no-ir-caches --run test/Visualization_Tests
+          $ENGINE_DIST_DIR/bin/enso.bat --no-ir-caches --run test/Image_Tests
+
+      - name: Compile the Standard Libraries (Windows)
+        shell: bash
+        if: runner.os == 'Windows'
+        run: |
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Base/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Database/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Examples/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Geo/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Google_Api/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Image/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Searcher/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Table/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Test/$DIST_VERSION
+          $ENGINE_DIST_DIR/bin/enso.bat --ir-caches --no-compile-dependencies --no-global-cache --compile $ENGINE_DIST_DIR/lib/Standard/Visualization/$DIST_VERSION
 
       - name: Test Engine Distribution With Caches (Windows)
         shell: bash

diff --git a/RELEASES.md b/RELEASES.md
@@ -1,5 +1,12 @@
 # Enso Next
 
+## Interpreter/Runtime
+
+- Added support for compiling Enso packages without execution
+  ([#1998](https://github.com/enso-org/enso/pull/1998)). This allows the
+  distribution of precompiled libraries which greatly improves language start-up
+  time.
+
 ## Enso 0.2.30 (2021-09-23)
 
 ## Interpreter/Runtime

diff --git a/engine/polyglot-api/src/main/java/org/enso/polyglot/MethodNames.java b/engine/polyglot-api/src/main/java/org/enso/polyglot/MethodNames.java
@@ -8,6 +8,7 @@ public static class TopScope {
     public static final String LEAK_CONTEXT = "leak_context";
     public static final String REGISTER_MODULE = "register_module";
     public static final String UNREGISTER_MODULE = "unregister_module";
+    public static final String COMPILE = "compile";
   }
 
   public static class Module {

diff --git a/engine/polyglot-api/src/main/java/org/enso/polyglot/RuntimeOptions.java b/engine/polyglot-api/src/main/java/org/enso/polyglot/RuntimeOptions.java
@@ -78,10 +78,11 @@ public class RuntimeOptions {
               WAIT_FOR_PENDING_SERIALIZATION_JOBS_KEY, WAIT_FOR_PENDING_SERIALIZATION_JOBS)
           .build();
 
-  public static final String NO_READ_IR_CACHES = optionName("onlyGenerateIrCaches");
-  public static final OptionKey<Boolean> NO_READ_IR_CACHES_KEY = new OptionKey<>(false);
-  private static final OptionDescriptor NO_READ_IR_CACHES_DESCRIPTOR =
-      OptionDescriptor.newBuilder(NO_READ_IR_CACHES_KEY, NO_READ_IR_CACHES).build();
+  public static final String USE_GLOBAL_IR_CACHE_LOCATION = optionName("useGlobalIrCacheLocation");
+  public static final OptionKey<Boolean> USE_GLOBAL_IR_CACHE_LOCATION_KEY = new OptionKey<>(true);
+  public static final OptionDescriptor USE_GLOBAL_IR_CACHE_LOCATION_DESCRIPTOR =
+      OptionDescriptor.newBuilder(USE_GLOBAL_IR_CACHE_LOCATION_KEY, USE_GLOBAL_IR_CACHE_LOCATION)
+          .build();
 
   public static final OptionDescriptors OPTION_DESCRIPTORS =
       OptionDescriptors.create(
@@ -98,7 +99,7 @@ public class RuntimeOptions {
               INTERPRETER_SEQUENTIAL_COMMAND_EXECUTION_DESCRIPTOR,
               DISABLE_IR_CACHES_DESCRIPTOR,
               WAIT_FOR_PENDING_SERIALIZATION_JOBS_DESCRIPTOR,
-              NO_READ_IR_CACHES_DESCRIPTOR));
+              USE_GLOBAL_IR_CACHE_LOCATION_DESCRIPTOR));
 
   /**
    * Canonicalizes the option name by prefixing it with the language name.

diff --git a/engine/polyglot-api/src/main/scala/org/enso/polyglot/TopScope.scala b/engine/polyglot-api/src/main/scala/org/enso/polyglot/TopScope.scala
@@ -33,4 +33,8 @@ class TopScope(private val value: Value) {
   def unregisterModule(qualifiedName: String): Unit = {
     value.invokeMember(UNREGISTER_MODULE, qualifiedName): Unit
   }
+
+  def compile(shouldCompileDependencies: Boolean): Unit = {
+    value.invokeMember(COMPILE, shouldCompileDependencies)
+  }
 }
diff --git a/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala b/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala
@@ -1,8 +1,5 @@
 package org.enso.runner
 
-import java.io.InputStream
-import java.io.OutputStream
-
 import org.enso.loggingservice.{JavaLoggingLogHandler, LogLevel}
 import org.enso.polyglot.debugger.{
   DebugServerInfo,
@@ -11,6 +8,8 @@ import org.enso.polyglot.debugger.{
 import org.enso.polyglot.{PolyglotContext, RuntimeOptions}
 import org.graalvm.polyglot.Context
 
+import java.io.{InputStream, OutputStream}
+
 /** Utility class for creating Graal polyglot contexts.
   */
 class ContextFactory {
@@ -24,9 +23,9 @@ class ContextFactory {
     * @param repl the Repl manager to use for this context
     * @param logLevel the log level for this context
     * @param enableIrCaches whether or not IR caching should be enabled
-    * @param enableIrCacheReading whether or not IR cache reading should be
-    *                             enabled
     * @param strictErrors whether or not to use strict errors
+    * @param useGlobalIrCacheLocation whether or not to use the global IR cache
+    *                                 location
     * @return configured Context instance
     */
   def create(
@@ -37,8 +36,8 @@ class ContextFactory {
     logLevel: LogLevel,
     logMasking: Boolean,
     enableIrCaches: Boolean,
-    enableIrCacheReading: Boolean,
-    strictErrors: Boolean = false
+    strictErrors: Boolean             = false,
+    useGlobalIrCacheLocation: Boolean = true
   ): PolyglotContext = {
     val context = Context
       .newBuilder()
@@ -47,11 +46,11 @@ class ContextFactory {
       .option(RuntimeOptions.PROJECT_ROOT, projectRoot)
       .option(RuntimeOptions.STRICT_ERRORS, strictErrors.toString)
       .option(RuntimeOptions.WAIT_FOR_PENDING_SERIALIZATION_JOBS, "true")
-      .option(RuntimeOptions.DISABLE_IR_CACHES, (!enableIrCaches).toString)
       .option(
-        RuntimeOptions.NO_READ_IR_CACHES,
-        (!enableIrCacheReading).toString
+        RuntimeOptions.USE_GLOBAL_IR_CACHE_LOCATION,
+        useGlobalIrCacheLocation.toString
       )
+      .option(RuntimeOptions.DISABLE_IR_CACHES, (!enableIrCaches).toString)
       .option(DebugServerInfo.ENABLE_OPTION, "true")
       .option(RuntimeOptions.LOG_MASKING, logMasking.toString)
       .option("js.foreign-object-prototype", "true")