Enable straight-to-jar compilation

.drone.yml

@@ -1,6 +1,9 @@
 matrix:
   CI_SCALA_VERSION:
     - 2.12.6
+  RUN_SCRIPTED:
+    - ./bin/run-ci-scripted.sh
+    - ./bin/run-ci-scripted-to-jar.sh
 
 clone:
   git:
@@ -30,6 +33,7 @@ pipeline:
       - export DRONE_DIR="/drone"
       - git fetch --tags && git log | head -n 20
       - ./bin/run-ci.sh
+      - ${RUN_SCRIPTED}
 
   rebuild_cache:
     image: appleboy/drone-sftp-cache

bin/run-ci-scripted-to-jar.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -eu
+set -o nounset
+
+PROJECT_ROOT="zincRoot"
+sbt -Dfile.encoding=UTF-8 \
+  -J-XX:ReservedCodeCacheSize=512M \
+  -J-Xms1024M -J-Xmx4096M -J-server \
+  "zincScripted/test:run --to-jar"

bin/run-ci-scripted.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -eu
+set -o nounset
+
+PROJECT_ROOT="zincRoot"
+sbt -Dfile.encoding=UTF-8 \
+  -J-XX:ReservedCodeCacheSize=512M \
+  -J-Xms1024M -J-Xmx4096M -J-server \
+  "zincScripted/test:run"

bin/run-ci.sh

@@ -13,5 +13,4 @@ sbt -Dfile.encoding=UTF-8 \
   "$PROJECT_ROOT/test:compile" \
   crossTestBridges \
   "publishBridges" \
-  "$PROJECT_ROOT/test" \
-  "zincScripted/test:run"
+  "$PROJECT_ROOT/test"

build.sbt

@@ -165,6 +165,15 @@ lazy val zinc = (project in file("zinc"))
   .settings(
     name := "zinc",
     mimaSettings,
+    mimaBinaryIssueFilters ++= Seq(
+      exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCompilerImpl.compileIncrementally"),
+      exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCompilerImpl.inputs"),
+      exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCompilerImpl.compile"),
+      exclude[DirectMissingMethodProblem]("sbt.internal.inc.MixedAnalyzingCompiler.config"),
+      exclude[DirectMissingMethodProblem]("sbt.internal.inc.MixedAnalyzingCompiler.makeConfig"),
+      exclude[DirectMissingMethodProblem]("sbt.internal.inc.MixedAnalyzingCompiler.this"),
+      exclude[DirectMissingMethodProblem]("sbt.internal.inc.CompileConfiguration.this")
+    )
   )
 
 lazy val zincTesting = (project in internalPath / "zinc-testing")
@@ -281,7 +290,11 @@ lazy val zincCore = (project in internalPath / "zinc-core")
         exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.findClassDependencies"),
         exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidateClassesInternally"),
         exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidateClassesExternally"),
-        exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.findAPIChange")
+        exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.findAPIChange"),
+        exclude[IncompatibleMethTypeProblem]("sbt.internal.inc.Incremental.prune"),
+        exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCompile.apply"),
+        exclude[DirectMissingMethodProblem]("sbt.internal.inc.AnalysisCallback#Builder.this"),
+        exclude[DirectMissingMethodProblem]("sbt.internal.inc.AnalysisCallback.this")
       )
     }
   )
@@ -391,7 +404,10 @@ lazy val compilerInterface212 = (project in internalPath / "compiler-interface")
         exclude[ReversedMissingMethodProblem]("xsbti.compile.ExternalHooks#Lookup.hashClasspath"),
         exclude[ReversedMissingMethodProblem]("xsbti.compile.ScalaInstance.loaderLibraryOnly"),
         exclude[DirectMissingMethodProblem]("xsbti.api.AnalyzedClass.of"),
-        exclude[DirectMissingMethodProblem]("xsbti.api.AnalyzedClass.create")
+        exclude[DirectMissingMethodProblem]("xsbti.api.AnalyzedClass.create"),
+        exclude[ReversedMissingMethodProblem]("xsbti.AnalysisCallback.classesInOutputJar"),
+        exclude[ReversedMissingMethodProblem]("xsbti.compile.IncrementalCompiler.compile"),
+        exclude[DirectMissingMethodProblem]("xsbti.compile.IncrementalCompiler.compile")
       )
     },
   )
@@ -745,7 +761,10 @@ lazy val zincClassfile212 = zincClassfileTemplate
   .settings(
     scalaVersion := scala212,
     crossScalaVersions := Seq(scala212),
-    target := (target in zincClassfileTemplate).value.getParentFile / "target-2.12"
+    target := (target in zincClassfileTemplate).value.getParentFile / "target-2.12",
+    mimaBinaryIssueFilters ++= Seq(
+      exclude[DirectMissingMethodProblem]("sbt.internal.inc.classfile.Analyze.apply")
+    )
   )
 
 // re-implementation of scripted engine
@@ -831,6 +850,7 @@ lazy val otherRootSettings = Seq(
   Scripted.scriptedPrescripted := { (_: File) => () },
   Scripted.scriptedUnpublished := scriptedUnpublishedTask.evaluated,
   Scripted.scriptedSource := (sourceDirectory in zinc).value / "sbt-test",
+  Scripted.scriptedCompileToJar := false,
   publishAll := {
     val _ = (publishLocal).all(ScopeFilter(inAnyProject)).value
   }
@@ -845,6 +865,7 @@ def scriptedTask: Def.Initialize[InputTask[Unit]] = Def.inputTask {
     scriptedSource.value,
     result,
     scriptedBufferLog.value,
+    scriptedCompileToJar.value,
     scriptedPrescripted.value
   )
 }
@@ -857,6 +878,7 @@ def scriptedUnpublishedTask: Def.Initialize[InputTask[Unit]] = Def.inputTask {
     scriptedSource.value,
     result,
     scriptedBufferLog.value,
+    scriptedCompileToJar.value,
     scriptedPrescripted.value
   )
 }

internal/compiler-bridge/src/main/scala/xsbt/API.scala

@@ -96,7 +96,7 @@ final class API(val global: CallbackGlobal) extends Compat with GlobalHelpers wi
    *
    * This method only takes care of non-local classes because local classes have no
    * relevance in the correctness of the algorithm and can be registered after genbcode.
-   * Local classes are only used to contruct the relations of products and to produce
+   * Local classes are only used to construct the relations of products and to produce
    * the list of generated files + stamps, but names referring to local classes **never**
    * show up in the name hashes of classes' APIs, hence never considered for name hashing.
    *
@@ -116,9 +116,17 @@ final class API(val global: CallbackGlobal) extends Compat with GlobalHelpers wi
       def registerProductNames(names: FlattenedNames): Unit = {
         // Guard against a local class in case it surreptitiously leaks here
         if (!symbol.isLocalClass) {
-          val classFileName = s"${names.binaryName}.class"
-          val outputDir = global.settings.outputDirs.outputDirFor(sourceFile).file
-          val classFile = new java.io.File(outputDir, classFileName)
+          val pathToClassFile = s"${names.binaryName}.class"
+          val classFile = {
+            JarUtils.outputJar match {
+              case Some(outputJar) =>
+                new java.io.File(JarUtils.classNameInJar(outputJar, pathToClassFile))
+              case None =>
+                val outputDir = global.settings.outputDirs.outputDirFor(sourceFile).file
+                new java.io.File(outputDir, pathToClassFile)
+            }
+          }
+
           val zincClassName = names.className
           val srcClassName = classNameAsString(symbol)
           callback.generatedNonLocalClass(sourceJavaFile, classFile, zincClassName, srcClassName)

internal/compiler-bridge/src/main/scala/xsbt/Analyzer.scala

@@ -7,7 +7,10 @@
 
 package xsbt
 
+import java.io.File
+
 import scala.tools.nsc.Phase
+import scala.collection.JavaConverters._
 
 object Analyzer {
   def name = "xsbt-analyzer"
@@ -22,15 +25,39 @@ final class Analyzer(val global: CallbackGlobal) extends LocateClassFile {
       "Finds concrete instances of provided superclasses, and application entry points."
     def name = Analyzer.name
 
+    /**
+     * When straight-to-jar compilation is enabled, returns the classes
+     * that are found in the jar of the last compilation. This method
+     * gets the existing classes from the analysis callback and adapts
+     * it for consumption in the compiler bridge.
+     *
+     * It's lazy because it triggers a read of the zip, which may be
+     * unnecessary if there are no local classes in a compilation unit.
+     */
+    private lazy val classesWrittenByGenbcode: Set[String] = {
+      JarUtils.outputJar match {
+        case Some(jar) =>
+          val classes = global.callback.classesInOutputJar().asScala
+          classes.map(JarUtils.classNameInJar(jar, _)).toSet
+        case None => Set.empty
+      }
+    }
+
     def apply(unit: CompilationUnit): Unit = {
       if (!unit.isJava) {
         val sourceFile = unit.source.file.file
         for (iclass <- unit.icode) {
           val sym = iclass.symbol
-          val outputDir = settings.outputDirs.outputDirFor(sym.sourceFile).file
+          lazy val outputDir = settings.outputDirs.outputDirFor(sym.sourceFile).file
           def addGenerated(separatorRequired: Boolean): Unit = {
-            val classFile = fileForClass(outputDir, sym, separatorRequired)
-            if (classFile.exists()) {
+            val locatedClass = {
+              JarUtils.outputJar match {
+                case Some(outputJar) => locateClassInJar(sym, outputJar, separatorRequired)
+                case None            => locatePlainClassFile(sym, separatorRequired)
+              }
+            }
+
+            locatedClass.foreach { classFile =>
               assert(sym.isClass, s"${sym.fullName} is not a class")
               // Use own map of local classes computed before lambdalift to ascertain class locality
               if (localToNonLocalClass.isLocal(sym).getOrElse(true)) {
@@ -49,5 +76,18 @@ final class Analyzer(val global: CallbackGlobal) extends LocateClassFile {
         }
       }
     }
+
+    private def locatePlainClassFile(sym: Symbol, separatorRequired: Boolean): Option[File] = {
+      val outputDir = settings.outputDirs.outputDirFor(sym.sourceFile).file
+      val classFile = fileForClass(outputDir, sym, separatorRequired)
+      if (classFile.exists()) Some(classFile) else None
+    }
+
+    private def locateClassInJar(sym: Symbol, jar: File, sepRequired: Boolean): Option[File] = {
+      val classFile = pathToClassFile(sym, sepRequired)
+      val classInJar = JarUtils.classNameInJar(jar, classFile)
+      if (!classesWrittenByGenbcode.contains(classInJar)) None
+      else Some(new File(classInJar))
+    }
   }
 }

internal/compiler-bridge/src/main/scala/xsbt/CallbackGlobal.scala

@@ -14,6 +14,8 @@ import scala.tools.nsc._
 import io.AbstractFile
 import java.io.File
 
+import scala.reflect.io.PlainFile
+
 /** Defines the interface of the incremental compiler hiding implementation details. */
 sealed abstract class CallbackGlobal(settings: Settings,
                                      reporter: reporters.Reporter,
@@ -38,6 +40,8 @@ sealed abstract class CallbackGlobal(settings: Settings,
     }
   }
 
+  lazy val JarUtils = new JarUtils(outputDirs)
+
   /**
    * Defines the sbt phase in which the dependency analysis is performed.
    * The reason why this is exposed in the callback global is because it's used
@@ -134,19 +138,44 @@ sealed class ZincCompiler(settings: Settings, dreporter: DelegatingReporter, out
 
   private final val fqnsToAssociatedFiles = perRunCaches.newMap[String, (AbstractFile, Boolean)]()
 
-  /** Returns the associated file of a fully qualified name and whether it's on the classpath. */
+  /**
+   * Returns the associated file of a fully qualified name and whether it's on the classpath.
+   * Note that the abstract file returned must exist.
+   */
   def findAssociatedFile(fqn: String): Option[(AbstractFile, Boolean)] = {
-    def getOutputClass(name: String): Option[AbstractFile] = {
-      // This could be improved if a hint where to look is given.
-      val className = name.replace('.', '/') + ".class"
-      outputDirs.map(new File(_, className)).find((_.exists)).map((AbstractFile.getFile(_)))
+    def findOnPreviousCompilationProducts(name: String): Option[AbstractFile] = {
+      // This class file path is relative to the output jar/directory and computed from class name
+      val classFilePath = name.replace('.', '/') + ".class"
+
+      JarUtils.outputJar match {
+        case Some(outputJar) =>
+          if (!callback.classesInOutputJar().contains(classFilePath)) None
+          else {
+            /*
+             * Important implementation detail: `classInJar` has the format of `$JAR!$CLASS_REF`
+             * which is, of course, a path to a file that does not exist. This file path is
+             * interpreted especially by Zinc to decompose the format under straight-to-jar
+             * compilation. For this strategy to work, `PlainFile` must **not** check that
+             * this file does exist or not because, if it does, it will return `null` in
+             * `processExternalDependency` and the dependency will not be correctly registered.
+             * If scalac breaks this contract (the check for existence is done when creating
+             * a normal reflect file but not a plain file), Zinc will not work correctly.
+             */
+            Some(new PlainFile(JarUtils.classNameInJar(outputJar, classFilePath)))
+          }
+
+        case None => // The compiler outputs class files in a classes directory (the default)
+          // This lookup could be improved if a hint where to look is given.
+          outputDirs.map(new File(_, classFilePath)).find(_.exists()).map(AbstractFile.getFile(_))
+      }
     }
 
     def findOnClassPath(name: String): Option[AbstractFile] =
       classPath.findClass(name).flatMap(_.binary.asInstanceOf[Option[AbstractFile]])
 
     fqnsToAssociatedFiles.get(fqn).orElse {
-      val newResult = getOutputClass(fqn).map(f => (f, true))
+      val newResult = findOnPreviousCompilationProducts(fqn)
+        .map(f => (f, true))
         .orElse(findOnClassPath(fqn).map(f => (f, false)))
       newResult.foreach(res => fqnsToAssociatedFiles.put(fqn, res))
       newResult

internal/compiler-bridge/src/main/scala/xsbt/JarUtils.scala

@@ -0,0 +1,37 @@
+package xsbt
+
+import java.io.File
+
+/**
+ * This is a utility class that provides a set of functions that
+ * are used to implement straight to jar compilation.
+ *
+ * [[sbt.internal.inc.JarUtils]] is an object that has similar purpose and
+ * duplicates some of the code, as it is difficult to share it. Any change
+ * in the logic of this file must be applied to the other `JarUtils` too!
+ */
+final class JarUtils(outputDirs: Iterable[File]) {
+  // This is an equivalent of asking if it runs on Windows where the separator is `\`
+  private val isSlashSeparator: Boolean = File.separatorChar == '/'
+
+  /**
+   * The jar file that is used as output for classes. If the output is
+   * not set to a single .jar file, value of this field is [[None]].
+   */
+  val outputJar: Option[File] = {
+    outputDirs match {
+      case Seq(file) if file.getName.endsWith(".jar") => Some(file)
+      case _                                          => None
+    }
+  }
+
+  /**
+   * Creates an identifier for a class located inside a jar.
+   *
+   * It follows the format to encode inter-jar dependencies that
+   * is established in [[sbt.internal.inc.JarUtils.ClassInJar]].
+   */
+  def classNameInJar(jar: File, classFilePath: String): String = {
+    s"$jar!${if (isSlashSeparator) classFilePath else classFilePath.replace('/', File.separatorChar)}"
+  }
+}

internal/compiler-bridge/src/main/scala/xsbt/LocateClassFile.scala

@@ -43,4 +43,7 @@ abstract class LocateClassFile extends Compat with ClassName {
 
   protected def fileForClass(outputDirectory: File, s: Symbol, separatorRequired: Boolean): File =
     new File(outputDirectory, flatclassName(s, File.separatorChar, separatorRequired) + ".class")
+
+  protected def pathToClassFile(s: Symbol, separatorRequired: Boolean): String =
+    flatclassName(s, File.separatorChar, separatorRequired) + ".class"
 }