Add an option to disable incremental compilation with zinc (#2851)

This change add an option to force a full recompile for a given module.

Pull request: #2851

Co-authored-by: Tobias Roeser <le.petit.fou@web.de>

docs/modules/ROOT/pages/Scala_Module_Config.adoc

@@ -144,3 +144,22 @@ As a consequence, Ammonite needs full Scala version specific releases.
 
 The older your used Mill version or the newer the Scala version you want to use, the higher is the risk that the default Ammonite version will not match.
 --
+
+== Disabling incremental compilation with Zinc
+
+By default all `ScalaModule`s use incremental compilation via https://github.com/sbt/zinc[Zinc] to
+only recompile sources that have changed since the last compile, or ones that have been invalidated
+by changes to upstream sources.
+
+If for any reason you want to disable incremental compilation for a module, you can override and set
+`zincIncrementalCompilation` to `false`
+
+.`build.sc`
+[source,scala,subs="attributes,verbatim"]
+----
+import mill._, scalalib._
+
+object foo extends ScalaModule {
+  def zincIncrementalCompilation = false
+}
+----

scalalib/api/src/mill/scalalib/api/ZincWorkerApi.scala

@@ -3,22 +3,83 @@ package mill.scalalib.api
 import mill.api.{CompileProblemReporter, PathRef}
 import mill.api.Loose.Agg
 
+import scala.annotation.nowarn
+
 object ZincWorkerApi {
   type Ctx = mill.api.Ctx.Dest with mill.api.Ctx.Log with mill.api.Ctx.Home
 }
 trait ZincWorkerApi {
 
   /** Compile a Java-only project */
+  def compileJava(
+      upstreamCompileOutput: Seq[CompilationResult],
+      sources: Agg[os.Path],
+      compileClasspath: Agg[os.Path],
+      javacOptions: Seq[String],
+      reporter: Option[CompileProblemReporter],
+      reportCachedProblems: Boolean,
+      incrementalCompilation: Boolean
+  )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] =
+    compileJava(
+      upstreamCompileOutput = upstreamCompileOutput,
+      sources = sources,
+      compileClasspath = compileClasspath,
+      javacOptions = javacOptions,
+      reporter = reporter,
+      reportCachedProblems = reportCachedProblems
+    ): @nowarn("cat=deprecation")
+
+  /** Compile a Java-only project */
+  @deprecated("Use override with `incrementalCompilation` parameter", "Mill 0.11.6")
   def compileJava(
       upstreamCompileOutput: Seq[CompilationResult],
       sources: Agg[os.Path],
       compileClasspath: Agg[os.Path],
       javacOptions: Seq[String],
       reporter: Option[CompileProblemReporter],
       reportCachedProblems: Boolean
-  )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult]
+  )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] =
+    compileJava(
+      upstreamCompileOutput = upstreamCompileOutput,
+      sources = sources,
+      compileClasspath = compileClasspath,
+      javacOptions = javacOptions,
+      reporter = reporter,
+      reportCachedProblems = reportCachedProblems,
+      incrementalCompilation = true
+    )
+
+  /** Compile a mixed Scala/Java or Scala-only project */
+  def compileMixed(
+      upstreamCompileOutput: Seq[CompilationResult],
+      sources: Agg[os.Path],
+      compileClasspath: Agg[os.Path],
+      javacOptions: Seq[String],
+      scalaVersion: String,
+      scalaOrganization: String,
+      scalacOptions: Seq[String],
+      compilerClasspath: Agg[PathRef],
+      scalacPluginClasspath: Agg[PathRef],
+      reporter: Option[CompileProblemReporter],
+      reportCachedProblems: Boolean,
+      incrementalCompilation: Boolean
+  )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] =
+    compileMixed(
+      upstreamCompileOutput = upstreamCompileOutput,
+      sources = sources,
+      compileClasspath = compileClasspath,
+      javacOptions = javacOptions,
+      scalaVersion = scalaVersion,
+      scalaOrganization = scalaOrganization,
+      scalacOptions = scalacOptions,
+      compilerClasspath = compilerClasspath,
+      scalacPluginClasspath = scalacPluginClasspath,
+      reporter = reporter,
+      reportCachedProblems = reportCachedProblems
+    ): @nowarn("cat=deprecation")
 
   /** Compile a mixed Scala/Java or Scala-only project */
+  @deprecated("Use override with `incrementalCompilation` parameter", "Mill 0.11.6")
   def compileMixed(
       upstreamCompileOutput: Seq[CompilationResult],
       sources: Agg[os.Path],
@@ -31,7 +92,21 @@ trait ZincWorkerApi {
       scalacPluginClasspath: Agg[PathRef],
       reporter: Option[CompileProblemReporter],
       reportCachedProblems: Boolean
-  )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult]
+  )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] =
+    compileMixed(
+      upstreamCompileOutput = upstreamCompileOutput,
+      sources = sources,
+      compileClasspath = compileClasspath,
+      javacOptions = javacOptions,
+      scalaVersion = scalaVersion,
+      scalaOrganization = scalaOrganization,
+      scalacOptions = scalacOptions,
+      compilerClasspath = compilerClasspath,
+      scalacPluginClasspath = scalacPluginClasspath,
+      reporter = reporter,
+      reportCachedProblems = reportCachedProblems,
+      incrementalCompilation = true
+    )
 
   def discoverMainClasses(compilationResult: CompilationResult): Seq[String]
 

scalalib/src/mill/scalalib/JavaModule.scala

@@ -355,6 +355,10 @@ trait JavaModule
     ).equalsIgnoreCase("true")
   }
 
+  def zincIncrementalCompilation: T[Boolean] = T {
+    true
+  }
+
   /**
    * Compiles the current module to generate compiled classfiles/bytecode.
    *
@@ -370,7 +374,8 @@ trait JavaModule
         compileClasspath = compileClasspath().map(_.path),
         javacOptions = javacOptions(),
         reporter = T.reporter.apply(hashCode),
-        reportCachedProblems = zincReportCachedProblems()
+        reportCachedProblems = zincReportCachedProblems(),
+        incrementalCompilation = zincIncrementalCompilation()
       )
   }
 

scalalib/src/mill/scalalib/ScalaModule.scala

@@ -228,7 +228,8 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer =>
         compilerClasspath = scalaCompilerClasspath(),
         scalacPluginClasspath = scalacPluginClasspath(),
         reporter = T.reporter.apply(hashCode),
-        reportCachedProblems = zincReportCachedProblems()
+        reportCachedProblems = zincReportCachedProblems(),
+        incrementalCompilation = zincIncrementalCompilation()
       )
   }
 
@@ -579,7 +580,8 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer =>
         compilerClasspath = scalaCompilerClasspath(),
         scalacPluginClasspath = semanticDbPluginClasspath(),
         reporter = None,
-        reportCachedProblems = zincReportCachedProblems()
+        reportCachedProblems = zincReportCachedProblems(),
+        incrementalCompilation = zincIncrementalCompilation()
       )
       .map(r =>
         SemanticDbJavaModule.copySemanticdbFiles(r.classes.path, T.workspace, T.dest / "data")

scalalib/src/mill/scalalib/SemanticDbJavaModule.scala

@@ -15,6 +15,7 @@ trait SemanticDbJavaModule extends CoursierModule {
   def zincWorker: ModuleRef[ZincWorkerModule]
   def upstreamCompileOutput: T[Seq[CompilationResult]]
   def zincReportCachedProblems: T[Boolean]
+  def zincIncrementalCompilation: T[Boolean]
   def allSourceFiles: T[Seq[PathRef]]
   def compile: T[mill.scalalib.api.CompilationResult]
   def bspBuildTarget: BspBuildTarget
@@ -116,7 +117,8 @@ trait SemanticDbJavaModule extends CoursierModule {
           (compileClasspath() ++ resolvedSemanticDbJavaPluginIvyDeps()).map(_.path),
         javacOptions = javacOpts,
         reporter = None,
-        reportCachedProblems = zincReportCachedProblems()
+        reportCachedProblems = zincReportCachedProblems(),
+        incrementalCompilation = zincIncrementalCompilation()
       ).map(r =>
         SemanticDbJavaModule.copySemanticdbFiles(r.classes.path, T.workspace, T.dest / "data")
       )

scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala

@@ -23,7 +23,6 @@ import xsbti.compile.{
   ClasspathOptions,
   CompileAnalysis,
   CompileOrder,
-  CompileProgress,
   Compilers,
   IncOptions,
   JavaTools,
@@ -34,7 +33,6 @@ import xsbti.{PathBasedFile, VirtualFile}
 
 import java.io.File
 import java.util.Optional
-import java.util.concurrent.ConcurrentHashMap
 import scala.collection.mutable
 import scala.ref.SoftReference
 import scala.util.Properties.isWin
@@ -289,13 +287,14 @@ class ZincWorkerImpl(
       .getOrElse(Seq.empty[String])
   }
 
-  def compileJava(
+  override def compileJava(
       upstreamCompileOutput: Seq[CompilationResult],
       sources: Agg[os.Path],
       compileClasspath: Agg[os.Path],
       javacOptions: Seq[String],
       reporter: Option[CompileProblemReporter],
-      reportCachedProblems: Boolean
+      reportCachedProblems: Boolean,
+      incrementalCompilation: Boolean
   )(implicit ctx: ZincWorkerApi.Ctx): Result[CompilationResult] = {
     compileInternal(
       upstreamCompileOutput = upstreamCompileOutput,
@@ -305,7 +304,8 @@ class ZincWorkerImpl(
       scalacOptions = Nil,
       compilers = javaOnlyCompilers(javacOptions),
       reporter = reporter,
-      reportCachedProblems = reportCachedProblems
+      reportCachedProblems = reportCachedProblems,
+      incrementalCompilation = incrementalCompilation
     )
   }
 
@@ -320,7 +320,8 @@ class ZincWorkerImpl(
       compilerClasspath: Agg[PathRef],
       scalacPluginClasspath: Agg[PathRef],
       reporter: Option[CompileProblemReporter],
-      reportCachedProblems: Boolean
+      reportCachedProblems: Boolean,
+      incrementalCompilation: Boolean
   )(implicit ctx: ZincWorkerApi.Ctx): Result[CompilationResult] = {
     withCompilers(
       scalaVersion = scalaVersion,
@@ -337,7 +338,8 @@ class ZincWorkerImpl(
         scalacOptions = scalacOptions,
         compilers = compilers,
         reporter = reporter,
-        reportCachedProblems: Boolean
+        reportCachedProblems: Boolean,
+        incrementalCompilation
       )
     }
   }
@@ -419,7 +421,8 @@ class ZincWorkerImpl(
       scalacOptions: Seq[String],
       compilers: Compilers,
       reporter: Option[CompileProblemReporter],
-      reportCachedProblems: Boolean
+      reportCachedProblems: Boolean,
+      incrementalCompilation: Boolean
   )(implicit ctx: ZincWorkerApi.Ctx): Result[CompilationResult] = {
     os.makeDir.all(ctx.dest)
 
@@ -514,12 +517,17 @@ class ZincWorkerImpl(
         earlyAnalysisStore = None,
         extra = Array()
       ),
-      pr = {
+      pr = if (incrementalCompilation) {
         val prev = store.get()
         PreviousResult.of(
           prev.map(_.getAnalysis): Optional[CompileAnalysis],
           prev.map(_.getMiniSetup): Optional[MiniSetup]
         )
+      } else {
+        PreviousResult.of(
+          Optional.empty[CompileAnalysis],
+          Optional.empty[MiniSetup]
+        )
       },
       temporaryClassesDirectory = java.util.Optional.empty(),
       converter = converter,