From e3ef411472a82388704b9aa19babe4da3910fa56 Mon Sep 17 00:00:00 2001
From: Tobias Roeser <le.petit.fou@web.de>
Date: Thu, 15 Feb 2024 21:34:50 +0100
Subject: [PATCH] Restructure `VersionFinder`, increase speed, added ticker
 messages (#3014)

Searching for updates can take quite some time, especially for larger
projects.

This change adds some progress messages via Mills ticker log messages.

More important, I restructured the implementation such that work is
distributed over more tasks. If the used evaluator is run with parallel
jobs, this will significantly increase the speed.

Pull request: https://github.com/com-lihaoyi/mill/pull/3014
---
 .../dependency/versions/VersionsFinder.scala  | 119 ++++++++++--------
 1 file changed, 65 insertions(+), 54 deletions(-)

diff --git a/scalalib/src/mill/scalalib/dependency/versions/VersionsFinder.scala b/scalalib/src/mill/scalalib/dependency/versions/VersionsFinder.scala
index e7859df9c8a..6fa52636b9d 100644
--- a/scalalib/src/mill/scalalib/dependency/versions/VersionsFinder.scala
+++ b/scalalib/src/mill/scalalib/dependency/versions/VersionsFinder.scala
@@ -1,13 +1,14 @@
 package mill.scalalib.dependency.versions
 
-import coursier.Dependency
-import mill.define.{BaseModule}
+import mill.define.{BaseModule, Task}
 import mill.eval.Evaluator
-import mill.scalalib.dependency.metadata.MetadataLoaderFactory
+import mill.scalalib.dependency.metadata.{MetadataLoader, MetadataLoaderFactory}
 import mill.scalalib.{JavaModule, Lib}
 import mill.api.Ctx.{Home, Log}
 import mill.T
 
+import java.util.concurrent.atomic.AtomicInteger
+
 private[dependency] object VersionsFinder {
 
   def findVersions(
@@ -20,61 +21,71 @@ private[dependency] object VersionsFinder {
       case javaModule: JavaModule => javaModule
     }
 
-    val resolvedDependencies = resolveDependencies(evaluator, javaModules)
-    resolveVersions(evaluator, resolvedDependencies)
-  }
+    val resolvedDependencies = evaluator.evalOrThrow() {
+      val progress = new Progress(javaModules.size)
+      javaModules.map(resolveDeps(progress))
+    }
 
-  private def resolveDependencies(
-      evaluator: Evaluator,
-      javaModules: Seq[JavaModule]
-  ): Seq[(JavaModule, Seq[Dependency])] = evaluator.evalOrThrow() {
-    javaModules.map { javaModule =>
-      T.task {
-        val bindDependency = javaModule.bindDependency()
-        val deps = javaModule.ivyDeps()
-        val compileIvyDeps = javaModule.compileIvyDeps()
-        val runIvyDeps = javaModule.runIvyDeps()
-        val repos = javaModule.repositoriesTask()
-        val mapDeps = javaModule.mapDependencies()
-        val custom = javaModule.resolutionCustomizer()
-        val cacheCustom = javaModule.coursierCacheCustomizer()
-
-        val (dependencies, _) =
-          Lib.resolveDependenciesMetadata(
-            repositories = repos,
-            deps = (deps ++ compileIvyDeps ++ runIvyDeps).map(bindDependency),
-            mapDependencies = Some(mapDeps),
-            customizer = custom,
-            coursierCacheCustomizer = cacheCustom,
-            ctx = Some(T.log)
-          )
-
-        (javaModule, dependencies)
-      }
+    evaluator.evalOrThrow() {
+      val progress = new Progress(resolvedDependencies.map(_._3.size).sum)
+      resolvedDependencies.map(resolveVersions(progress))
     }
   }
 
-  private def resolveVersions(
-      evaluator: Evaluator,
-      resolvedDependencies: Seq[ResolvedDependencies]
-  ): Seq[ModuleDependenciesVersions] =
-    resolvedDependencies.map {
-      case (javaModule, dependencies) =>
-        val metadataLoaders =
-          evaluator.evalOrThrow()(javaModule.repositoriesTask)
-            .flatMap(MetadataLoaderFactory(_))
-
-        val versions = dependencies.map { dependency =>
-          val currentVersion = Version(dependency.version)
-          val allVersions =
-            metadataLoaders
-              .flatMap(_.getVersions(dependency.module))
-              .toSet
-          DependencyVersions(dependency, currentVersion, allVersions)
-        }
-
-        ModuleDependenciesVersions(javaModule.toString, versions)
+  class Progress(val count: Int) {
+    private val counter = new AtomicInteger(1)
+    def next(): Int = counter.getAndIncrement()
+  }
+
+  private def resolveDeps(progress: Progress)(
+      javaModule: JavaModule
+  ): Task[ResolvedDependencies] =
+    T.task {
+      T.log.ticker(s"Resolving dependencies [${progress.next()}/${progress.count}]: ${javaModule}")
+
+      val bindDependency = javaModule.bindDependency()
+      val deps = javaModule.ivyDeps()
+      val compileIvyDeps = javaModule.compileIvyDeps()
+      val runIvyDeps = javaModule.runIvyDeps()
+      val repos = javaModule.repositoriesTask()
+      val mapDeps = javaModule.mapDependencies()
+      val custom = javaModule.resolutionCustomizer()
+      val cacheCustom = javaModule.coursierCacheCustomizer()
+
+      val metadataLoaders = repos.flatMap(MetadataLoaderFactory(_))
+
+      val (dependencies, _) =
+        Lib.resolveDependenciesMetadata(
+          repositories = repos,
+          deps = (deps ++ compileIvyDeps ++ runIvyDeps).map(bindDependency),
+          mapDependencies = Some(mapDeps),
+          customizer = custom,
+          coursierCacheCustomizer = cacheCustom,
+          ctx = Some(T.log)
+        )
+
+      (javaModule, metadataLoaders, dependencies)
+    }
+
+  private def resolveVersions(progres: Progress)(
+      resolvedDependencies: ResolvedDependencies
+  ): Task[ModuleDependenciesVersions] = T.task {
+    val (javaModule, metadataLoaders, dependencies) = resolvedDependencies
+
+    val versions = dependencies.map { dependency =>
+      T.log.ticker(
+        s"Analyzing dependencies [${progres.next()}/${progres.count}]: ${javaModule} / ${dependency.module}"
+      )
+      val currentVersion = Version(dependency.version)
+      val allVersions =
+        metadataLoaders
+          .flatMap(_.getVersions(dependency.module))
+          .toSet
+      DependencyVersions(dependency, currentVersion, allVersions)
     }
 
-  private type ResolvedDependencies = (JavaModule, Seq[coursier.Dependency])
+    ModuleDependenciesVersions(javaModule.toString, versions)
+  }
+
+  private type ResolvedDependencies = (JavaModule, Seq[MetadataLoader], Seq[coursier.Dependency])
 }