In-memory shading

core/src/main/scala/com/eed3si9n/jarjarabrams/Main.scala

@@ -17,8 +17,19 @@ class Main {
       .toList
       .map(Shader.toShadeRule)
     val verbose = java.lang.Boolean.getBoolean("verbose")
-    val skipManifest = java.lang.Boolean.getBoolean("skipManifest");
-    Shader.shadeFile(rules, inJar, outJar, verbose, skipManifest)
+    val skipManifest = java.lang.Boolean.getBoolean("skipManifest")
+    val resetTimestamp = sys.props.get("resetTimestamp") match {
+      case Some(_) => java.lang.Boolean.getBoolean("resetTimestamp")
+      case None    => true
+    }
+    Shader.shadeFile(
+      rules,
+      inJar,
+      outJar,
+      verbose,
+      skipManifest,
+      resetTimestamp
+    )
   }
 }
 

core/src/main/scala/com/eed3si9n/jarjarabrams/Shader.scala

@@ -3,7 +3,7 @@ package com.eed3si9n.jarjarabrams
 import java.nio.file.{ Files, Path, StandardOpenOption }
 import com.eed3si9n.jarjar.{ JJProcessor, _ }
 import com.eed3si9n.jarjar.util.EntryStruct
-import Zip.{ createDirectories, resetModifiedTime }
+import Zip.createDirectories
 import scala.collection.JavaConverters._
 
 object Shader {
@@ -12,27 +12,16 @@ object Shader {
       inputJar: Path,
       outputJar: Path,
       verbose: Boolean,
-      skipManifest: Boolean
+      skipManifest: Boolean,
+      resetTimestamp: Boolean
   ): Unit = {
-    val tempDir = Files.createTempDirectory("jarjar-in")
-    val outDir = Files.createTempDirectory("jarjar-out")
-    val tempJar = Files.createTempFile("jarjar", ".jar")
-    Zip.unzip(inputJar, tempDir)
-    shadeDirectory(
-      rules,
-      outDir,
-      makeMappings(tempDir),
-      verbose = verbose,
-      skipManifest = skipManifest
-    )
-    Zip.zip(makeMappings(outDir), tempJar)
-    resetModifiedTime(tempJar)
-    if (Files.exists(outputJar)) {
-      Files.delete(outputJar)
+    val shader = bytecodeShader(rules, verbose, skipManifest)
+    Zip.flatMap(inputJar, outputJar, resetTimestamp) { struct0 =>
+      shader(struct0.data, struct0.name).map {
+        case (shadedBytes, shadedName) =>
+          Zip.entryStruct(shadedName, struct0.time, shadedBytes, struct0.skipTransform)
+      }
     }
-    createDirectories(outputJar.getParent)
-    Files.copy(tempJar, outputJar)
-    resetModifiedTime(outputJar)
   }
 
   def makeMappings(dir: Path): List[(Path, String)] =

core/src/main/scala/com/eed3si9n/jarjarabrams/Using.scala

@@ -10,6 +10,7 @@ import java.io.{
   OutputStream
 }
 import java.nio.file.{ Files, Path }
+import java.util.jar.{ JarFile, JarOutputStream }
 import java.util.zip.ZipInputStream
 import scala.util.control.NonFatal
 import scala.reflect.ClassTag
@@ -30,9 +31,12 @@ abstract class Using[Source, A] {
 object Using {
   def fileOutputStream(append: Boolean): Using[Path, OutputStream] =
     file(f => new BufferedOutputStream(new FileOutputStream(f.toFile, append)))
+  val jarOutputStream: Using[Path, JarOutputStream] =
+    file(f => new JarOutputStream(new BufferedOutputStream(new FileOutputStream(f.toFile))))
   val fileInputStream: Using[Path, InputStream] =
     file(f => new BufferedInputStream(new FileInputStream(f.toFile)))
-
+  val jarFile: Using[Path, JarFile] =
+    file(f => new JarFile(f.toFile))
   val zipInputStream = wrap((in: InputStream) => new ZipInputStream(in))
 
   def file[A1 <: AutoCloseable](action: Path => A1): Using[Path, A1] =

core/src/main/scala/com/eed3si9n/jarjarabrams/Zip.scala

@@ -1,192 +1,134 @@
 package com.eed3si9n.jarjarabrams
 
+import com.eed3si9n.jarjar.util.EntryStruct
 import java.nio.file.{ Files, NoSuchFileException, Path }
 import java.nio.file.attribute.FileTime
-import java.io.{ BufferedOutputStream, File, FileNotFoundException, InputStream, OutputStream }
-import java.util.jar.{ Attributes, JarEntry, JarFile, JarOutputStream, Manifest }
-import java.util.zip.{ CRC32, ZipEntry, ZipInputStream, ZipOutputStream }
+import java.io.{ ByteArrayOutputStream, FileNotFoundException, InputStream, OutputStream }
+import java.util.jar.JarEntry
 import scala.annotation.tailrec
-import scala.collection.immutable.TreeSet
+import scala.collection.JavaConverters._
 import scala.collection.mutable
-import scala.util.control.NonFatal
 
 object Zip {
 
   /** The size of the byte or char buffer used in various methods. */
   private final val BufferSize = 8192
   // 2010-01-01
   private final val default2010Timestamp = 1262304000000L
-
-  def zip(sources: Iterable[(Path, String)], outputJar: Path): Unit =
-    archive(sources.toSeq, outputJar, None, default2010Timestamp)
-
-  def unzip(from: Path, toDirectory: Path): Set[Path] =
-    Using.fileInputStream(from)(in => unzipStream(in, toDirectory))
-
-  private def archive(
-      mapping: Seq[(Path, String)],
-      outputFile: Path,
-      manifest: Option[Manifest],
-      time: Long
-  ) = {
-    // The zip 'setTime' methods try to convert from the given time to the local time based
-    // on java.util.TimeZone.getDefault(). When explicitly specifying the timestamp, we assume
-    // this already has been done if desired, so we need to 'convert back' here:
-    val localTime = time - java.util.TimeZone.getDefault().getOffset(time)
-    if (Files.isDirectory(outputFile))
-      sys.error(s"specified output file $outputFile is a directory.")
-    else {
-      val outputDir = outputFile.getParent
-      createDirectories(outputDir)
-      withZipOutput(outputFile, manifest, localTime) { output =>
-        val createEntry: (String => ZipEntry) =
-          if (manifest.isDefined) new JarEntry(_) else new ZipEntry(_)
-        writeZip(mapping, output, localTime)(createEntry)
-      }
-    }
-  }
-
-  private def writeZip(sources: Seq[(Path, String)], output: ZipOutputStream, time: Long)(
-      createEntry: String => ZipEntry
-  ) = {
-    val files = sources
-      .flatMap {
-        case (file, name) =>
-          if (Files.isRegularFile(file)) (file, normalizeToSlash(name)) :: Nil
-          else Nil
-      }
-      .sortBy {
-        case (_, name) =>
-          name
-      }
-
-    // The CRC32 for an empty value, needed to store directories in zip files
-    val emptyCRC = new CRC32().getValue()
-
-    def addDirectoryEntry(name: String) = {
-      output putNextEntry makeDirectoryEntry(name)
-      output.closeEntry()
-    }
-
-    def makeDirectoryEntry(name: String) = {
-      val e = createEntry(name)
-      e.setTime(time)
-      e.setSize(0)
-      e.setMethod(ZipEntry.STORED)
-      e.setCrc(emptyCRC)
-      e
-    }
-
-    def makeFileEntry(file: Path, name: String) = {
-      val e = createEntry(name)
-      e.setTime(time)
-      e
-    }
-    def addFileEntry(file: Path, name: String) = {
-      output putNextEntry makeFileEntry(file, name)
-      transfer(file, output)
-      output.closeEntry()
-    }
-
-    // Calculate directories and add them to the generated Zip
-    allDirectoryPaths(files).foreach(addDirectoryEntry(_))
-
-    // Add all files to the generated Zip
-    files.foreach { case (file, name) => addFileEntry(file, name) }
-  }
-
-  private def normalizeToSlash(name: String) = {
-    val sep = File.separatorChar
-    if (sep == '/') name else name.replace(sep, '/')
-  }
-
-  // map a path a/b/c to List("a", "b")
-  private def relativeComponents(path: String): List[String] =
-    path.split("/").toList.dropRight(1)
-
-  // map components List("a", "b", "c") to List("a/b/c/", "a/b/", "a/", "")
-  private def directories(path: List[String]): List[String] =
-    path.foldLeft(List(""))((e, l) => (e.head + l + "/") :: e)
-
-  // map a path a/b/c to List("a/b/", "a/")
-  private def directoryPaths(path: String): List[String] =
-    directories(relativeComponents(path)).filter(_.length > 1)
-
-  // produce a sorted list of all the subdirectories of all provided files
-  private def allDirectoryPaths(files: Iterable[(Path, String)]) =
-    TreeSet[String]() ++ (files.flatMap { case (_, name) => directoryPaths(name) })
-
-  private def withZipOutput(file: Path, manifest: Option[Manifest], time: Long)(
-      f: ZipOutputStream => Unit
-  ) = {
-    Using.fileOutputStream(false)(file) { fileOut =>
-      val (zipOut, _) =
-        manifest match {
-          case Some(mf) =>
-            import Attributes.Name.MANIFEST_VERSION
-            val main = mf.getMainAttributes
-            if (!main.containsKey(MANIFEST_VERSION))
-              main.put(MANIFEST_VERSION, "1.0")
-            val os = new JarOutputStream(fileOut)
-            val e = new ZipEntry(JarFile.MANIFEST_NAME)
-            e.setTime(time)
-            os.putNextEntry(e)
-            mf.write(new BufferedOutputStream(os))
-            os.closeEntry()
-            (os, "jar")
-          case None => (new ZipOutputStream(fileOut), "zip")
-        }
-      try {
-        f(zipOut)
-      } finally {
-        zipOut.close
+  // ZIP timestamps have a resolution of 2 seconds.
+  // see http://www.info-zip.org/FAQ.html#limits
+  final private val minimumTimestampIncrement = 2000L
+  // 1980-01-03
+  // Technically it should be 1980-01-01, but Java checks against
+  // the year segment in localtime and ends up capturing timezone,
+  // so to be safe we should consider Jan 3 to be the minimum.
+  //
+  //   $ unzip -l example/byte-buddy-agent.jar | head
+  // Archive:  example/byte-buddy-agent.jar
+  //   Length      Date    Time    Name
+  // ---------  ---------- -----   ----
+  //         0  00-00-1980 04:08   META-INF/
+  //       988  00-00-1980 04:08   META-INF/MANIFEST.MF
+  private final val minimumTimestamp = 315705600L
+
+  def list(inputJar: Path): List[(String, Long)] =
+    Using.jarFile(inputJar) { in =>
+      in.entries.asScala.toList.map { entry =>
+        (entry.getName, entry.getTime)
       }
     }
-  }
-
-  private def unzipStream(
-      from: InputStream,
-      toDirectory: Path,
-      preserveLastModified: Boolean = false
-  ): Set[Path] = {
-    createDirectories(toDirectory)
-    Using.zipInputStream(from)(zipInput => extract(zipInput, toDirectory, preserveLastModified))
-  }
 
-  private def extract(
-      from: ZipInputStream,
-      toDirectory: Path,
-      preserveLastModified: Boolean
-  ) = {
-    val set = new mutable.HashSet[Path]
-    @tailrec def next(): Unit = {
-      val entry = from.getNextEntry
-      if (entry == null) ()
-      else {
-        val name = entry.getName
-        val target = toDirectory.resolve(name)
-        if (entry.isDirectory) createDirectories(target)
-        else {
-          set += target
-          try {
-            Using.fileOutputStream(false)(target)(out => transfer(from, out))
-          } catch {
-            case NonFatal(e) =>
-              throw new RuntimeException(
-                "error extracting Zip entry '" + name + "' to '" + target + "'",
-                e
-              )
+  /**
+   * Treating JAR file as a set of EntryStruct, this implements a
+   * functional processing, intended for in-memory shading.
+   */
+  def flatMap(
+      inputJar: Path,
+      outputJar: Path,
+      resetTimestamp: Boolean
+  )(f: EntryStruct => Option[EntryStruct]): Path =
+    Using.jarFile(inputJar) { in =>
+      val tempJar = Files.createTempFile("jarjar", ".jar")
+      val names = new mutable.HashSet[String]
+      Using.jarOutputStream(tempJar) { out =>
+        in.entries.asScala.toList.foreach { entry0 =>
+          val struct0 = entryStruct(
+            entry0.getName,
+            entry0.getTime, {
+              val baos = new ByteArrayOutputStream()
+              transfer(in.getInputStream(entry0), baos)
+              baos.toByteArray()
+            },
+            false
+          )
+          f(struct0) match {
+            case Some(struct) =>
+              if (names.add(struct.name)) {
+                val entry = new JarEntry(struct.name)
+                val time =
+                  if (resetTimestamp) hardcodedZipTimestamp(struct.name)
+                  else enforceMinimum(struct.time)
+                entry.setTime(time)
+                entry.setCompressedSize(-1)
+                out.putNextEntry(entry)
+                out.write(struct.data)
+              } else if (struct.name.endsWith("/")) ()
+              else {
+                // duplicate entry
+                // throw?
+              }
+            case None => ()
           }
         }
-        if (preserveLastModified) setModifiedTimeOrFalse(target, entry.getTime)
-        else resetModifiedTime(target)
-
-        from.closeEntry()
-        next()
       }
+      if (Files.exists(outputJar)) {
+        Files.delete(outputJar)
+      }
+      Files.copy(tempJar, outputJar)
+      resetModifiedTime(outputJar)
+      outputJar
     }
-    next()
-    Set() ++ set
+
+  private val localized2010Timestamp = localizeTimestamp(default2010Timestamp)
+  private val localized2010TimestampPlus2s = localizeTimestamp(
+    default2010Timestamp + minimumTimestampIncrement
+  )
+  private val localizedMinimumTimestamp = localizeTimestamp(minimumTimestamp)
+
+  /**
+   * Returns the normalized timestamp for a jar entry based on its name.
+   * This is necessary supposedly since javac will, when loading a class X,
+   * prefer a source file to a class file, if both files have the same timestamp. Therefore, we need
+   * to adjust the timestamp for class files to slightly after the normalized time.
+   */
+  private def hardcodedZipTimestamp(name: String): Long =
+    if (name.endsWith(".class")) localized2010TimestampPlus2s
+    else localized2010Timestamp
+
+  private def enforceMinimum(timestampInLocal: Long): Long =
+    if (timestampInLocal < localizedMinimumTimestamp) localizedMinimumTimestamp
+    else timestampInLocal
+
+  /*
+   * The zip 'setTime' methods try to convert from the given time to the local time based
+   * on java.util.TimeZone.getDefault(). When explicitly specifying the timestamp, we assume
+   * this already has been done if desired, so we need to 'convert back' here.
+   */
+  private def localizeTimestamp(timestampInUtc: Long): Long =
+    timestampInUtc - java.util.TimeZone.getDefault().getOffset(timestampInUtc)
+
+  def entryStruct(
+      name: String,
+      time: Long,
+      data: Array[Byte],
+      skipTransform: Boolean = false
+  ): EntryStruct = {
+    val struct = new EntryStruct()
+    struct.name = name
+    struct.time = time
+    struct.data = data
+    struct.skipTransform = skipTransform
+    struct
   }
 
   def resetModifiedTime(file: Path): Boolean =