Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Android library publishing support #344

Open
wants to merge 5 commits into
base: development
Choose a base branch
from
Open

Android library publishing support #344

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f29cdf6
naive aar support
sugarmanz May 31, 2022
4f05c07
add packaging to deploy script
sugarmanz Jun 1, 2022
4bcfe8b
move exclusions to assemble target attribute
sugarmanz Jun 1, 2022
5f8b345
use neverlink attribute over explicit exclusions
sugarmanz Jun 1, 2022
286edb9
cleanup jar assembler
sugarmanz Jun 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 103 additions & 35 deletions maven/JarAssembler.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,20 @@ import java.io.BufferedInputStream
import java.io.BufferedOutputStream
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's cool that this AAR support is added with minimal code change! However, I think the domains of Java and Android are conceptually different (even though Android uses Java code extensively.) As such, I would propose:

  • Rename this file (and class) to Assembler. Given that we are in the maven package, that's unambiguous. Assembler should contain all the logic common to both JAR and AAR.
  • Introduce two inner classes of Assembler: Jar and Aar, which encapsulate the JAR and AAR-specific code. (They will, presumably, need a reference to the Assembler class in their constructors.)

import java.io.File
import java.io.FileOutputStream
import java.lang.RuntimeException
import java.io.InputStream
import java.nio.charset.Charset
import java.nio.file.Path
import java.nio.file.Paths
import java.util.concurrent.Callable
import java.util.zip.ZipEntry
import java.util.zip.ZipFile
import java.util.zip.ZipInputStream
import java.util.zip.ZipOutputStream
import kotlin.collections.HashMap
import kotlin.RuntimeException
import kotlin.system.exitProcess

typealias Entries = MutableMap<String, ByteArray>
private fun Entries(): Entries = mutableMapOf()

@Command(name = "jar-assembler", mixinStandardHelpOptions = true)
class JarAssembler : Callable<Unit> {
Expand All @@ -58,53 +61,118 @@ class JarAssembler : Callable<Unit> {
@Option(names = ["--jars"], split = ";")
lateinit var jars: Array<File>

private val entries = HashMap<String, ByteArray>()
private val entryNames = mutableSetOf<String>()

override fun call() {
ZipOutputStream(BufferedOutputStream(FileOutputStream(outputFile))).use { out ->
if (pomFile != null) {
Entries().apply {
pomFile?.readBytes()?.let { pomContents ->
val pomPath = "META-INF/maven/${groupId}/${artifactId}/pom.xml"
entries += preCreateDirectories(Paths.get(pomPath))
entries[pomPath] = pomFile!!.readBytes()
this += preCreateDirectories(Paths.get(pomPath))
this[pomPath] = pomContents
}
for (jar in jars) {
ZipFile(jar).use { jarZip ->
jarZip.entries().asSequence().forEach { entry ->
if (entryNames.contains(entry.name)) {
throw RuntimeException("duplicate entry in the JAR: ${entry.name}")
}
if (entry.name.contains("META-INF")) {
// pom.xml will be added by us
return@forEach
}
if (entry.isDirectory) {
// needed directories would be added by us
return@forEach
}
entryNames.add(entry.name)
BufferedInputStream(jarZip.getInputStream(entry)).use { inputStream ->
val sourceFileBytes = inputStream.readBytes()
val resultLocation = getFinalPath(entry, sourceFileBytes)
entries += preCreateDirectories(Paths.get(resultLocation))
entries[resultLocation] = sourceFileBytes

ZipOutputStream(BufferedOutputStream(FileOutputStream(outputFile))).use {
if (outputFile.extension == "aar") assembleAar(it)
else assembleClassesJar(it)
}
}
}

/** Assemble a class JAR containing the transitive class closure from [jars] and any pre-existing entries in [this] */
private fun Entries.assembleClassesJar(output: ZipOutputStream, jars: List<File> = this@JarAssembler.jars.toList()) {
for (jar in jars) {
if (jar.extension == "aar") {
throw RuntimeException("cannot package AAR within classes JAR")
}
processZip(ZipFile(jar))
}

writeEntries(output)
}

/** Assemble an AAR from a base AAR containing the transitive class closure from the additional [jars] and any pre-existing entries in [this] */
private fun Entries.assembleAar(output: ZipOutputStream) {
val classes = Entries()
processZip(jars.single { it.extension == "aar" }.let(::ZipFile)) { aar, entry ->
validateEntry(entry)?.let {
if (entry.name == "classes.jar") {
// pull out classes in nested JAR
entry.let(aar::getInputStream).let(::ZipInputStream).use { classesJar ->
var zipEntry: ZipEntry? = classesJar.nextEntry
while (zipEntry != null) {
classes.processEntry(classesJar, zipEntry)
zipEntry = classesJar.nextEntry
}
}
} else {
// add to top-level entries
processEntry(aar, entry)
}
}
entries.keys.sorted().forEach {
val newEntry = ZipEntry(it)
out.putNextEntry(newEntry)
out.write(entries[it]!!)
}
}

// write classes jar first
ZipEntry("classes.jar").let(output::putNextEntry)
val classJar = ZipOutputStream(output)
classes.assembleClassesJar(classJar, jars.filter { it.extension != "aar" })
classJar.finish()

// write the rest of the entries
writeEntries(output)
}

/** [process] each [ZipEntry] in [file] within the context of [this] */
private fun Entries.processZip(file: ZipFile, process: Entries.(zip: ZipFile, entry: ZipEntry) -> Unit = { zip, entry -> processEntry(zip, entry) }) = file.use { zip ->
zip.entries().asSequence().forEach { entry ->
process(zip, entry)
}
}

/** Validate [ZipEntry] and add information to [this] entries map */
private fun Entries.processEntry(zip: ZipFile, entry: ZipEntry): Unit = BufferedInputStream(zip.getInputStream(entry)).use {
processEntry(it, entry)
}

/** Validate [ZipEntry] and add information to [this] entries map */
private fun Entries.processEntry(inputStream: InputStream, entry: ZipEntry) {
validateEntry(entry)?.let {
val sourceFileBytes = inputStream.readBytes()
val resultLocation = getFinalPath(it, sourceFileBytes)
this += preCreateDirectories(Paths.get(resultLocation))
this[resultLocation] = sourceFileBytes
}
}

/** Return null if this [entry] shouldn't be processed */
private fun Entries.validateEntry(entry: ZipEntry): ZipEntry? = when {
entry.isDirectory -> {
// needed directories would be added by us
null
}
entry.name.contains("META-INF/maven") -> {
// pom.xml will be added by us
null
}
keys.contains(entry.name) -> {
// TODO: Investigate why I'm getting duplicates
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How serious is this? Can we remove it?

println("I have a duplicate entry: ${entry.name}")
null
// throw RuntimeException("duplicate entry in the JAR: ${entry.name}")
}
else -> entry
}

/** Write entries captured in [this] to [output] */
private fun Entries.writeEntries(output: ZipOutputStream) {
entries.sortedBy(Map.Entry<String, *>::key).forEach { (key, entry) ->
output.putNextEntry(ZipEntry(key))
output.write(entry)
}
}

/**
* For path "a/b/c.java" inserts "a/" and "a/b/ into `entries`
*/
private fun preCreateDirectories(path: Path): Map<String, ByteArray> {
val newEntries = HashMap<String, ByteArray>()
val newEntries = Entries()
for (i in path.nameCount-1 downTo 1) {
val subPath = path.subpath(0, i).toString() + "/"
newEntries[subPath] = ByteArray(0)
Expand Down
9 changes: 9 additions & 0 deletions maven/PomGenerator.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ class PomGenerator : Callable<Unit> {
@Option(names = ["--target_deps_coordinates"])
lateinit var dependencyCoordinates: String

@Option(names = ["--packaging"])
var packaging = ""

fun getLicenseInfo(license_id: String): Pair<String, String> {
return when {
license_id.equals("apache") -> {
Expand Down Expand Up @@ -224,6 +227,12 @@ class PomGenerator : Callable<Unit> {
versionElem.appendChild(pom.createTextNode(version))
rootElement.appendChild(versionElem)

if (packaging.isNotEmpty() && packaging != "jar") {
val packagingElem = pom.createElement("packaging")
packagingElem.appendChild(pom.createTextNode(packaging))
rootElement.appendChild(packagingElem)
}

// add dependency information
rootElement.appendChild(dependencies(pom, version, workspace_refs))

Expand Down
49 changes: 39 additions & 10 deletions maven/rules.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@
# under the License.
#

# Known generic labels to automatically not include in closure
_DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS = [
Label("@bazel_tools//tools/android:android_jar"),
]

def _is_android_library(target):
return AndroidLibraryAarInfo in target

def _parse_maven_coordinates(coordinates_string, enforce_version_template=True):
coordinates = coordinates_string.split(':')
Expand Down Expand Up @@ -47,13 +54,14 @@ def _generate_version_file(ctx):

def _generate_pom_file(ctx, version_file):
target = ctx.attr.target
maven_coordinates = _parse_maven_coordinates(target[JarInfo].name)
jar_info = target[JarInfo]
maven_coordinates = _parse_maven_coordinates(jar_info.name)
pom_file = ctx.actions.declare_file("{}_pom.xml".format(ctx.attr.name))

pom_deps = []
for pom_dependency in [dep for dep in target[JarInfo].deps.to_list() if dep.type == 'pom']:
for pom_dependency in [dep for dep in jar_info.deps.to_list() if dep.type == 'pom']:
pom_dependency = pom_dependency.maven_coordinates
if pom_dependency == target[JarInfo].name:
if pom_dependency == jar_info.name:
continue
pom_dependency_coordinates = _parse_maven_coordinates(pom_dependency, False)
pom_dependency_artifact = pom_dependency_coordinates.group_id + ":" + pom_dependency_coordinates.artifact_id
Expand All @@ -78,6 +86,7 @@ def _generate_pom_file(ctx, version_file):
"--version_file=" + version_file.path,
"--output_file=" + pom_file.path,
"--workspace_refs_file=" + ctx.file.workspace_refs.path,
"--packaging=" + jar_info.packaging
],
)

Expand All @@ -88,12 +97,14 @@ def _generate_class_jar(ctx, pom_file):
maven_coordinates = _parse_maven_coordinates(target[JarInfo].name)

jar = None
if hasattr(target, "files") and target.files.to_list() and target.files.to_list()[0].extension == "jar":
if (_is_android_library(target)):
jar = target[AndroidLibraryAarInfo].aar
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The information we need here is only the path to the JAR/AAR file.

Is there any way we could retrieve this in a way that is generic to both Java JARs and Android AARs, without requiring a check for _is_android_library? Since ideally we would like to keep this rule as generalised as possible.

elif hasattr(target, "files") and target.files.to_list() and target.files.to_list()[0].extension == "jar":
jar = target[JavaInfo].outputs.jars[0].class_jar
else:
fail("Could not find JAR file to deploy in {}".format(target))

output_jar = ctx.actions.declare_file("{}:{}.jar".format(maven_coordinates.group_id, maven_coordinates.artifact_id))
output_jar = ctx.actions.declare_file("{}:{}.{}".format(maven_coordinates.group_id, maven_coordinates.artifact_id, target[JarInfo].packaging))

class_jar_deps = [dep.class_jar for dep in target[JarInfo].deps.to_list() if dep.type == 'jar']
class_jar_paths = [jar.path] + [target.path for target in class_jar_deps]
Expand All @@ -119,7 +130,7 @@ def _generate_source_jar(ctx):

srcjar = None

if hasattr(target, "files") and target.files.to_list() and target.files.to_list()[0].extension == "jar":
if _is_android_library(target) or (hasattr(target, "files") and target.files.to_list() and target.files.to_list()[0].extension == "jar"):
for output in target[JavaInfo].outputs.jars:
if output.source_jar and (output.source_jar.basename.endswith("-src.jar") or output.source_jar.basename.endswith("-sources.jar")):
srcjar = output.source_jar
Expand Down Expand Up @@ -159,7 +170,7 @@ def _assemble_maven_impl(ctx):

return [
DefaultInfo(files = depset(output_files)),
MavenDeploymentInfo(jar = class_jar, pom = pom_file, srcjar = source_jar)
MavenDeploymentInfo(packaging = ctx.attr.target[JarInfo].packaging, jar = class_jar, pom = pom_file, srcjar = source_jar)
]

def find_maven_coordinates(target, tags):
Expand All @@ -176,6 +187,7 @@ JarInfo = provider(
fields = {
"name": "The name of a the JAR (Maven coordinates)",
"deps": "The list of dependencies of this JAR. A dependency may be of two types, POM or JAR.",
"packaging": "The type of target to publish (jar, war, aar, etc.)"
},
)

Expand All @@ -184,22 +196,33 @@ def _aggregate_dependency_info_impl(target, ctx):
deps = getattr(ctx.rule.attr, "deps", [])
runtime_deps = getattr(ctx.rule.attr, "runtime_deps", [])
exports = getattr(ctx.rule.attr, "exports", [])
neverlink = getattr(ctx.rule.attr, "neverlink", False)
deps_all = deps + exports + runtime_deps

maven_coordinates = find_maven_coordinates(target, tags)
dependencies = []
packaging = "aar" if _is_android_library(target) else "jar"

# depend via POM
if maven_coordinates:
dependencies = [struct(
target = target,
type = "pom",
maven_coordinates = maven_coordinates
)]
# Hacky way to ignore something we don't care about but not crash
elif neverlink or target.label in _DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS:
return JarInfo(
name = None,
deps = depset([]),
packaging = None,
)
# include runtime output jars
elif target[JavaInfo].runtime_output_jars:
elif JavaInfo in target:
jars = target[JavaInfo].runtime_output_jars
source_jars = target[JavaInfo].source_jars
dependencies = [struct(
target = target,
type = "jar",
class_jar = jar,
source_jar = source_jar,
Expand All @@ -209,7 +232,7 @@ def _aggregate_dependency_info_impl(target, ctx):
else:
fail("Unsure how to package dependency for target: %s" % target)

return JarInfo(
jar_info = JarInfo(
name = maven_coordinates,
deps = depset(dependencies, transitive = [
# Filter transitive JARs from dependency that has maven coordinates
Expand All @@ -218,8 +241,11 @@ def _aggregate_dependency_info_impl(target, ctx):
depset([dep for dep in target[JarInfo].deps.to_list() if dep.type == 'pom'])
if target[JarInfo].name else target[JarInfo].deps for target in deps_all
]),
packaging = packaging,
)

return jar_info

aggregate_dependency_info = aspect(
attr_aspects = [
"jars",
Expand Down Expand Up @@ -301,6 +327,7 @@ assemble_maven = rule(

MavenDeploymentInfo = provider(
fields = {
'packaging': 'The type of target to publish (jar, war, aar, etc.)',
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Given that we are deploying either a JAR or an AAR file, I think it would be more intuitive if we simply read the extension of the file being passed into the deploy_maven rule (instead of relying on a new field in MavenDeploymentInfo) to determine the packaging used in deploy.py.

'jar': 'JAR file to deploy',
'srcjar': 'JAR file with sources',
'pom': 'Accompanying pom.xml file'
Expand All @@ -314,6 +341,7 @@ def _deploy_maven_impl(ctx):
lib_jar_link = "lib.jar"
src_jar_link = "lib.srcjar"
pom_xml_link = ctx.attr.target[MavenDeploymentInfo].pom.basename
packaging = ctx.attr.target[MavenDeploymentInfo].packaging

ctx.actions.expand_template(
template = ctx.file._deployment_script,
Expand All @@ -323,7 +351,8 @@ def _deploy_maven_impl(ctx):
"$SRCJAR_PATH": src_jar_link,
"$POM_PATH": pom_xml_link,
"{snapshot}": ctx.attr.snapshot,
"{release}": ctx.attr.release
"{release}": ctx.attr.release,
"$PACKAGING": packaging,
}
)

Expand Down
Loading