diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/efficientBinaryFormat/ByteReadingBuffer.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/efficientBinaryFormat/ByteReadingBuffer.kt new file mode 100644 index 000000000..4c31719f0 --- /dev/null +++ b/formats/json-tests/commonTest/src/kotlinx/serialization/efficientBinaryFormat/ByteReadingBuffer.kt @@ -0,0 +1,80 @@ +/* + * Copyright 2017-2025 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.serialization.efficientBinaryFormat + +class ByteReadingBuffer(val buffer: ByteArray) { + private var next = 0 + + private fun nextByte(): Int { + return buffer[next++].toInt() and 0xff + } + + private fun nextByteL(): Long { + return buffer[next++].toLong() and 0xffL + } + + operator fun get(pos: Int): Byte { + if(pos !in 0.. Unit) { + val len = readInt() + var remaining = len + while (remaining > 1024) { + remaining -= 1024 + val chunk = CharArray(1024) { readChar() } + consumeChunk(chunk.concatToString()) + } + val chars = CharArray(remaining) { readChar() } + consumeChunk(chars.concatToString()) + } + +} \ No newline at end of file diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/efficientBinaryFormat/ByteWritingBuffer.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/efficientBinaryFormat/ByteWritingBuffer.kt new file mode 100644 index 000000000..a6142403a --- /dev/null +++ b/formats/json-tests/commonTest/src/kotlinx/serialization/efficientBinaryFormat/ByteWritingBuffer.kt @@ -0,0 +1,89 @@ +/* + * Copyright 2017-2025 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.serialization.efficientBinaryFormat + +import kotlin.experimental.and + +class ByteWritingBuffer() { + private var buffer = ByteArray(8192) + private var next = 0 + val size + get() = next + + operator fun get(pos: Int): Byte { + if(pos !in 0.. encodeToByteArray( + serializer: SerializationStrategy, + value: T + ): ByteArray { + val encoder = Encoder(serializersModule) + serializer.serialize(encoder, value) + return encoder.byteBuffer.toByteArray() + } + + override fun decodeFromByteArray( + deserializer: DeserializationStrategy, + bytes: ByteArray + ): T { + val decoder = Decoder(serializersModule, bytes) + return deserializer.deserialize(decoder) + } + + class Encoder( + override val serializersModule: SerializersModule, + internal val byteBuffer: ByteWritingBuffer = ByteWritingBuffer(), + elementsCount: Int = -1 + ): AbstractEncoder() { + var lastWrittenIndex = -1 + var currentIndex = -1 + val notInStruct = elementsCount < 0 + + val pending : Array<(() -> Unit)?> = when { + elementsCount <=0 -> emptyArray() + else -> arrayOfNulls(elementsCount) + } + + override fun encodeBoolean(value: Boolean) = writeOrSuspend { byteBuffer.writeByte(if (value) 1 else 0) } + override fun encodeByte(value: Byte) = writeOrSuspend { byteBuffer.writeByte(value) } + override fun encodeShort(value: Short) = writeOrSuspend { byteBuffer.writeShort(value) } + override fun encodeInt(value: Int) = writeOrSuspend { byteBuffer.writeInt(value) } + override fun encodeLong(value: Long) = writeOrSuspend { byteBuffer.writeLong(value) } + override fun encodeFloat(value: Float) = writeOrSuspend { byteBuffer.writeFloat(value) } + override fun encodeDouble(value: Double) = writeOrSuspend { byteBuffer.writeDouble(value) } + override fun encodeChar(value: Char) = writeOrSuspend { byteBuffer.writeChar(value) } + override fun encodeString(value: String) = writeOrSuspend { byteBuffer.writeString(value) } + override fun encodeEnum(enumDescriptor: SerialDescriptor, index: Int) = writeOrSuspend { + byteBuffer.writeInt(index) + } + + @ExperimentalSerializationApi + override fun encodeNullableSerializableValue( + serializer: SerializationStrategy, + value: T? + ) { + writeOrSuspend { + super.encodeNullableSerializableValue(serializer, value) + } + } + + override fun encodeSerializableValue(serializer: SerializationStrategy, value: T) { + writeOrSuspend { + super.encodeSerializableValue(serializer, value) + } + } + + @Suppress("NOTHING_TO_INLINE") + private inline fun writeOrSuspend(noinline action: () -> Unit) { + val c = currentIndex + currentIndex = -1 + when { + notInStruct || c<0 -> action() + lastWrittenIndex < -1 -> pending[c] = action + lastWrittenIndex + 1 == c -> { + ++lastWrittenIndex + action() + } + c < pending.size -> pending[c] = action + else -> error("Unexpected index") + } + } + + override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean { + currentIndex = index + return true + } + + @ExperimentalSerializationApi + override fun shouldEncodeElementDefault(descriptor: SerialDescriptor, index: Int): Boolean = true + + override fun beginStructure(descriptor: SerialDescriptor): CompositeEncoder { + return Encoder(serializersModule, byteBuffer, descriptor.elementsCount) + } + + override fun beginCollection(descriptor: SerialDescriptor, collectionSize: Int): CompositeEncoder { + encodeInt(collectionSize) + return Encoder(serializersModule, byteBuffer, -1) + } + + override fun endStructure(descriptor: SerialDescriptor) { + currentIndex = -2 // mark negative to ensure writing + for (i in 0 until pending.size) { + pending[i]?.invoke() + } + } + + override fun encodeNull() = encodeBoolean(false) + override fun encodeNotNullMark() = encodeBoolean(true) + + } + + class Decoder(override val serializersModule: SerializersModule, private val reader: ByteReadingBuffer) : AbstractDecoder(), ChunkedDecoder { + + constructor(serializersModule: SerializersModule, bytes: ByteArray) : this( + serializersModule, + ByteReadingBuffer(bytes) + ) + + private var nextElementIndex = 0 +// private var currentDesc: SerialDescriptor? = null + + override fun decodeBoolean(): Boolean = reader.readByte().toInt() != 0 + + override fun decodeByte(): Byte = reader.readByte() + + override fun decodeShort(): Short = reader.readShort() + + override fun decodeInt(): Int = reader.readInt() + + override fun decodeLong(): Long = reader.readLong() + + override fun decodeFloat(): Float = reader.readFloat() + + override fun decodeDouble(): Double = reader.readDouble() + + override fun decodeChar(): Char = reader.readChar() + + override fun decodeString(): String = reader.readString() + + @ExperimentalSerializationApi + override fun decodeStringChunked(consumeChunk: (String) -> Unit) { + reader.readString(consumeChunk) + } + + override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = reader.readInt() + + override fun decodeNotNullMark(): Boolean = decodeBoolean() + + @ExperimentalSerializationApi + override fun decodeSequentially(): Boolean = true + + override fun decodeCollectionSize(descriptor: SerialDescriptor): Int = reader.readInt() + + override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder { + return Decoder(serializersModule, reader) + } + + override fun endStructure(descriptor: SerialDescriptor) { + check(nextElementIndex ==0 || descriptor.elementsCount == nextElementIndex) { "Type: ${descriptor.serialName} not fully read: ${descriptor.elementsCount} != $nextElementIndex" } + } + + override fun decodeElementIndex(descriptor: SerialDescriptor): Int { + return when (nextElementIndex) { + descriptor.elementsCount -> CompositeDecoder.DECODE_DONE + else -> nextElementIndex++ + } + } + } +} \ No newline at end of file diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonCustomSerializersTest.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonCustomSerializersTest.kt index 351866df9..35f0c320b 100644 --- a/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonCustomSerializersTest.kt +++ b/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonCustomSerializersTest.kt @@ -41,7 +41,7 @@ class JsonCustomSerializersTest : JsonTestBase() { override fun serialize(encoder: Encoder, value: C) { val elemOutput = encoder.beginStructure(descriptor) elemOutput.encodeIntElement(descriptor, 1, value.b) - if (value.a != 31) elemOutput.encodeIntElement(descriptor, 0, value.a) + if (value.a != 31 || elemOutput.shouldEncodeElementDefault(descriptor, 0)) elemOutput.encodeIntElement(descriptor, 0, value.a) elemOutput.endStructure(descriptor) } } @@ -57,7 +57,9 @@ class JsonCustomSerializersTest : JsonTestBase() { override fun serialize(encoder: Encoder, value: CList2) { val elemOutput = encoder.beginStructure(descriptor) elemOutput.encodeSerializableElement(descriptor, 1, ListSerializer(C), value.c) - if (value.d != 5) elemOutput.encodeIntElement(descriptor, 0, value.d) + if (value.d != 5 || elemOutput.shouldEncodeElementDefault(descriptor, 0)) { + elemOutput.encodeIntElement(descriptor, 0, value.d) + } elemOutput.endStructure(descriptor) } } @@ -69,7 +71,9 @@ class JsonCustomSerializersTest : JsonTestBase() { companion object : KSerializer { override fun serialize(encoder: Encoder, value: CList3) { val elemOutput = encoder.beginStructure(descriptor) - if (value.e.isNotEmpty()) elemOutput.encodeSerializableElement(descriptor, 0, ListSerializer(C), value.e) + if (value.e.isNotEmpty() || elemOutput.shouldEncodeElementDefault(descriptor, 0)) { + elemOutput.encodeSerializableElement(descriptor, 0, ListSerializer(C), value.e) + } elemOutput.encodeIntElement(descriptor, 1, value.f) elemOutput.endStructure(descriptor) } @@ -83,7 +87,9 @@ class JsonCustomSerializersTest : JsonTestBase() { override fun serialize(encoder: Encoder, value: CList4) { val elemOutput = encoder.beginStructure(descriptor) elemOutput.encodeIntElement(descriptor, 1, value.h) - if (value.g.isNotEmpty()) elemOutput.encodeSerializableElement(descriptor, 0, ListSerializer(C), value.g) + if (value.g.isNotEmpty() || elemOutput.shouldEncodeElementDefault(descriptor, 0)) { + elemOutput.encodeSerializableElement(descriptor, 0, ListSerializer(C), value.g) + } elemOutput.endStructure(descriptor) } } @@ -96,10 +102,12 @@ class JsonCustomSerializersTest : JsonTestBase() { override fun serialize(encoder: Encoder, value: CList5) { val elemOutput = encoder.beginStructure(descriptor) elemOutput.encodeIntElement(descriptor, 1, value.h) - if (value.g.isNotEmpty()) elemOutput.encodeSerializableElement( - descriptor, 0, ListSerializer(Int.serializer()), - value.g - ) + if (value.g.isNotEmpty() || elemOutput.shouldEncodeElementDefault(descriptor, 0)) { + elemOutput.encodeSerializableElement( + descriptor, 0, ListSerializer(Int.serializer()), + value.g + ) + } elemOutput.endStructure(descriptor) } } diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonErrorMessagesTest.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonErrorMessagesTest.kt index da73bc3a2..1042bbf9e 100644 --- a/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonErrorMessagesTest.kt +++ b/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonErrorMessagesTest.kt @@ -131,7 +131,7 @@ class JsonErrorMessagesTest : JsonTestBase() { // it can be some kind of path `{"foo:bar:baz":"my:resource:locator:{123}"}` or even URI used as a string key/value. // So if the closing quote is missing, there's really no way to correctly tell where the key or value is supposed to end. // Although we may try to unify these messages for consistency. - if (mode in setOf(JsonTestingMode.STREAMING, JsonTestingMode.TREE)) + if (mode in setOf(JsonTestingMode.STREAMING, JsonTestingMode.TREE, JsonTestingMode.EFFICIENT_BINARY)) assertContains( message, "Unexpected JSON token at offset 7: Expected quotation mark '\"', but had ':' instead at path: \$" @@ -145,7 +145,7 @@ class JsonErrorMessagesTest : JsonTestBase() { checkSerializationException({ default.decodeFromString(serString, input2, mode) }, { message -> - if (mode in setOf(JsonTestingMode.STREAMING, JsonTestingMode.TREE)) + if (mode in setOf(JsonTestingMode.STREAMING, JsonTestingMode.TREE, JsonTestingMode.EFFICIENT_BINARY)) assertContains( message, "Unexpected JSON token at offset 13: Expected quotation mark '\"', but had '}' instead at path: \$" diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt index de8cfb38b..d7a264b0b 100644 --- a/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt +++ b/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt @@ -6,6 +6,7 @@ package kotlinx.serialization.json import kotlinx.io.* import kotlinx.serialization.* +import kotlinx.serialization.efficientBinaryFormat.EfficientBinaryFormat import kotlinx.serialization.json.internal.* import kotlinx.serialization.json.io.* import kotlinx.serialization.json.okio.decodeFromBufferedSource @@ -25,7 +26,8 @@ enum class JsonTestingMode { TREE, OKIO_STREAMS, JAVA_STREAMS, - KXIO_STREAMS; + KXIO_STREAMS, + EFFICIENT_BINARY; companion object { fun value(i: Int) = values()[i] @@ -41,6 +43,7 @@ abstract class JsonTestBase { return encodeToString(serializer, value, jsonTestingMode) } + @OptIn(ExperimentalStdlibApi::class) internal fun Json.encodeToString( serializer: SerializationStrategy, value: T, @@ -67,6 +70,18 @@ abstract class JsonTestBase { encodeToSink(serializer, value, buffer) buffer.readString() } + JsonTestingMode.EFFICIENT_BINARY -> { + val ebf = EfficientBinaryFormat() + val bytes = runCatching { ebf.encodeToByteArray(serializer, value) }.getOrElse { e-> + null//throw e + } + if (bytes != null && serializer is KSerializer<*>) { + val decoded = ebf.decodeFromByteArray((serializer as KSerializer), bytes) + encodeToString(serializer, decoded) + } else { + encodeToString(serializer, value) + } + } } internal inline fun Json.decodeFromString(source: String, jsonTestingMode: JsonTestingMode): T { @@ -74,6 +89,7 @@ abstract class JsonTestBase { return decodeFromString(deserializer, source, jsonTestingMode) } + @OptIn(ExperimentalStdlibApi::class) internal fun Json.decodeFromString( deserializer: DeserializationStrategy, source: String, @@ -100,6 +116,20 @@ abstract class JsonTestBase { buffer.writeString(source) decodeFromSource(deserializer, buffer) } + JsonTestingMode.EFFICIENT_BINARY -> { + when (deserializer){ + is KSerializer<*> -> { + val s = deserializer as KSerializer + val value = decodeFromString(deserializer, source) + runCatching { + val ebf = EfficientBinaryFormat() + val binaryValue = EfficientBinaryFormat().encodeToByteArray(s, value) + ebf.decodeFromByteArray(s, binaryValue) + }.getOrElse { value } + } + else -> decodeFromString(deserializer, source) + } + } } protected open fun parametrizedTest(test: (JsonTestingMode) -> Unit) { @@ -108,6 +138,8 @@ abstract class JsonTestBase { add(runCatching { test(JsonTestingMode.TREE) }) add(runCatching { test(JsonTestingMode.OKIO_STREAMS) }) add(runCatching { test(JsonTestingMode.KXIO_STREAMS) }) + add(runCatching { test(JsonTestingMode.EFFICIENT_BINARY) } + .recover { e -> if ("Json format" in (e.message ?: "")) Unit else throw e }) if (isJvm()) { add(runCatching { test(JsonTestingMode.JAVA_STREAMS) }) @@ -129,12 +161,70 @@ abstract class JsonTestBase { } } + /** A test runner that effectively handles the json tests to also test serialization to + * "efficient" binary. This mainly checks serializer implementations. + */ + private inner class EfficientBinary( + val json: Json, + val ebf: EfficientBinaryFormat = EfficientBinaryFormat(), + ) : StringFormat { + override val serializersModule: SerializersModule = ebf.serializersModule + + private var bytes: ByteArray? = null + private var jsonStr: String? = null + + @OptIn(ExperimentalStdlibApi::class) + override fun encodeToString(serializer: SerializationStrategy, value: T): String { + bytes = runCatching { ebf.encodeToByteArray(serializer, value) } + .onFailure { if ("Json format" !in it.message!!) { + json.encodeToString(serializer, value) // trigger throwing the json exception if the exception is there + throw it + } } + .getOrNull() + return json.encodeToString(serializer, value).also { + if (bytes != null) jsonStr = it + } + } + + @OptIn(ExperimentalStdlibApi::class) + override fun decodeFromString(deserializer: DeserializationStrategy, string: String): T { + /* + * to retain compatibility with json we support different cases. If + * the string has been encoded already use that. Instead, if the + * deserializer is also a serializer (the default) then use that to + * get the value from json and encode that to bytes which are then + * decoded. In this case capture and ignore cases that require a + * json encoder. + * + * Finally fall back to json decoding (nothing can be done) + */ + + var bytes = this@EfficientBinary.bytes + if (string == jsonStr && bytes != null) { + return ebf.decodeFromByteArray(deserializer, bytes) + } else if (deserializer is SerializationStrategy<*>) { + val value = json.decodeFromString(deserializer, string) + // + @Suppress("UNCHECKED_CAST") + runCatching { ebf.encodeToByteArray(deserializer as SerializationStrategy, value) }.onSuccess { r -> + bytes = r + jsonStr = string + return ebf.decodeFromByteArray(deserializer, bytes) + }.onFailure { e -> + if ("Json format" !in e.message!!) throw e + } + } + return json.decodeFromString(deserializer, string) + } + } + protected fun parametrizedTest(json: Json, test: StringFormat.() -> Unit) { val streamingResult = runCatching { SwitchableJson(json, JsonTestingMode.STREAMING).test() } val treeResult = runCatching { SwitchableJson(json, JsonTestingMode.TREE).test() } val okioResult = runCatching { SwitchableJson(json, JsonTestingMode.OKIO_STREAMS).test() } val kxioResult = runCatching { SwitchableJson(json, JsonTestingMode.KXIO_STREAMS).test() } - processResults(listOf(streamingResult, treeResult, okioResult, kxioResult)) + val efficientBinaryResult = runCatching { EfficientBinary(json).test() } + processResults(listOf(streamingResult, treeResult, okioResult, kxioResult, efficientBinaryResult)) } protected fun processResults(results: List>) {