diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..a7d99dc
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,9 @@
+# maven
+**/target
+
+# intellij
+.idea/**
+
+# intellij project files
+*.iml
+**/*.iml
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..261eeb9
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/compressedint/pom.xml b/compressedint/pom.xml
new file mode 100644
index 0000000..7d72d65
--- /dev/null
+++ b/compressedint/pom.xml
@@ -0,0 +1,52 @@
+
+
+
+
+
+ zchunk-parent
+ de.bmarwell.zchunk
+ 1.0.0-SNAPSHOT
+
+ 4.0.0
+
+ compressedint
+
+
+
+
+ org.immutables
+ value
+ provided
+
+
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ test
+
+
+
+
+
diff --git a/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/AbstractCompressedInt.java b/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/AbstractCompressedInt.java
new file mode 100644
index 0000000..8ee5581
--- /dev/null
+++ b/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/AbstractCompressedInt.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.compressedint;
+
+import java.math.BigInteger;
+import java.util.StringJoiner;
+import org.immutables.value.Value;
+
+@Value.Immutable
+public abstract class AbstractCompressedInt implements CompressedInt {
+
+ @Override
+ @Value.Lazy
+ public BigInteger getValue() {
+ return CompressedIntUtil.decompress(getCompressedBytes());
+ }
+
+ @Override
+ @Value.Lazy
+ public long getLongValue() {
+ return getValue().longValueExact();
+ }
+
+ @Override
+ @Value.Lazy
+ public long getUnsignedLongValue() {
+ return getValue().longValue();
+ }
+
+ @Override
+ @Value.Lazy
+ public int getIntValue() {
+ return getValue().intValueExact();
+ }
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", CompressedInt.class.getSimpleName() + "[", "]")
+ .add("compressedBytes=" + new BigInteger(1, getCompressedBytes()).toString(16))
+ .add("unsignedValue=" + getValue().toString())
+ .toString();
+ }
+}
diff --git a/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/CompressedInt.java b/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/CompressedInt.java
new file mode 100644
index 0000000..eaa8056
--- /dev/null
+++ b/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/CompressedInt.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.compressedint;
+
+import java.math.BigInteger;
+import org.immutables.value.Value;
+
+public interface CompressedInt {
+
+ byte[] getCompressedBytes();
+
+ /**
+ * If you have few experience with unsigned values in java, consider using this value instead.
+ *
+ * @return a biginteger which will always output a positive value.
+ */
+ @Value.Lazy
+ BigInteger getValue();
+
+ @Value.Lazy
+ long getLongValue();
+
+ @Value.Lazy
+ long getUnsignedLongValue();
+
+ @Value.Lazy
+ int getIntValue();
+}
diff --git a/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/CompressedIntFactory.java b/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/CompressedIntFactory.java
new file mode 100644
index 0000000..bf9af56
--- /dev/null
+++ b/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/CompressedIntFactory.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.compressedint;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public final class CompressedIntFactory {
+
+ private CompressedIntFactory() {
+ // util class
+ }
+
+ public static CompressedInt fromCompressedBytes(final byte[] input) {
+ if (input.length > CompressedIntUtil.MAX_COMPRESSED_INT_LENGTH) {
+ throw new IllegalArgumentException(
+ "Input length [" + input.length + "] bytes is too large! Max allowed: " + CompressedIntUtil.MAX_COMPRESSED_INT_LENGTH);
+ }
+
+ return ImmutableCompressedInt.builder()
+ .compressedBytes(input)
+ .build();
+ }
+
+ /**
+ * Convert an unsigned long to a compressed int.
+ *
+ *
Hint: If your long yields -1L, it's instead {@code 0xffffffffffffffff}. But java treats it as
+ * signed long.
+ *
+ * @param unsignedLongValue
+ * a long value which gets interpreted as unsigned.
+ * @return a compressedInt.
+ */
+ public static CompressedInt valueOf(final long unsignedLongValue) {
+ final byte[] unsignedBytes = CompressedIntUtil.compress(unsignedLongValue);
+
+ return fromCompressedBytes(unsignedBytes);
+ }
+
+ public static CompressedInt readCompressedInt(final InputStream inputStream) throws IOException {
+ int currentByte;
+ int byteCounter = 0;
+ final byte[] buffer = new byte[CompressedIntUtil.MAX_COMPRESSED_INT_LENGTH];
+
+ while ((currentByte = inputStream.read()) != -1) {
+ if (byteCounter >= CompressedIntUtil.MAX_COMPRESSED_INT_LENGTH) {
+ throw new IllegalArgumentException("CompressedInt too large. Giving up after reading [" + byteCounter + "] bytes.");
+ }
+
+ buffer[byteCounter] = (byte) currentByte;
+
+ if ((currentByte & CompressedIntUtil.COMPRESSED_INT_LAST_BYTE_FLAG) == CompressedIntUtil.COMPRESSED_INT_LAST_BYTE_FLAG) {
+ break;
+ }
+
+ byteCounter++;
+ }
+
+ final byte[] read = new byte[byteCounter + 1];
+ System.arraycopy(buffer, 0, read, 0, byteCounter + 1);
+
+ return fromCompressedBytes(read);
+ }
+}
diff --git a/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/CompressedIntUtil.java b/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/CompressedIntUtil.java
new file mode 100644
index 0000000..3f53340
--- /dev/null
+++ b/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/CompressedIntUtil.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.compressedint;
+
+import static java.math.BigInteger.ONE;
+
+import java.math.BigInteger;
+
+public final class CompressedIntUtil {
+
+ public static final int COMPRESSED_INT_LAST_BYTE_FLAG = 0b10000000;
+
+ /**
+ * The maximum value is 0xffffffffffffffff (which equals signed -1).
+ */
+ public static final long MAX_VALUE = 0xffffffffffffffffL;
+
+ /**
+ * This bitmask is used to read in an unsigned long as positive value.
+ */
+ public static final BigInteger UNSIGNED_LONG_MASK = ONE.shiftLeft(Long.SIZE).subtract(ONE);
+
+ /**
+ * Since an (unsigned) long (64 bits / 8 byte) is the maximum length we allow at this point,
+ * the max length of a compressed integer is 10 bytes.
+ * 10 bytes are needed to encode -1L (or 0xffffffffffffffff) as compressed int.
+ */
+ public static final int MAX_COMPRESSED_INT_LENGTH = (Long.SIZE / 7) + 1;
+
+ private CompressedIntUtil() {
+ // private util
+ }
+
+
+ public static byte[] compress(final long unsignedIntValue) {
+ long modValue = unsignedIntValue;
+ final byte[] tmp = new byte[MAX_COMPRESSED_INT_LENGTH];
+ int byteIndex = 0;
+
+ while (true) {
+ // get the rightmost 7 bits
+ final byte currentByte = (byte) (modValue & 0b01111111);
+ // unsigned(!) shift by seven bits.
+ modValue >>>= 7L;
+ tmp[byteIndex] = currentByte;
+
+ if (modValue == 0L) {
+ // this is the last byte we encoded. Make sure we set the last-byte-flag.
+ tmp[byteIndex] |= COMPRESSED_INT_LAST_BYTE_FLAG;
+ break;
+ }
+
+ byteIndex++;
+ }
+
+ final byte[] out = new byte[byteIndex + 1];
+ System.arraycopy(tmp, 0, out, 0, byteIndex + 1);
+
+ return out;
+ }
+
+ public static BigInteger decompress(final byte[] compressedUnsignedInt) {
+ if (compressedUnsignedInt.length > MAX_COMPRESSED_INT_LENGTH) {
+ throw new IllegalArgumentException("Compressed int too big!");
+ }
+
+ long result = 0L;
+ int shift = 0;
+
+ for (final byte b : compressedUnsignedInt) {
+ final byte leadingZero = (byte) (b & ~COMPRESSED_INT_LAST_BYTE_FLAG);
+
+ result |= leadingZero << shift;
+ shift += 7;
+ }
+
+ return BigInteger.valueOf(result).and(UNSIGNED_LONG_MASK);
+ }
+
+}
diff --git a/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/package-info.java b/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/package-info.java
new file mode 100644
index 0000000..3c93bb2
--- /dev/null
+++ b/compressedint/src/main/java/de/bmarwell/zchunk/compressedint/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@Value.Style(stagedBuilder = true, jdkOnly = true)
+package de.bmarwell.zchunk.compressedint;
+
+import org.immutables.value.Value;
diff --git a/compressedint/src/test/java/de/bmarwell/zchunk/compressedint/CompressedIntTest.java b/compressedint/src/test/java/de/bmarwell/zchunk/compressedint/CompressedIntTest.java
new file mode 100644
index 0000000..73e04bf
--- /dev/null
+++ b/compressedint/src/test/java/de/bmarwell/zchunk/compressedint/CompressedIntTest.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.compressedint;
+
+import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.logging.Logger;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class CompressedIntTest {
+
+ private static final Logger LOG = Logger.getLogger(CompressedIntTest.class.getCanonicalName());
+
+ private static final byte[] CP_394 = new byte[]{(byte) 0b00001010, (byte) 0b10000011};
+
+ public static String byteArrayToBinaryString(final byte[] input) {
+ return new BigInteger(1, input).toString(2);
+ }
+
+ @Test
+ public void testToCompressedInt() {
+ final CompressedInt ci = CompressedIntFactory.valueOf(394L);
+
+ final String binaryString = byteArrayToBinaryString(ci.getCompressedBytes());
+
+ Assertions.assertEquals("101010000011", binaryString);
+ }
+
+ @Test
+ public void testToCompressedInt_max() {
+ final CompressedInt bytes = CompressedIntFactory.valueOf(Integer.MAX_VALUE);
+
+ final String binaryString = byteArrayToBinaryString(bytes.getCompressedBytes());
+
+ Assertions.assertEquals("111111101111111011111110111111110000111", binaryString);
+ }
+
+ @Test
+ public void testToCompressedInt_ulong_max() {
+ final byte[] ulongAsBytes = new byte[8];
+ Arrays.fill(ulongAsBytes, (byte) 0xff);
+ final long unsignedLongValue = new BigInteger(1, ulongAsBytes).longValue();
+
+ final CompressedInt bytes = CompressedIntFactory.valueOf(unsignedLongValue);
+
+ final String binaryString = byteArrayToBinaryString(bytes.getCompressedBytes());
+
+ Assertions.assertEquals("1111111011111110111111101111111011111110111111101111111011111110111111110000001", binaryString);
+ }
+
+ @Test
+ public void testReadCompressedInt() {
+ final byte[] maxLong = new byte[8];
+ Arrays.fill(maxLong, (byte) 0xff);
+ final CompressedInt compressedInt = CompressedIntFactory.fromCompressedBytes(maxLong);
+
+ Assertions.assertAll(
+ () -> Assertions.assertEquals("18446744073709551615", compressedInt.getValue().toString())
+ );
+ }
+
+ @Test
+ public void testReadCompressedInt_maxValue() {
+ // 1111111 01111111 01111111 01111111 01111111 01111111 01111111 011111110 11111111 0000001
+ // repeated max value on purpose: guard against changes.
+ final byte[] bytes = new byte[]{
+ 0b1111111, 0b01111111, 0b01111111, 0b01111111, 0b01111111, 0b01111111, 0b01111111, (byte) 0b011111110, (byte) 0b11111111, 0b0000001
+ };
+
+ final CompressedInt compressedInt = CompressedIntFactory.fromCompressedBytes(bytes);
+
+ Assertions.assertAll(
+ () -> Assertions.assertEquals("18446744073709551615", compressedInt.getValue().toString())
+ );
+ }
+
+ @Test
+ public void testCompressedIntMaxLength() {
+ Assertions.assertAll(
+ () -> Assertions.assertEquals(10, CompressedIntUtil.MAX_COMPRESSED_INT_LENGTH),
+ () -> Assertions.assertThrows(IllegalArgumentException.class, () -> CompressedIntFactory.fromCompressedBytes(new byte[11]))
+ );
+ }
+
+ @Test
+ public void testToCompressedInt_unsigned_394() {
+ final CompressedInt ci = CompressedIntFactory.valueOf(394);
+
+ Assertions.assertAll(
+ () -> Assertions.assertEquals(2, ci.getCompressedBytes().length),
+ () -> Assertions.assertArrayEquals(CP_394, ci.getCompressedBytes())
+ );
+ }
+
+ @Test
+ public void testFromBytes_6582() {
+ final CompressedInt compressedInt = CompressedIntFactory.fromCompressedBytes(new byte[]{0x65, (byte) 0x82});
+
+ Assertions.assertAll(
+ () -> Assertions.assertEquals(2L, compressedInt.getCompressedBytes().length),
+ () -> Assertions.assertEquals(357L, compressedInt.getLongValue())
+ );
+ }
+
+ @Test
+ public void testToUnsignedInt_zero() {
+ final byte[] input = new byte[]{(byte) 0b10000000};
+ final CompressedInt anInt = CompressedIntFactory.fromCompressedBytes(input);
+ final long unsignedLong = anInt.getIntValue();
+
+ Assertions.assertEquals(0L, unsignedLong);
+ }
+
+ @Test
+ public void testToUnsignedInt_one() {
+ final byte[] input = new byte[]{(byte) 0b10000001};
+ final CompressedInt anInt = CompressedIntFactory.fromCompressedBytes(input);
+ final long unsignedLong = anInt.getIntValue();
+
+ Assertions.assertEquals(1L, unsignedLong);
+ }
+
+ @Test
+ public void testToUnsignedInt() {
+ final CompressedInt anInt = CompressedIntFactory.fromCompressedBytes(CP_394);
+ final long unsignedLong = anInt.getLongValue();
+
+ Assertions.assertEquals(394L, unsignedLong);
+ }
+
+ @Test
+ public void testExceptionOnBigLongToInt() {
+ final CompressedInt compressedInt = CompressedIntFactory.valueOf(0xffffffff00ffffL);
+
+ Assertions.assertThrows(ArithmeticException.class, compressedInt::getIntValue);
+ }
+
+ @Test
+ public void testExceptionOnBigLongToLong() {
+ final CompressedInt compressedInt = CompressedIntFactory.valueOf(0xffffffff00ffffL);
+
+ Assertions.assertThrows(ArithmeticException.class, compressedInt::getLongValue);
+ }
+
+ @Test
+ public void testExceptionOnBigLongToUnsignedLong() {
+ final CompressedInt compressedInt = CompressedIntFactory.valueOf(-1L);
+
+ Assertions.assertEquals(-1L, compressedInt.getUnsignedLongValue());
+ }
+
+ @Test
+ public void testExceptionOnBigLongToUnsignedLong_minus2() {
+ final CompressedInt compressedInt = CompressedIntFactory.valueOf(-2L);
+
+ Assertions.assertEquals(-2L, compressedInt.getUnsignedLongValue());
+ }
+}
diff --git a/compression/compression-api/pom.xml b/compression/compression-api/pom.xml
new file mode 100644
index 0000000..e63e3ec
--- /dev/null
+++ b/compression/compression-api/pom.xml
@@ -0,0 +1,65 @@
+
+
+
+
+ 4.0.0
+
+
+ de.bmarwell.zchunk
+ zchunk-parent
+ 1.0.0-SNAPSHOT
+ ../..
+
+
+ compression-api
+
+
+
+ de.bmarwell.zchunk
+ compressedint
+ 1.0.0-SNAPSHOT
+
+
+
+
+ org.immutables
+ value
+ provided
+
+
+
+ org.checkerframework
+ checker-qual
+ provided
+
+
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ test
+
+
+
+
diff --git a/compression/compression-api/src/main/java/de/bmarwell/zchunk/compression/algo/unknown/UnknownAlgorithm.java b/compression/compression-api/src/main/java/de/bmarwell/zchunk/compression/algo/unknown/UnknownAlgorithm.java
new file mode 100644
index 0000000..2addfd0
--- /dev/null
+++ b/compression/compression-api/src/main/java/de/bmarwell/zchunk/compression/algo/unknown/UnknownAlgorithm.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.compression.algo.unknown;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import de.bmarwell.zchunk.compressedint.CompressedIntFactory;
+import de.bmarwell.zchunk.compression.api.CompressionAlgorithm;
+import java.io.InputStream;
+import java.util.function.Function;
+
+public class UnknownAlgorithm implements CompressionAlgorithm {
+
+ @Override
+ public CompressedInt getCompressionTypeValue() {
+ return CompressedIntFactory.valueOf(-1L);
+ }
+
+ @Override
+ public String getName() {
+ return "unknown";
+ }
+
+ @Override
+ public Function getOutputStreamSupplier() {
+ throw new UnsupportedOperationException("not implemented");
+ }
+}
diff --git a/compression/compression-api/src/main/java/de/bmarwell/zchunk/compression/api/CompressionAlgorithm.java b/compression/compression-api/src/main/java/de/bmarwell/zchunk/compression/api/CompressionAlgorithm.java
new file mode 100644
index 0000000..2fa042d
--- /dev/null
+++ b/compression/compression-api/src/main/java/de/bmarwell/zchunk/compression/api/CompressionAlgorithm.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.compression.api;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import java.io.InputStream;
+import java.util.function.Function;
+import org.immutables.value.Value;
+
+@Value.Immutable
+public interface CompressionAlgorithm {
+
+ CompressedInt getCompressionTypeValue();
+
+ String getName();
+
+ Function getOutputStreamSupplier();
+}
diff --git a/compression/compression-api/src/main/java/de/bmarwell/zchunk/compression/api/CompressionAlgorithmFactory.java b/compression/compression-api/src/main/java/de/bmarwell/zchunk/compression/api/CompressionAlgorithmFactory.java
new file mode 100644
index 0000000..b422d7a
--- /dev/null
+++ b/compression/compression-api/src/main/java/de/bmarwell/zchunk/compression/api/CompressionAlgorithmFactory.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.compression.api;
+
+import static java.util.stream.Collectors.toMap;
+
+import de.bmarwell.zchunk.compression.algo.unknown.UnknownAlgorithm;
+import de.bmarwell.zchunk.compression.api.internal.ReflectionUtil;
+import java.util.AbstractMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+public final class CompressionAlgorithmFactory {
+
+ private static final Package THIS_PACKAGE = CompressionAlgorithmFactory.class.getPackage();
+ private static final String ROOT_PACKAGE = THIS_PACKAGE.getName().replaceAll("\\.api$", ".algo");
+
+ private CompressionAlgorithmFactory() {
+ // util class.
+ }
+
+ private static Map.@Nullable Entry> mapEntryOrNull(final Class clazz) {
+ return ReflectionUtil.newInstance(clazz)
+ .map(compInstance -> new AbstractMap.SimpleEntry<>(compInstance.getCompressionTypeValue().getLongValue(), clazz))
+ .orElse(null);
+ }
+
+ public static CompressionAlgorithm forType(final long compressionType) {
+ return Optional.ofNullable(getTypeMappings().get(compressionType))
+ .flatMap(ReflectionUtil::newInstance)
+ .orElseGet(UnknownAlgorithm::new);
+ }
+
+ /* Utility methods */
+
+ private static List> getImplementations() {
+ return ResourceHolder.newInstance(ROOT_PACKAGE).getImplementations();
+ }
+
+ private static Map> getTypeMappings() {
+ return ResourceHolder.newInstance(ROOT_PACKAGE).getTypeMapping();
+ }
+
+
+ private static class ResourceHolder {
+
+ private static ResourceHolder INSTANCE = null;
+
+ private final List> implementations;
+
+ private final Map> typeMapping;
+
+ public ResourceHolder(final String rootPackage) {
+ this.implementations = ReflectionUtil.loadImplementations(rootPackage, CompressionAlgorithm.class);
+ this.typeMapping = loadTypeMapping(this.implementations);
+ }
+
+ public static ResourceHolder newInstance(final String rootPackage) {
+ if (INSTANCE == null) {
+ INSTANCE = new ResourceHolder(rootPackage);
+ }
+
+ return INSTANCE;
+ }
+
+ private Map> loadTypeMapping(final List> implementations) {
+ return implementations.stream()
+ .map(CompressionAlgorithmFactory::mapEntryOrNull)
+ .filter(Objects::nonNull)
+ .collect(toMap(Map.Entry::getKey, Map.Entry::getValue));
+ }
+
+ public List> getImplementations() {
+ return this.implementations;
+ }
+
+ public Map> getTypeMapping() {
+ return this.typeMapping;
+ }
+ }
+}
diff --git a/compression/compression-api/src/main/java/de/bmarwell/zchunk/compression/api/internal/ReflectionUtil.java b/compression/compression-api/src/main/java/de/bmarwell/zchunk/compression/api/internal/ReflectionUtil.java
new file mode 100644
index 0000000..e5d7ee7
--- /dev/null
+++ b/compression/compression-api/src/main/java/de/bmarwell/zchunk/compression/api/internal/ReflectionUtil.java
@@ -0,0 +1,152 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.compression.api.internal;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static java.util.stream.Collectors.toList;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Predicate;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+public final class ReflectionUtil {
+
+ private static final Logger LOG = Logger.getLogger(ReflectionUtil.class.getCanonicalName());
+
+ private ReflectionUtil() {
+ // util
+ }
+
+ public static List> loadImplementations(final String rootPackage, final Class clazz) {
+ final List> classes = getClasses(rootPackage, clazz);
+
+ return classes.stream()
+ .filter(classImplementsCompressionAlgorithm(clazz))
+ .collect(toList());
+ }
+
+
+ private static List> getClasses(final String rootPackage, final Class targetClazz) {
+ try {
+ final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ final String path = rootPackage.replace('.', '/');
+ final Enumeration resources = classLoader.getResources(path);
+ final List dirs = new ArrayList<>();
+
+ while (resources.hasMoreElements()) {
+ final URL resource = resources.nextElement();
+ dirs.add(new File(resource.getFile()));
+ }
+
+ return dirs.stream()
+ .map(dir -> findClasses(dir, rootPackage, targetClazz))
+ .flatMap(Collection::stream)
+ .collect(toList());
+ } catch (final IOException ioEx) {
+ return emptyList();
+ }
+ }
+
+
+ /**
+ * Recursive method used to find all classes in a given directory and subdirs.
+ *
+ * @param directory
+ * The base directory
+ * @param packageName
+ * The package name for classes found inside the base directory
+ * @return The classes
+ */
+ private static List> findClasses(final File directory, final String packageName, final Class clazzType) {
+ if (!directory.exists()) {
+ return Collections.emptyList();
+ }
+
+ final List files = getListFromArray(directory.listFiles());
+
+ return files.stream()
+ .map(file -> findClasses(packageName, clazzType, file))
+ .flatMap(Collection::stream)
+ .collect(toList());
+ }
+
+ private static List> findClasses(final String packageName, final Class clazzType, final File file) {
+ if (file.isDirectory()) {
+ if (file.getName().contains(".")) {
+ // bad match.
+ return emptyList();
+ }
+
+ return findClasses(file, packageName + "." + file.getName(), clazzType);
+ }
+
+ if (file.getName().endsWith(".class")) {
+ final @Nullable Class aClass = loadClass(packageName, clazzType, file);
+ if (aClass != null) {
+ return singletonList(aClass);
+ }
+ }
+
+ return emptyList();
+ }
+
+ @Nullable
+ private static Class loadClass(final String packageName, final Class clazzType, final File file) {
+ try {
+ final Class aClass = Class.forName(packageName + '.' + file.getName().substring(0, file.getName().length() - 6));
+ if (classImplementsCompressionAlgorithm(clazzType).test(aClass)) {
+ return (Class) aClass;
+ }
+ } catch (final ClassNotFoundException e) {
+ LOG.log(Level.WARNING, e, () -> String.format("Class file [%s] found, but unable to create instance.", file.getAbsolutePath()));
+ }
+
+ return null;
+ }
+
+ private static List getListFromArray(final T[] input) {
+ return Optional.ofNullable(input)
+ .map(Arrays::asList)
+ .orElseGet(Collections::emptyList);
+ }
+
+ public static Optional newInstance(final Class clazz) {
+ try {
+ return Optional.of(clazz.newInstance());
+ } catch (final InstantiationException | IllegalAccessException e) {
+ LOG.log(Level.WARNING, e, () -> String.format("Unable to instantiate class [%s]. Skipping.", clazz));
+ return Optional.empty();
+ }
+ }
+
+ private static Predicate> classImplementsCompressionAlgorithm(final Class type) {
+ return clazz -> getListFromArray(type.getInterfaces()).contains(type);
+ }
+
+}
diff --git a/compression/compression-api/src/test/java/de/bmarwell/zchunk/compression/api/CompressionAlgorithmFactoryTest.java b/compression/compression-api/src/test/java/de/bmarwell/zchunk/compression/api/CompressionAlgorithmFactoryTest.java
new file mode 100644
index 0000000..c96db2c
--- /dev/null
+++ b/compression/compression-api/src/test/java/de/bmarwell/zchunk/compression/api/CompressionAlgorithmFactoryTest.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.compression.api;
+
+import de.bmarwell.zchunk.compression.algo.unknown.UnknownAlgorithm;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class CompressionAlgorithmFactoryTest {
+
+ @Test
+ public void testGetUnknown() {
+ final CompressionAlgorithm algorithm = CompressionAlgorithmFactory.forType(-1L);
+
+ Assertions.assertEquals(algorithm.getClass(), UnknownAlgorithm.class);
+ }
+}
diff --git a/compression/compression-api/src/test/resources/logging.properties b/compression/compression-api/src/test/resources/logging.properties
new file mode 100644
index 0000000..caa25eb
--- /dev/null
+++ b/compression/compression-api/src/test/resources/logging.properties
@@ -0,0 +1,26 @@
+#
+# Copyright 2019, the zchunk-java contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+handlers=java.util.logging.ConsoleHandler
+# default log level
+.level=FINE
+# console handler settings
+java.util.logging.ConsoleHandler.level=ALL
+java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
+java.util.logging.SimpleFormatter.format=[%1$tFT%1$tT.%1$tLZ] [%4$-6s] %2$s - %5$s%6$s%n
+# class specific settings
+de.bmarwell.zchunk.compression.level=FINER
+# 3rd party
+org.junit.platform.level=INFO
diff --git a/fileformat/pom.xml b/fileformat/pom.xml
new file mode 100644
index 0000000..303a899
--- /dev/null
+++ b/fileformat/pom.xml
@@ -0,0 +1,73 @@
+
+
+
+
+
+ zchunk-parent
+ de.bmarwell.zchunk
+ 1.0.0-SNAPSHOT
+
+ 4.0.0
+
+ zchunk-fileformat
+
+
+
+
+ de.bmarwell.zchunk
+ compressedint
+ 1.0.0-SNAPSHOT
+
+
+
+
+ de.bmarwell.zchunk
+ compression-api
+ 1.0.0-SNAPSHOT
+
+
+
+
+
+ org.immutables
+ value
+ provided
+
+
+
+ org.checkerframework
+ checker-qual
+ provided
+
+
+
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ test
+
+
+
+
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/HeaderChecksumType.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/HeaderChecksumType.java
new file mode 100644
index 0000000..e5cd704
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/HeaderChecksumType.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.StringJoiner;
+
+/**
+ * 0 = SHA-1
+ * 1 = SHA-256
+ */
+public enum HeaderChecksumType {
+ SHA1("SHA-1"),
+ SHA256("SHA-256");
+
+ private final String digestAlgorithm;
+ private final int digestLength;
+
+ HeaderChecksumType(final String digestAlgorithm) {
+ try {
+ this.digestAlgorithm = digestAlgorithm;
+ this.digestLength = MessageDigest.getInstance(digestAlgorithm).getDigestLength();
+ } catch (final NoSuchAlgorithmException algoEx) {
+ throw new IllegalArgumentException("Unable to create hashing algorithm: [" + digestAlgorithm + "]. Check your JVM settings.", algoEx);
+ }
+ }
+
+ public int getDigestLength() {
+ return this.digestLength;
+ }
+
+ public MessageDigest digest() {
+ try {
+ return MessageDigest.getInstance(this.digestAlgorithm);
+ } catch (final NoSuchAlgorithmException algoEx) {
+ throw new UnsupportedOperationException("Not implemented: [" + this.digestAlgorithm + "].");
+ }
+ }
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", HeaderChecksumType.class.getSimpleName() + "[", "]")
+ .add("digestAlgorithm=" + this.digestAlgorithm)
+ .add("digestLength=" + this.digestLength)
+ .add("ordinal=" + this.ordinal())
+ .toString();
+ }
+
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/IndexChecksumType.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/IndexChecksumType.java
new file mode 100644
index 0000000..6636ae5
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/IndexChecksumType.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.StringJoiner;
+
+/**
+ * Current values:
+ * 0 = SHA-1
+ * 1 = SHA-256
+ * 2 = SHA-512
+ * 3 = SHA-512/128 (first 128 bits of SHA-512 checksum)
+ */
+public enum IndexChecksumType {
+ SHA1("SHA-1", -1),
+ SHA256("SHA-256", -1),
+ SHA512("SHA-512", -1),
+ SHA512_128("SHA-512", 16);
+
+ private final MessageDigest digestAlgorithm;
+ private final int length;
+
+ IndexChecksumType(final String digestAlgorithm, final int length) {
+ try {
+ this.digestAlgorithm = MessageDigest.getInstance(digestAlgorithm);
+ if (length != -1) {
+ this.length = length;
+ } else {
+ this.length = this.digestAlgorithm.getDigestLength();
+ }
+ } catch (final NoSuchAlgorithmException algoEx) {
+ throw new IllegalArgumentException("Unable to create hashing algorithm: [" + digestAlgorithm + "]. Check your JVM settings.", algoEx);
+ }
+ }
+
+ public int actualChecksumLength() {
+ return this.length;
+ }
+
+ public byte[] digest(final byte[] input) {
+ final byte[] digest = this.digestAlgorithm.digest(input);
+
+ if (this.length != this.digestAlgorithm.getDigestLength()) {
+ final byte[] actualDigest = new byte[this.length];
+ System.arraycopy(digest, 0, actualDigest, 0, this.length);
+ return actualDigest;
+ }
+
+ return digest;
+ }
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", IndexChecksumType.class.getSimpleName() + "[", "]")
+ .add("digestAlgorithm=" + this.digestAlgorithm.getAlgorithm())
+ .add("actualChecksumLength=" + this.length)
+ .add("ordinal=" + this.ordinal())
+ .toString();
+ }
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/OptionalElement.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/OptionalElement.java
new file mode 100644
index 0000000..9ada22d
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/OptionalElement.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import java.math.BigInteger;
+import org.immutables.value.Value;
+
+@Value.Immutable
+public interface OptionalElement {
+
+ CompressedInt getId();
+
+ CompressedInt getDataSize();
+
+ byte[] getData();
+
+ @Value.Derived
+ default long getTotalLength() {
+ return BigInteger.valueOf(getId().getCompressedBytes().length)
+ .add(BigInteger.valueOf(getDataSize().getCompressedBytes().length))
+ // either this or getData().length
+ .add(getDataSize().getValue())
+ .longValueExact();
+ }
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/PrefaceFlag.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/PrefaceFlag.java
new file mode 100644
index 0000000..6578eeb
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/PrefaceFlag.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import de.bmarwell.zchunk.fileformat.util.ByteUtils;
+import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.StringJoiner;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+/*
+ * Current flags are:
+ * bit 0: File has data streams
+ * bit 1: File has optional elements
+ */
+public enum PrefaceFlag {
+ HAS_DATA_STREAMS(0b00000001L),
+ HAS_OPTIONAL_ELEMENTS(0b00000010L);
+
+ private final long bitflag;
+
+ PrefaceFlag(final long flag) {
+ this.bitflag = flag;
+ }
+
+ public static Set getPrefaceFlags(final CompressedInt ci) {
+ final AtomicReference remainingFlagLong = new AtomicReference<>(ci.getValue());
+
+ return getPrefaceFlags(remainingFlagLong);
+ }
+
+ private static Set getPrefaceFlags(final AtomicReference remainingFlagLong) {
+ final Set foundFlags = Arrays.stream(PrefaceFlag.values())
+ .filter(currentFlag -> ByteUtils.decrease(remainingFlagLong, currentFlag.getBitflag()))
+ .collect(Collectors.toSet());
+
+ if (remainingFlagLong.get().longValue() != 0L) {
+ throw new UnsupportedOperationException(
+ "Flags not supported yet: [" + ByteUtils.longToBinaryString(remainingFlagLong.get().longValue()) + "].");
+ }
+
+ return foundFlags;
+ }
+
+ public long getBitflag() {
+ return this.bitflag;
+ }
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", PrefaceFlag.class.getSimpleName() + "[", "]")
+ .add("bitflag=" + this.bitflag)
+ .toString();
+ }
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkConstants.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkConstants.java
new file mode 100644
index 0000000..0489827
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkConstants.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import static java.security.Security.getProviders;
+
+import de.bmarwell.zchunk.compressedint.CompressedIntUtil;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.security.Provider;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public final class ZChunkConstants {
+
+ private ZChunkConstants() {
+ // util
+ }
+
+ public static class Header {
+
+ public static final byte[] FILE_MAGIC = new byte[]{'\0', 'Z', 'C', 'K', '1'};
+
+ private static final int MAX_CHECKSUM_SIZE = getLargestHashSize();
+ public static int MAX_LEAD_SIZE = FILE_MAGIC.length
+ // checksum type (ci)
+ + CompressedIntUtil.MAX_COMPRESSED_INT_LENGTH
+ // header size (ci)
+ + CompressedIntUtil.MAX_COMPRESSED_INT_LENGTH
+ + MAX_CHECKSUM_SIZE;
+
+ private static int getLargestHashSize() {
+ final String digestClassName = MessageDigest.class.getSimpleName();
+ final String aliasPrefix = "Alg.Alias." + digestClassName + ".";
+
+ return Arrays.stream(getProviders())
+ .flatMap(prov -> {
+ final Set algorithms = new HashSet<>(0);
+
+ prov.getServices().stream()
+ .filter(s -> digestClassName.equalsIgnoreCase(s.getType()))
+ .map(Provider.Service::getAlgorithm)
+ .collect(Collectors.toCollection(() -> algorithms));
+
+ prov.keySet().stream()
+ .map(Object::toString)
+ .filter(k -> k.startsWith(aliasPrefix))
+ .map(k -> String.format("\"%s\" -> \"%s\"", k.substring(aliasPrefix.length()), prov.get(k).toString()))
+ .collect(Collectors.toCollection(() -> algorithms));
+
+ return algorithms.stream();
+ })
+ .map(algo -> {
+ try {
+ return MessageDigest.getInstance(algo);
+ } catch (NoSuchAlgorithmException e) {
+ return null;
+ }
+ })
+ .filter(Objects::nonNull)
+ .mapToInt(MessageDigest::getDigestLength)
+ .max().orElse(512 / 8);
+ }
+ }
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkFile.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkFile.java
new file mode 100644
index 0000000..eede773
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkFile.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import org.immutables.value.Value;
+
+
+@Value.Immutable
+public interface ZChunkFile {
+
+ ZChunkHeader getHeader();
+
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeader.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeader.java
new file mode 100644
index 0000000..a9f9053
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeader.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import org.immutables.value.Value;
+
+/**
+ * The header consists of four parts:
+ *
+ *
+ *
The lead: Everything necessary to validate the header
+ *
The preface: Metadata about the zchunk file
+ *
The index: Details about each chunk
+ *
The signatures: Signatures used to sign the zchunk file
+ *
+ */
+@Value.Immutable
+public interface ZChunkHeader {
+
+ ZChunkHeaderLead getLead();
+
+ ZChunkHeaderPreface getPreface();
+
+ ZChunkHeaderIndex getIndex();
+
+ ZChunkHeaderSignatures getSignatures();
+
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderChunkInfo.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderChunkInfo.java
new file mode 100644
index 0000000..c5d28a4
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderChunkInfo.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import de.bmarwell.zchunk.fileformat.util.ByteUtils;
+import java.util.Optional;
+import java.util.StringJoiner;
+import org.immutables.value.Value;
+
+/**
+ *
+ * (Chunk stream will only exist if flag 0 is set to 1)
+ * [+===================+================+===================+
+ * [| Chunk stream (ci) | Chunk checksum | Chunk length (ci) |
+ * [+===================+================+===================+
+ *
+ * +==========================+]
+ * | Uncompressed length (ci) |] ...
+ * +==========================+]
+ *
+ */
+
+@Value.Immutable
+public abstract class ZChunkHeaderChunkInfo {
+
+ public abstract long getCurrentIndex();
+
+ public abstract Optional getChunkStream();
+
+ public abstract byte[] getChunkChecksum();
+
+ public abstract CompressedInt getChunkLength();
+
+ public abstract CompressedInt getChunkUncompressedLength();
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", ZChunkHeaderChunkInfo.class.getSimpleName() + "[", "]")
+ .add("index=" + getCurrentIndex())
+ .add("chunkStream=" + ByteUtils.byteArrayToHexString(getChunkStream().orElse(new byte[0])))
+ .add("chunkChecksum=" + ByteUtils.byteArrayToHexString(getChunkChecksum()))
+ .add("chunkLength=" + getChunkLength())
+ .add("chunkUncompressedLength=" + getChunkUncompressedLength())
+ .toString();
+ }
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderFactory.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderFactory.java
new file mode 100644
index 0000000..5cc936c
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderFactory.java
@@ -0,0 +1,250 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import static de.bmarwell.zchunk.fileformat.ZChunkConstants.Header.MAX_LEAD_SIZE;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import de.bmarwell.zchunk.compressedint.CompressedIntFactory;
+import de.bmarwell.zchunk.compression.api.CompressionAlgorithm;
+import de.bmarwell.zchunk.compression.api.CompressionAlgorithmFactory;
+import de.bmarwell.zchunk.compression.api.ImmutableCompressionAlgorithm;
+import de.bmarwell.zchunk.fileformat.err.InvalidFileException;
+import de.bmarwell.zchunk.fileformat.parser.ZChunkIndexParser;
+import de.bmarwell.zchunk.fileformat.parser.ZChunkLeadParser;
+import de.bmarwell.zchunk.fileformat.parser.ZChunkPrefaceParser;
+import de.bmarwell.zchunk.fileformat.util.OffsetUtil;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.RandomAccessFile;
+import java.math.BigInteger;
+import java.nio.channels.Channels;
+import java.util.Set;
+import java.util.logging.Logger;
+
+public final class ZChunkHeaderFactory {
+
+ private static final Logger LOG = Logger.getLogger(ZChunkHeaderFactory.class.getCanonicalName());
+
+ private ZChunkHeaderFactory() {
+ //
+ }
+
+ /**
+ * Reads in a zchunk file.
+ *
+ *
The header part will stay in memory (heap).
+ * The data streams and/or chunks will be available as inputstream, but are not
+ * eagerly loaded into memory.
+ *
+ * @param input
+ * the input file.
+ * @return a {@link ZChunkFile} instance.
+ * @throws InvalidFileException
+ * if the input file is not a zchunk file.
+ * @throws NullPointerException
+ * if the input file is {@code null}.
+ */
+ public static ZChunkFile fromFile(final File input) {
+ final ZChunkHeader header = getZChunkFileHeader(input);
+
+ return ImmutableZChunkFile.builder().header(header).build();
+ }
+
+ private static ZChunkHeader getZChunkFileHeader(final File input) {
+ final ZChunkHeaderLead lead = readFileHeaderLead(input);
+ final byte[] completeHeader = readCompleteHeader(input, lead);
+ final ZChunkHeaderPreface preface = readHeaderPreface(completeHeader, lead);
+ final ZChunkHeaderIndex index = readHeaderIndex(completeHeader, lead, preface);
+ final ZChunkHeaderSignatures signatures = readSignatureIndex(completeHeader, lead, preface, index);
+
+ return ImmutableZChunkHeader.builder()
+ .lead(lead)
+ .preface(preface)
+ .index(index)
+ .signatures(signatures)
+ .build();
+ }
+
+ public static ZChunkHeader fromStream(final InputStream byteStream) {
+ try {
+ final byte[] leadBytes = new byte[MAX_LEAD_SIZE];
+ final int read = byteStream.read(leadBytes);
+
+ if (read < MAX_LEAD_SIZE) {
+ throw new IllegalArgumentException("Unable to read enough bytes from bytestream!");
+ }
+
+ final ZChunkHeaderLead lead = readFileHeaderLead(leadBytes);
+ final byte[] completeHeader = new byte[OffsetUtil.getTotalHeaderSize(lead)];
+ System.arraycopy(leadBytes, 0, completeHeader, 0, leadBytes.length);
+ final long bytesRemaining = OffsetUtil.getLeadLength(lead) - read;
+
+ if (BigInteger.valueOf(bytesRemaining).compareTo(BigInteger.valueOf(Integer.MAX_VALUE)) == 1) {
+ throw new IllegalStateException("Cannot read remaining [" + bytesRemaining + "] bytes. Value too large for integer.");
+ }
+
+ final byte[] headerBytesRemaining = new byte[Math.toIntExact(bytesRemaining)];
+ byteStream.read(headerBytesRemaining);
+ System.arraycopy(headerBytesRemaining, 0, completeHeader, leadBytes.length, headerBytesRemaining.length);
+
+ final ZChunkHeaderPreface headerPreface = readHeaderPreface(completeHeader, lead);
+
+ final ZChunkHeaderIndex index = null;
+ final ZChunkHeaderSignatures signature = ImmutableZChunkHeaderSignatures.builder()
+ .signatureCount(CompressedIntFactory.valueOf(0L))
+ .build();
+
+ return ImmutableZChunkHeader.builder()
+ .lead(lead)
+ .preface(headerPreface)
+ .index(index)
+ .signatures(signature)
+ .build();
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Unable to read enough bytes from bytestream!", ioEx);
+ }
+ }
+
+ public static ZChunkHeaderLead readFileHeaderLead(final byte[] input) {
+ if (input.length < MAX_LEAD_SIZE) {
+ throw new IllegalArgumentException("No enough bytes to read lead.");
+ }
+
+ final ZChunkLeadParser leadParser = ZChunkLeadParser.fromBytes(input);
+
+ return getZChunkFileHeaderLeadFromParser(leadParser);
+ }
+
+ public static ZChunkHeaderLead readFileHeaderLead(final File input) {
+ final ZChunkLeadParser leadParser = ZChunkLeadParser.fromFile(input);
+
+ return getZChunkFileHeaderLeadFromParser(leadParser);
+ }
+
+ private static ZChunkHeaderLead getZChunkFileHeaderLeadFromParser(final ZChunkLeadParser leadParser) {
+ return ImmutableZChunkHeaderLead.builder()
+ .id(leadParser.readLeadId())
+ .checksumTypeInt(leadParser.readLeadCksumType())
+ .headerSize(leadParser.readHeaderSize())
+ .checksum(leadParser.readHeaderChecksum())
+ .build();
+ }
+
+ private static byte[] readCompleteHeader(final File input, final ZChunkHeaderLead lead) {
+ try (final FileInputStream fis = new FileInputStream(input)) {
+ final int totalHeaderSize = OffsetUtil.getTotalHeaderSize(lead);
+ final byte[] buffer = new byte[totalHeaderSize];
+ final int readCount = fis.read(buffer);
+
+ if (readCount < totalHeaderSize) {
+ throw new IllegalArgumentException("Cannot read header, file too short?");
+ }
+
+ return buffer;
+ } catch (final IOException ioEx) {
+ throw new InvalidFileException("File too short?", input, ioEx);
+ }
+ }
+
+ public static ZChunkHeaderPreface readHeaderPreface(final byte[] completeHeader, final ZChunkHeaderLead lead) {
+ final ZChunkPrefaceParser prefaceParser = ZChunkPrefaceParser.fromBytes(completeHeader, lead);
+
+ return getZChunkFileHeaderPrefaceFromParser(prefaceParser);
+ }
+
+ private static ZChunkHeaderPreface getZChunkFileHeaderPrefaceFromParser(final ZChunkPrefaceParser prefaceParser) {
+ final CompressedInt prefaceFlagsInt = prefaceParser.readFlagsInt();
+ final Set flags = PrefaceFlag.getPrefaceFlags(prefaceFlagsInt);
+
+ final CompressionAlgorithm compressionAlgorithm = ImmutableCompressionAlgorithm.builder()
+ .compressionTypeValue(prefaceParser.readCompressionType())
+ .name("unknown")
+ .outputStreamSupplier(a -> a)
+ .build();
+
+ return ImmutableZChunkHeaderPreface.builder()
+ .totalDataChecksum(prefaceParser.readTotalDataCksum())
+ .prefaceFlagsInt(prefaceFlagsInt)
+ .compressionAlgorithm(compressionAlgorithm)
+ .optionalElementCount(CompressedIntFactory.valueOf(0))
+ .addAllPrefaceFlags(flags)
+ .build();
+ }
+
+ public static ZChunkHeaderPreface readFileHeaderPreface(final File input, final ZChunkHeaderLead lead) {
+ return readFileHeaderPreface(input, lead.getChecksumType(), OffsetUtil.getLeadLength(lead));
+ }
+
+ public static ZChunkHeaderPreface readFileHeaderPreface(final File zckFile, final HeaderChecksumType headerChecksumType,
+ final long leadLength) {
+ final byte[] cksum = new byte[headerChecksumType.getDigestLength()];
+
+ try (final RandomAccessFile randomAccessFile = new RandomAccessFile(zckFile, "r")) {
+ randomAccessFile.seek(leadLength);
+ randomAccessFile.read(cksum);
+
+ final InputStream inputStream = Channels.newInputStream(randomAccessFile.getChannel());
+ final CompressedInt flags = getPrefaceFlagsFromInputStream(inputStream);
+
+ final CompressedInt compressionType = CompressedIntFactory.readCompressedInt(inputStream);
+ final CompressionAlgorithm compressionAlgorithm = CompressionAlgorithmFactory.forType(compressionType.getLongValue());
+
+ return ImmutableZChunkHeaderPreface.builder()
+ .totalDataChecksum(cksum)
+ .prefaceFlagsInt(flags)
+ .compressionAlgorithm(compressionAlgorithm)
+ .optionalElementCount(CompressedIntFactory.valueOf(0))
+ .addAllPrefaceFlags(PrefaceFlag.getPrefaceFlags(flags))
+ .build();
+ } catch (final IOException ioEx) {
+ throw new InvalidFileException("Unable to read preface of file.", zckFile, ioEx);
+ }
+ }
+
+ public static CompressedInt getPrefaceFlagsFromInputStream(final InputStream inputStream) throws IOException {
+ return CompressedIntFactory.readCompressedInt(inputStream);
+ }
+
+ private static ZChunkHeaderIndex readHeaderIndex(final byte[] completeHeader, final ZChunkHeaderLead lead,
+ final ZChunkHeaderPreface preface) {
+ final ZChunkIndexParser parser = ZChunkIndexParser.fromBytes(completeHeader, lead, preface);
+
+ final CompressedInt indexChecksumType = parser.readIndexCksumType();
+
+ return ImmutableZChunkHeaderIndex.builder()
+ .indexSize(parser.readIndexSize())
+ .chunkChecksumTypeInt(indexChecksumType)
+ .chunkCount(parser.readChunkCount())
+ .dictChecksum(parser.readDictChecksum())
+ .dictLength(parser.readDictLength())
+ .uncompressedDictLength(parser.readUncompressedDictLength())
+ .addAllChunkInfo(parser.readChunkInfos())
+ .build();
+ }
+
+ private static ZChunkHeaderSignatures readSignatureIndex(
+ final byte[] completeHeader, final ZChunkHeaderLead lead, final ZChunkHeaderPreface preface, final ZChunkHeaderIndex index) {
+
+ return ImmutableZChunkHeaderSignatures.builder()
+ .signatureCount(CompressedIntFactory.valueOf(0L))
+ .build();
+ }
+
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderIndex.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderIndex.java
new file mode 100644
index 0000000..0e7fff5
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderIndex.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import de.bmarwell.zchunk.fileformat.util.ByteUtils;
+import java.util.List;
+import java.util.Optional;
+import java.util.StringJoiner;
+import org.immutables.value.Value;
+
+/**
+ *
+ * +=================+==========================+==================+
+ * | Index size (ci) | Chunk checksum type (ci) | Chunk count (ci) |
+ * +=================+==========================+==================+
+ *
+ * (Dict stream will only exist if flag 0 is set to 1)
+ * +==================+===============+==================+
+ * | Dict stream (ci) | Dict checksum | Dict length (ci) |
+ * +==================+===============+==================+
+ *
+ * +===============================+
+ * | Uncompressed dict length (ci) |
+ * +===============================+
+ *
+ * (Chunk stream will only exist if flag 0 is set to 1)
+ * [+===================+================+===================+
+ * [| Chunk stream (ci) | Chunk checksum | Chunk length (ci) |
+ * [+===================+================+===================+
+ *
+ * +==========================+]
+ * | Uncompressed length (ci) |] ...
+ * +==========================+]
+ *
+ */
+@Value.Immutable
+public abstract class ZChunkHeaderIndex {
+
+ public abstract CompressedInt getIndexSize();
+
+ public abstract CompressedInt getChunkChecksumTypeInt();
+
+ @Value.Derived
+ public IndexChecksumType getChunkChecksumType() {
+ return IndexChecksumType.values()[getChunkChecksumTypeInt().getIntValue()];
+ }
+
+ public abstract CompressedInt getChunkCount();
+
+ public abstract Optional getDictStream();
+
+ /**
+ * Must be all zeros if no dict present.
+ *
+ * @return dict checksum or all zeros.
+ */
+ public abstract byte[] getDictChecksum();
+
+ /**
+ * Dict length or zero.
+ *
+ * @return dict length.
+ */
+ public abstract CompressedInt getDictLength();
+
+ public abstract CompressedInt getUncompressedDictLength();
+
+ public abstract List getChunkInfo();
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", ZChunkHeaderIndex.class.getSimpleName() + "[", "]")
+ .add("indexSize=" + getIndexSize())
+ .add("chunkChecksumType=" + getChunkChecksumType())
+ .add("chunkCount=" + getChunkCount())
+ .add("dictStream=" + ByteUtils.byteArrayToHexString(getDictStream().orElse(new byte[0])))
+ .add("dictChecksum=" + ByteUtils.byteArrayToHexString(getDictChecksum()))
+ .add("dictLength=" + getDictLength())
+ .add("dictUncompressedLength=" + getUncompressedDictLength())
+ .add("chunkInfo=" + getChunkInfo())
+ .toString();
+ }
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderLead.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderLead.java
new file mode 100644
index 0000000..96a8621
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderLead.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import de.bmarwell.zchunk.fileformat.err.InvalidFileException;
+import de.bmarwell.zchunk.fileformat.util.ByteUtils;
+import java.util.Arrays;
+import java.util.StringJoiner;
+import org.immutables.value.Value;
+
+/**
+ * The lead:
+ *
+ *
+ */
+@Value.Immutable
+public abstract class ZChunkHeaderLead {
+
+ public abstract byte[] getId();
+
+ public abstract CompressedInt getChecksumTypeInt();
+
+ @Value.Derived
+ public HeaderChecksumType getChecksumType() {
+ return HeaderChecksumType.values()[getChecksumTypeInt().getIntValue()];
+ }
+
+ /**
+ * Header size:
+ * This is an integer containing the size of the header, not including the lead.
+ *
+ * @return Remaining bytes after the header.
+ */
+ public abstract CompressedInt getHeaderSize();
+
+ /**
+ * Checksum of the whole header.
+ *
+ * @return the checksum of the whole header.
+ */
+ public abstract byte[] getChecksum();
+
+ @Value.Check
+ public void checkLead() {
+ if (!Arrays.equals(ZChunkConstants.Header.FILE_MAGIC, this.getId())) {
+ throw new InvalidFileException("file magic differs: [" + ByteUtils.byteArrayToHexString(this.getId()) + "].");
+ }
+ }
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", ZChunkHeaderLead.class.getSimpleName() + "[", "]")
+ .add("id='" + ByteUtils.byteArrayToHexString(getId()) + "'")
+ .add("cksumtype=" + getChecksumTypeInt().getValue().toString())
+ .add("cksumtype='" + getChecksumType() + "'")
+ .add("headerSize=" + getHeaderSize().getValue().toString())
+ .add("cksum='" + ByteUtils.byteArrayToHexString(getChecksum()) + "'")
+ .toString();
+ }
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderPreface.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderPreface.java
new file mode 100644
index 0000000..b337753
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderPreface.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import de.bmarwell.zchunk.compression.api.CompressionAlgorithm;
+import de.bmarwell.zchunk.fileformat.util.ByteUtils;
+import java.util.List;
+import java.util.Set;
+import java.util.StringJoiner;
+import org.immutables.value.Value;
+
+/**
+ *
+ * +===============+============+========================+
+ * | Data checksum | Flags (ci) | Compression type (ci ) |
+ * +===============+============+========================+
+ *
+ * (Optional elements will only be set if flag 1 is set to 1)
+ *
+ * +=============================+
+ * | Optional element count (ci) |
+ * +=============================+
+ *
+ * [+==========================+=================================+
+ * [| Optional element id (ci) | Optional element data size (ci) |
+ * [+==========================+=================================+
+ *
+ * +=======================+]
+ * | Optional element data |] ...
+ * +=======================+]
+ *
+ */
+@Value.Immutable
+public abstract class ZChunkHeaderPreface {
+
+ /**
+ * Returns the checksum of the data segment.
+ *
+ * @return the checksum of the data segment.
+ */
+ public abstract byte[] getTotalDataChecksum();
+
+ public abstract CompressedInt getPrefaceFlagsInt();
+
+ public abstract Set getPrefaceFlags();
+
+ @Value.Derived
+ public boolean hasOptionalElements() {
+ return getPrefaceFlags().contains(PrefaceFlag.HAS_OPTIONAL_ELEMENTS);
+ }
+
+ public abstract CompressionAlgorithm getCompressionAlgorithm();
+
+ public abstract CompressedInt getOptionalElementCount();
+
+ public abstract List getOptionalElements();
+
+ /**
+ * Flags
+ * This is a compressed integer containing a bitmask of the flags. All unused
+ * flags MUST be set to 0. If a decoder sees a flag set that it doesn't
+ * recognize, it MUST exit with an error.
+ */
+ @Value.Check
+ public void testUnknownFlags() {
+ final Set flags = getPrefaceFlags();
+
+ if (flags.contains(PrefaceFlag.HAS_DATA_STREAMS)) {
+ throw new UnsupportedOperationException("Not implemented: " + PrefaceFlag.HAS_DATA_STREAMS);
+ }
+
+ if (flags.contains(PrefaceFlag.HAS_OPTIONAL_ELEMENTS)) {
+ throw new UnsupportedOperationException("Not implemented: " + PrefaceFlag.HAS_OPTIONAL_ELEMENTS);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", ZChunkHeaderPreface.class.getSimpleName() + "[", "]")
+ .add("cksum='" + ByteUtils.byteArrayToHexString(getTotalDataChecksum()) + "'")
+ .add("flags='" + getPrefaceFlags() + "'")
+ .add("compressionAlgorithm=" + getCompressionAlgorithm())
+ .add("optionalElementCount=" + getOptionalElementCount())
+ .add("optionalElements=" + getOptionalElements())
+ .toString();
+ }
+
+
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderSignatures.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderSignatures.java
new file mode 100644
index 0000000..16b06fd
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderSignatures.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import org.immutables.value.Value;
+
+@Value.Immutable
+public abstract class ZChunkHeaderSignatures {
+
+ public abstract CompressedInt getSignatureCount();
+
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/err/InvalidFileException.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/err/InvalidFileException.java
new file mode 100644
index 0000000..81e7f2b
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/err/InvalidFileException.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat.err;
+
+import java.io.File;
+import java.util.Optional;
+import java.util.StringJoiner;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+public class InvalidFileException extends RuntimeException {
+
+ private final @Nullable File sourceFile;
+
+ public InvalidFileException(final String message) {
+ super(message);
+ this.sourceFile = null;
+ }
+
+ public InvalidFileException(final String message, final File zckFile) {
+ super(message);
+ this.sourceFile = zckFile;
+ }
+
+ public InvalidFileException(final String message, final File zckFile, final Throwable cause) {
+ super(message, cause);
+ this.sourceFile = zckFile;
+ }
+
+ public Optional getSourceFile() {
+ return Optional.ofNullable(this.sourceFile);
+ }
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", InvalidFileException.class.getSimpleName() + "[", "]")
+ .add("super=" + super.toString())
+ .add("sourceFile='" + getSourceFile().map(File::getAbsolutePath).orElse("unknown") + "'")
+ .toString();
+ }
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/package-info.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/package-info.java
new file mode 100644
index 0000000..7ecc485
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@Value.Style(stagedBuilder = true, jdkOnly = true)
+package de.bmarwell.zchunk.fileformat;
+
+import org.immutables.value.Value;
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/parser/ZChunkIndexParser.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/parser/ZChunkIndexParser.java
new file mode 100644
index 0000000..ea84f89
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/parser/ZChunkIndexParser.java
@@ -0,0 +1,224 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat.parser;
+
+import static java.util.Collections.emptyList;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import de.bmarwell.zchunk.compressedint.CompressedIntFactory;
+import de.bmarwell.zchunk.fileformat.ImmutableZChunkHeaderChunkInfo;
+import de.bmarwell.zchunk.fileformat.IndexChecksumType;
+import de.bmarwell.zchunk.fileformat.PrefaceFlag;
+import de.bmarwell.zchunk.fileformat.ZChunkHeaderChunkInfo;
+import de.bmarwell.zchunk.fileformat.ZChunkHeaderLead;
+import de.bmarwell.zchunk.fileformat.ZChunkHeaderPreface;
+import de.bmarwell.zchunk.fileformat.util.OffsetUtil;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+
+public final class ZChunkIndexParser {
+
+ private final byte[] completeHeader;
+ private final ZChunkHeaderLead lead;
+ private final ZChunkHeaderPreface preface;
+ private final long indexStart;
+ private long cksumTypeOffset = -1;
+ private long chunkCountOffset = -1;
+ private long optionalDictStreamOffset = -1L;
+ private long dictChecksumOffest = -1L;
+ private long dictLengthOffset = -1L;
+ private long dictUncompressedLengthOffset = -1L;
+ private long chunkStreamOffset = -1L;
+ private CompressedInt chunkCount;
+ private IndexChecksumType chunkChecksumType;
+
+ private ZChunkIndexParser(final byte[] completeHeader, final ZChunkHeaderLead lead, final ZChunkHeaderPreface preface) {
+ this.completeHeader = completeHeader;
+ this.lead = lead;
+ this.preface = preface;
+ this.indexStart = OffsetUtil.getLeadLength(lead) + OffsetUtil.getPrefaceLength(preface);
+ }
+
+ public static ZChunkIndexParser fromBytes(final byte[] completeHeader, final ZChunkHeaderLead lead, final ZChunkHeaderPreface preface) {
+ return new ZChunkIndexParser(completeHeader, lead, preface);
+ }
+
+ public CompressedInt readIndexSize() {
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.completeHeader)) {
+ bis.skip(this.indexStart);
+ final CompressedInt compressedInt = CompressedIntFactory.readCompressedInt(bis);
+ this.cksumTypeOffset = this.indexStart + compressedInt.getCompressedBytes().length;
+
+ return compressedInt;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Cannot read compressed int at offset [" + this.indexStart + "]!", ioEx);
+ }
+ }
+
+ public CompressedInt readIndexCksumType() {
+ if (this.cksumTypeOffset == -1L) {
+ readIndexSize();
+ }
+
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.completeHeader)) {
+ bis.skip(this.cksumTypeOffset);
+ final CompressedInt compressedInt = CompressedIntFactory.readCompressedInt(bis);
+ this.chunkCountOffset = this.cksumTypeOffset + compressedInt.getCompressedBytes().length;
+ this.chunkChecksumType = IndexChecksumType.values()[compressedInt.getIntValue()];
+
+ return compressedInt;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Cannot read compressed int at offset [" + this.cksumTypeOffset + "]!", ioEx);
+ }
+ }
+
+ public CompressedInt readChunkCount() {
+ if (this.chunkCountOffset == -1) {
+ readIndexCksumType();
+ }
+
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.completeHeader)) {
+ bis.skip(this.chunkCountOffset);
+ final CompressedInt compressedInt = CompressedIntFactory.readCompressedInt(bis);
+ this.optionalDictStreamOffset = this.chunkCountOffset + compressedInt.getCompressedBytes().length;
+ this.chunkCount = compressedInt;
+
+ return compressedInt;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Cannot read compressed int at offset [" + this.chunkCountOffset + "]!", ioEx);
+ }
+ }
+
+ public byte[] readDictStream() {
+ if (this.optionalDictStreamOffset == -1L) {
+ readChunkCount();
+ }
+
+ if (!this.preface.getPrefaceFlags().contains(PrefaceFlag.HAS_DATA_STREAMS)) {
+ this.dictChecksumOffest = this.optionalDictStreamOffset;
+ return new byte[0];
+ }
+
+ // TODO: implement property.
+ this.dictChecksumOffest = this.optionalDictStreamOffset;
+ return new byte[0];
+ }
+
+ public byte[] readDictChecksum() {
+ if (this.dictChecksumOffest == -1L) {
+ readDictStream();
+ }
+
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.completeHeader)) {
+ bis.skip(this.dictChecksumOffest);
+
+ // TODO. Which checksum does the dict use? chunk or header checksum?
+ final int dictChecksumLength = this.chunkChecksumType.actualChecksumLength();
+ final byte[] dictChecksum = new byte[dictChecksumLength];
+ bis.read(dictChecksum);
+
+ this.dictLengthOffset = this.dictChecksumOffest + dictChecksumLength;
+
+ return dictChecksum;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Cannot read byte[] int at offset [" + this.dictChecksumOffest + "]!", ioEx);
+ }
+ }
+
+ public CompressedInt readDictLength() {
+ if (this.dictLengthOffset == -1L) {
+ readDictChecksum();
+ }
+
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.completeHeader)) {
+ bis.skip(this.dictLengthOffset);
+
+ final CompressedInt compressedInt = CompressedIntFactory.readCompressedInt(bis);
+ this.dictUncompressedLengthOffset = this.dictLengthOffset + compressedInt.getCompressedBytes().length;
+
+ return compressedInt;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Cannot read compressed int at offset [" + this.dictLengthOffset + "]!", ioEx);
+ }
+ }
+
+ public CompressedInt readUncompressedDictLength() {
+ if (this.dictUncompressedLengthOffset == -1L) {
+ readDictChecksum();
+ }
+
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.completeHeader)) {
+ bis.skip(this.dictUncompressedLengthOffset);
+
+ final CompressedInt compressedInt = CompressedIntFactory.readCompressedInt(bis);
+ this.chunkStreamOffset = this.dictUncompressedLengthOffset + compressedInt.getCompressedBytes().length;
+
+ return compressedInt;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Cannot read compressed int at offset [" + this.dictLengthOffset + "]!", ioEx);
+ }
+ }
+
+ public Iterable readChunkInfos() {
+ if (this.chunkStreamOffset == -1L) {
+ readUncompressedDictLength();
+ }
+
+ if (this.chunkCount.getValue().equals(BigInteger.ZERO)) {
+ return emptyList();
+ }
+
+ final List chunkInfo = new ArrayList<>();
+
+ long currentOffset = 0;
+ // first chunk is the dict chunk.
+ for (long chunkNumber = 0; chunkNumber < this.chunkCount.getLongValue() - 1L; chunkNumber++) {
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.completeHeader)) {
+ bis.skip(this.chunkStreamOffset + currentOffset);
+ if (this.preface.getPrefaceFlags().contains(PrefaceFlag.HAS_DATA_STREAMS)) {
+ //TODO: chunkStream
+ throw new UnsupportedOperationException("data streams not implemented.");
+ }
+
+ final byte[] chunkChecksum = new byte[this.chunkChecksumType.actualChecksumLength()];
+ bis.read(chunkChecksum);
+
+ final CompressedInt compressedChunkLength = CompressedIntFactory.readCompressedInt(bis);
+ final CompressedInt uncompressedChunkLength = CompressedIntFactory.readCompressedInt(bis);
+
+ chunkInfo.add(ImmutableZChunkHeaderChunkInfo.builder()
+ .currentIndex(chunkNumber)
+ .chunkChecksum(chunkChecksum)
+ .chunkLength(compressedChunkLength)
+ .chunkUncompressedLength(uncompressedChunkLength)
+ .build());
+
+ currentOffset += this.chunkChecksumType.actualChecksumLength()
+ + compressedChunkLength.getCompressedBytes().length
+ + uncompressedChunkLength.getCompressedBytes().length;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException(
+ "Cannot read chunk info no. [" + chunkNumber + "] at offset [" + this.chunkStreamOffset + "]!", ioEx);
+ }
+ }
+
+ return chunkInfo;
+ }
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/parser/ZChunkLeadParser.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/parser/ZChunkLeadParser.java
new file mode 100644
index 0000000..3dfe80e
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/parser/ZChunkLeadParser.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat.parser;
+
+import static de.bmarwell.zchunk.fileformat.ZChunkConstants.Header.FILE_MAGIC;
+import static de.bmarwell.zchunk.fileformat.ZChunkConstants.Header.MAX_LEAD_SIZE;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import de.bmarwell.zchunk.compressedint.CompressedIntFactory;
+import de.bmarwell.zchunk.fileformat.HeaderChecksumType;
+import de.bmarwell.zchunk.fileformat.err.InvalidFileException;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+public class ZChunkLeadParser {
+
+ private final byte[] leadBytes = new byte[MAX_LEAD_SIZE];
+ private HeaderChecksumType checksumType;
+ private long headerSizeOffset;
+ private long headerChecksumOffset;
+
+ /**
+ * Private constructor because of possible unsafe operations.
+ *
+ * @param leadBytes
+ * the bytes belonging to the lead.
+ */
+ private ZChunkLeadParser(final byte[] leadBytes) {
+ if (leadBytes.length < MAX_LEAD_SIZE) {
+ throw new IllegalArgumentException("");
+ }
+
+ System.arraycopy(leadBytes, 0, this.leadBytes, 0, MAX_LEAD_SIZE);
+ }
+
+ /**
+ * Create a parser from the given bytes.
+ *
+ * @param leadBytes
+ * the bytes to take as a lead.
+ * @return a parser instance.
+ */
+ public static ZChunkLeadParser fromBytes(final byte[] leadBytes) {
+ return new ZChunkLeadParser(leadBytes);
+ }
+
+ public static ZChunkLeadParser fromFile(final File input) {
+ try (final FileInputStream fr = new FileInputStream(input)) {
+ final byte[] buffer = new byte[MAX_LEAD_SIZE];
+ final int read = fr.read(buffer);
+
+ if (read < MAX_LEAD_SIZE) {
+ throw new InvalidFileException(getExceptionMessage(input), input);
+ }
+
+ if (read != buffer.length) {
+ throw new InvalidFileException(getExceptionMessage(input), input);
+ }
+
+ return new ZChunkLeadParser(buffer);
+ } catch (final IOException ioEx) {
+ throw new InvalidFileException(getExceptionMessage(input), input, ioEx);
+ }
+ }
+
+ private static String getExceptionMessage(final File input) {
+ return String.format("Unable to read [%d] bytes from file [%s].", MAX_LEAD_SIZE, input.getAbsolutePath());
+ }
+
+ public byte[] readLeadId() {
+ final int length = FILE_MAGIC.length;
+
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.leadBytes)) {
+ final byte[] leadid = new byte[length];
+ bis.read(leadid);
+
+ return leadid;
+ } catch (final IOException e) {
+ throw new IllegalArgumentException("Not a zchunk lead.");
+ }
+ }
+
+ public CompressedInt readLeadCksumType() {
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.leadBytes)) {
+ final long skip = bis.skip(FILE_MAGIC.length);
+ if (skip < FILE_MAGIC.length) {
+ throw new IllegalStateException("Unable to skip [" + FILE_MAGIC.length + "] bytes!");
+ }
+
+ final CompressedInt checksumType = CompressedIntFactory.readCompressedInt(bis);
+ this.headerSizeOffset = FILE_MAGIC.length + (long) checksumType.getCompressedBytes().length;
+ this.checksumType = HeaderChecksumType.values()[checksumType.getIntValue()];
+
+ return checksumType;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Cannot read compressed int at offset [" + FILE_MAGIC.length + "]!", ioEx);
+ }
+ }
+
+ public CompressedInt readHeaderSize() {
+ if (this.headerSizeOffset == -1L) {
+ readLeadCksumType();
+ }
+
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.leadBytes)) {
+ bis.skip(this.headerSizeOffset);
+ final CompressedInt compressedInt = CompressedIntFactory.readCompressedInt(bis);
+ this.headerChecksumOffset = this.headerSizeOffset + compressedInt.getCompressedBytes().length;
+
+ return compressedInt;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Cannot read compressed int at offset [" + this.headerSizeOffset + "]!", ioEx);
+ }
+ }
+
+ public byte[] readHeaderChecksum() {
+ if (this.headerChecksumOffset == -1L) {
+ readHeaderSize();
+ }
+
+ final int cksumLength = this.checksumType.getDigestLength();
+
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.leadBytes)) {
+ bis.skip(this.headerChecksumOffset);
+ final byte[] buffer = new byte[cksumLength];
+ final int readBytes = bis.read(buffer);
+
+ if (readBytes < cksumLength) {
+ throw new IllegalStateException("Unable to read [" + cksumLength + "] bytes for checksum!");
+ }
+
+ return buffer;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Cannot read cksum of length [" + cksumLength + "] at offset [" + this.headerChecksumOffset + "]!",
+ ioEx);
+ }
+ }
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/parser/ZChunkPrefaceParser.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/parser/ZChunkPrefaceParser.java
new file mode 100644
index 0000000..b083f97
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/parser/ZChunkPrefaceParser.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat.parser;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import de.bmarwell.zchunk.compressedint.CompressedIntFactory;
+import de.bmarwell.zchunk.fileformat.ZChunkHeaderLead;
+import de.bmarwell.zchunk.fileformat.util.OffsetUtil;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+
+public class ZChunkPrefaceParser {
+
+ private final byte[] header;
+ private final ZChunkHeaderLead lead;
+ private final long flagsOffset;
+ private final long prefaceOffset;
+ private long compressionTypeOffset = -1L;
+
+ private ZChunkPrefaceParser(final byte[] completeHeader, final ZChunkHeaderLead lead) {
+ this.header = completeHeader;
+ this.lead = lead;
+ this.prefaceOffset = OffsetUtil.getLeadLength(lead);
+ this.flagsOffset = this.prefaceOffset + lead.getChecksumType().getDigestLength();
+ }
+
+ public static ZChunkPrefaceParser fromBytes(final byte[] completeHeader, final ZChunkHeaderLead lead) {
+ if (completeHeader.length < OffsetUtil.getTotalHeaderSize(lead)) {
+ throw new IllegalArgumentException("Byte array to short!");
+ }
+
+ return new ZChunkPrefaceParser(completeHeader, lead);
+ }
+
+ public byte[] readTotalDataCksum() {
+ final int digestLength = this.lead.getChecksumType().getDigestLength();
+
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.header)) {
+ final long skip = bis.skip(this.prefaceOffset);
+ final byte[] cksum = new byte[digestLength];
+ final int read = bis.read(cksum, 0, digestLength);
+
+ if (read < digestLength) {
+ throw new IllegalArgumentException("Cannot read cksum of length [" + digestLength + "] at offset [" + this.prefaceOffset + "]!");
+ }
+
+ return cksum;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Cannot read cksum of length [" + digestLength + "] at offset [" + this.prefaceOffset + "]!",
+ ioEx);
+ }
+ }
+
+ public CompressedInt readFlagsInt() {
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.header)) {
+ bis.skip(this.flagsOffset);
+ final CompressedInt compressedInt = CompressedIntFactory.readCompressedInt(bis);
+ this.compressionTypeOffset = this.flagsOffset + compressedInt.getCompressedBytes().length;
+
+ return compressedInt;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Cannot read compressed int at offset [" + this.flagsOffset + "]!", ioEx);
+ }
+ }
+
+ public CompressedInt readCompressionType() {
+ if (this.compressionTypeOffset == -1L) {
+ readFlagsInt();
+ }
+
+ try (final ByteArrayInputStream bis = new ByteArrayInputStream(this.header)) {
+ bis.skip(this.compressionTypeOffset);
+ final CompressedInt compressedInt = CompressedIntFactory.readCompressedInt(bis);
+
+ return compressedInt;
+ } catch (final IOException ioEx) {
+ throw new IllegalArgumentException("Cannot read compressed int at offset [" + this.compressionTypeOffset + "]!", ioEx);
+ }
+ }
+
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/util/ByteUtils.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/util/ByteUtils.java
new file mode 100644
index 0000000..ad51669
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/util/ByteUtils.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat.util;
+
+import de.bmarwell.zchunk.compressedint.CompressedIntUtil;
+import java.math.BigInteger;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+public final class ByteUtils {
+
+ private ByteUtils() {
+ // util class
+ }
+
+ public static String byteArrayToHexString(final byte[] input) {
+ if (input.length == 0) {
+ return "";
+ }
+
+ final String hexString = new BigInteger(1, input).toString(16);
+
+ if (hexString.length() < input.length * 2) {
+ final int missing = input.length * 2 - hexString.length();
+ final String zeros = new String(new char[missing]).replace("\0", "0");
+
+ return zeros + hexString;
+ }
+
+ return hexString;
+ }
+
+ public static String byteArrayToBinaryString(final byte[] input) {
+ return new BigInteger(1, input).toString(2);
+ }
+
+ public static String longToBinaryString(final long input) {
+ return BigInteger.valueOf(input).toString(2);
+ }
+
+ public static byte[] hexStringToByteArray(final String inputString) {
+ final String cleanString = inputString.replaceAll(" ", "").toLowerCase(Locale.ENGLISH);
+ final int len = cleanString.length();
+
+ if ((len % 2) != 0) {
+ throw new IllegalArgumentException("not a valid hex string: [" + cleanString + "].");
+ }
+
+ if (!cleanString.matches("^([0-9a-f][0-9a-f])+$")) {
+ throw new IllegalArgumentException("hex string does not consist of only hex chars: [" + cleanString + "].");
+ }
+
+ final byte[] data = new byte[len / 2];
+
+ for (int i = 0; i < len; i += 2) {
+ final int i1 = Character.digit(cleanString.charAt(i), 16) << 4;
+ final int i2 = Character.digit(cleanString.charAt(i + 1), 16);
+
+ data[i / 2] = (byte) (i1 + i2);
+ }
+
+ return data;
+ }
+
+ public static boolean decrease(final AtomicReference remainingFlagLong, final long bitflag) {
+ final AtomicBoolean changed = new AtomicBoolean();
+ remainingFlagLong.getAndUpdate(curr -> getLongUnaryOperator(curr, bitflag, changed));
+
+ return changed.get();
+ }
+
+ public static BigInteger getLongUnaryOperator(final BigInteger curr, final long bitflag, final AtomicBoolean changed) {
+ // todo: check bigint is not bigger than (long.max * 2) + 1;
+ final long longValue = curr.longValue();
+
+ if ((longValue & bitflag) == bitflag) {
+ changed.set(true);
+ return BigInteger.valueOf(longValue & ~bitflag).and(CompressedIntUtil.UNSIGNED_LONG_MASK);
+ }
+
+ return BigInteger.valueOf(longValue).and(CompressedIntUtil.UNSIGNED_LONG_MASK);
+ }
+
+
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/util/ChecksumUtil.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/util/ChecksumUtil.java
new file mode 100644
index 0000000..c42ac63
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/util/ChecksumUtil.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat.util;
+
+import static java.util.Arrays.asList;
+import static java.util.stream.Collectors.toList;
+
+import de.bmarwell.zchunk.fileformat.HeaderChecksumType;
+import de.bmarwell.zchunk.fileformat.ZChunkHeader;
+import de.bmarwell.zchunk.fileformat.ZChunkHeaderChunkInfo;
+import de.bmarwell.zchunk.fileformat.ZChunkHeaderIndex;
+import de.bmarwell.zchunk.fileformat.ZChunkHeaderLead;
+import de.bmarwell.zchunk.fileformat.ZChunkHeaderPreface;
+import de.bmarwell.zchunk.fileformat.ZChunkHeaderSignatures;
+import java.security.MessageDigest;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Logger;
+
+public final class ChecksumUtil {
+
+ private static final Logger LOG = Logger.getLogger(ChecksumUtil.class.getCanonicalName());
+
+ private ChecksumUtil() {
+ // util
+ }
+
+ public static boolean isValid(final ZChunkHeader header) {
+ final byte[] expectedChecksum = header.getLead().getChecksum();
+ final byte[] calculatedChecksum = calculateHeaderChecksum(header);
+
+ return Arrays.equals(expectedChecksum, calculatedChecksum);
+ }
+
+ public static byte[] calculateHeaderChecksum(final ZChunkHeader header) {
+ final HeaderChecksumType digestAlgorithm = header.getLead().getChecksumType();
+ final MessageDigest digest = digestAlgorithm.digest();
+
+ digest.update(getLeadBytes(header.getLead()));
+ digest.update(getPrefaceBytes(header.getPreface()));
+ digest.update(getIndexBytes(header.getIndex()));
+ digest.update(getSignatureBytes(header.getSignatures()));
+
+ return digest.digest();
+ }
+
+ public static byte[] getHeaderWithoutChecksum(final ZChunkHeader header) {
+ final ZChunkHeaderLead lead = header.getLead();
+ final byte[] leadBytes = getLeadBytes(lead);
+
+ final ZChunkHeaderPreface preface = header.getPreface();
+ final byte[] prefaceBytes = getPrefaceBytes(preface);
+
+ final ZChunkHeaderIndex index = header.getIndex();
+ final byte[] indexBytes = getIndexBytes(index);
+
+ final byte[] sigs = getSignatureBytes(header.getSignatures());
+
+ return concat(leadBytes, prefaceBytes, indexBytes, sigs);
+ }
+
+ private static byte[] getSignatureBytes(final ZChunkHeaderSignatures signatures) {
+ return concat(
+ signatures.getSignatureCount().getCompressedBytes()
+ );
+ }
+
+ private static byte[] getIndexBytes(final ZChunkHeaderIndex index) {
+ final byte[] chunkInfos = getChunkInfoBytes(index.getChunkInfo());
+
+ final byte[] indexBytes = concat(
+ index.getIndexSize().getCompressedBytes(),
+ index.getChunkChecksumTypeInt().getCompressedBytes(),
+ index.getChunkCount().getCompressedBytes(),
+ index.getDictStream().orElse(new byte[0]),
+ index.getDictChecksum(),
+ index.getDictLength().getCompressedBytes(),
+ index.getUncompressedDictLength().getCompressedBytes(),
+ chunkInfos
+ );
+
+ return indexBytes;
+ }
+
+ private static byte[] getChunkInfoBytes(final List chunkInfos) {
+ final List collect = chunkInfos.stream()
+ .map(ChecksumUtil::getChunkInfoBytes)
+ .collect(toList());
+
+ return concatByteArrays(collect);
+ }
+
+ private static byte[] getChunkInfoBytes(final ZChunkHeaderChunkInfo info) {
+ return concat(
+ info.getChunkStream().orElse(new byte[0]),
+ info.getChunkChecksum(),
+ info.getChunkLength().getCompressedBytes(),
+ info.getChunkUncompressedLength().getCompressedBytes()
+ );
+ }
+
+
+ private static byte[] getPrefaceBytes(final ZChunkHeaderPreface preface) {
+ final byte[] prefaceBytes = concat(
+ preface.getTotalDataChecksum(),
+ preface.getPrefaceFlagsInt().getCompressedBytes(),
+ preface.getCompressionAlgorithm().getCompressionTypeValue().getCompressedBytes()
+ );
+
+ if (preface.hasOptionalElements()) {
+ final byte[] prefaceWithOptional = concat(
+ prefaceBytes,
+ preface.getOptionalElementCount().getCompressedBytes()
+ // TODO: optional elements header
+ );
+
+ return prefaceWithOptional;
+ }
+
+ return prefaceBytes;
+ }
+
+ private static byte[] getLeadBytes(final ZChunkHeaderLead lead) {
+ final byte[] leadBytes = concat(
+ lead.getId(),
+ lead.getChecksumTypeInt().getCompressedBytes(),
+ lead.getHeaderSize().getCompressedBytes()
+ );
+
+ return leadBytes;
+ }
+
+ private static byte[] concat(final byte[]... bytes) {
+ final List bytes1 = asList(bytes);
+
+ return concatByteArrays(bytes1);
+ }
+
+ private static byte[] concatByteArrays(final List byteList) {
+ final int totalLength = byteList.stream()
+ .mapToInt(by -> by.length)
+ .sum();
+
+ final AtomicInteger targetOffset = new AtomicInteger();
+ final byte[] target = new byte[totalLength];
+
+ byteList.forEach(by -> targetOffset.getAndUpdate(off -> {
+ System.arraycopy(by, 0, target, off, by.length);
+ return off + by.length;
+ }));
+ return target;
+ }
+
+
+}
diff --git a/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/util/OffsetUtil.java b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/util/OffsetUtil.java
new file mode 100644
index 0000000..0afe766
--- /dev/null
+++ b/fileformat/src/main/java/de/bmarwell/zchunk/fileformat/util/OffsetUtil.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat.util;
+
+import de.bmarwell.zchunk.fileformat.OptionalElement;
+import de.bmarwell.zchunk.fileformat.ZChunkHeaderLead;
+import de.bmarwell.zchunk.fileformat.ZChunkHeaderPreface;
+import java.math.BigInteger;
+
+public final class OffsetUtil {
+
+ private OffsetUtil() {
+ // util.
+ }
+
+ /**
+ * The length of the lead. After the lead, the prefix will begin (2nd part of the header).
+ *
+ * @return the lead length (id/magic length + checksum ci length + headersize ci length + cksum length).
+ */
+ public static int getLeadLength(final ZChunkHeaderLead lead) {
+ return BigInteger.valueOf(lead.getId().length)
+ .add(BigInteger.valueOf(lead.getChecksumTypeInt().getCompressedBytes().length))
+ .add(BigInteger.valueOf(lead.getHeaderSize().getCompressedBytes().length))
+ .add(BigInteger.valueOf(lead.getChecksum().length))
+ .intValueExact();
+ }
+
+ public static int getTotalHeaderSize(final ZChunkHeaderLead lead) {
+ return getLeadLength(lead) + lead.getHeaderSize().getIntValue();
+ }
+
+ public static long getPrefaceLength(final ZChunkHeaderPreface preface) {
+ /*
+ * optional element count only exists if the flag is set.
+ */
+ final long optElementCountBytes = getOptElementCountBytes(preface);
+
+ return BigInteger.valueOf(preface.getTotalDataChecksum().length)
+ // plus highest preface flag
+ .add(BigInteger.valueOf(preface.getPrefaceFlagsInt().getCompressedBytes().length))
+ // plus length of compression type
+ .add(BigInteger.valueOf(preface.getCompressionAlgorithm().getCompressionTypeValue().getCompressedBytes().length))
+ // this might even be 0 if the flag was not set.
+ .add(BigInteger.valueOf(optElementCountBytes))
+ .add(BigInteger.valueOf(
+ preface.getOptionalElements().stream()
+ .mapToLong(OptionalElement::getTotalLength)
+ .sum()
+ ))
+ .longValueExact();
+ }
+
+ private static long getOptElementCountBytes(final ZChunkHeaderPreface preface) {
+ if (preface.getOptionalElementCount().getLongValue() == 0L) {
+ return 0L;
+ }
+
+ return preface.getOptionalElementCount().getCompressedBytes().length;
+ }
+
+}
diff --git a/fileformat/src/test/java/de/bmarwell/zchunk/fileformat/ZChunkFileTest.java b/fileformat/src/test/java/de/bmarwell/zchunk/fileformat/ZChunkFileTest.java
new file mode 100644
index 0000000..1b0bc80
--- /dev/null
+++ b/fileformat/src/test/java/de/bmarwell/zchunk/fileformat/ZChunkFileTest.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import de.bmarwell.zchunk.fileformat.util.ByteUtils;
+import de.bmarwell.zchunk.fileformat.util.ChecksumUtil;
+import de.bmarwell.zchunk.fileformat.util.OffsetUtil;
+import java.io.File;
+import java.util.logging.Logger;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+@SuppressWarnings("SpellCheckingInspection")
+public class ZChunkFileTest {
+
+ private static final Logger LOG = Logger.getLogger(ZChunkFileTest.class.getCanonicalName());
+
+ private static final File TEST_FILE = new File(ZChunkFileTest.class.getResource("/testfiles/LICENSE.dict.fodt.zck").getPath());
+ private static final File TEST_FILE_INVALID = new File(
+ ZChunkFileTest.class.getResource("/testfiles/LICENSE.dict.fodt.zck.invalid").getPath());
+
+ @Test
+ public void testFileFormat() {
+ final ZChunkFile zChunkFile = ZChunkHeaderFactory.fromFile(TEST_FILE);
+
+ final ZChunkHeader header = zChunkFile.getHeader();
+
+ Assertions.assertAll(
+ () -> testLead(header.getLead()),
+ () -> testPreface(header.getPreface()),
+ () -> testIndex(header.getIndex()),
+ () -> checkChecksum(header)
+ );
+ }
+
+ private void checkChecksum(final ZChunkHeader header) {
+ final byte[] exp = header.getLead().getChecksum();
+ final byte[] actual = ChecksumUtil.calculateHeaderChecksum(header);
+
+ Assertions.assertAll(
+ () -> Assertions.assertArrayEquals(exp, actual),
+ () -> Assertions.assertTrue(ChecksumUtil.isValid(header))
+ );
+ }
+
+ @Test
+ public void testInvalidFile() {
+ Assertions.assertAll(
+ () -> Assertions.assertThrows(IllegalArgumentException.class, () -> ZChunkHeaderFactory.readFileHeaderLead(TEST_FILE_INVALID)),
+ () -> Assertions.assertThrows(IllegalArgumentException.class, () -> ZChunkHeaderFactory.fromFile(TEST_FILE_INVALID))
+ );
+ }
+
+ private void testLead(final ZChunkHeaderLead lead) {
+ final byte[] expectedCksum = ByteUtils.hexStringToByteArray("f666ca3e0330c42aa4bfbb58ab0576788d720cbbb1af0291f11e256b7bda2e79");
+
+ Assertions.assertAll(
+ () -> Assertions.assertArrayEquals(ZChunkConstants.Header.FILE_MAGIC, lead.getId()),
+ () -> Assertions.assertEquals(1, lead.getChecksumType().ordinal()),
+ () -> Assertions.assertEquals(394L, lead.getHeaderSize().getLongValue()),
+ () -> Assertions.assertEquals(394, lead.getHeaderSize().getIntValue()),
+ // TODO: 434 (reported by zck_read_header) vs 432 (this implementation).
+ () -> Assertions.assertEquals(434L, OffsetUtil.getTotalHeaderSize(lead)),
+ () -> Assertions.assertArrayEquals(expectedCksum, lead.getChecksum())
+ );
+ }
+
+ private void testPreface(final ZChunkHeaderPreface preface) {
+ final byte[] expectedDataCksum = ByteUtils.hexStringToByteArray("772aa76adc41e290dadff603b761ba02faad4681df05030539ae0ecd925ccd05");
+
+ Assertions.assertAll(
+ () -> Assertions.assertArrayEquals(expectedDataCksum, preface.getTotalDataChecksum()),
+ () -> Assertions.assertTrue(preface.getPrefaceFlags().isEmpty()),
+ () -> Assertions.assertEquals(2L, preface.getCompressionAlgorithm().getCompressionTypeValue().getLongValue()),
+ () -> Assertions.assertTrue(preface.getOptionalElements().isEmpty())
+ );
+ }
+
+ private void testIndex(final ZChunkHeaderIndex index) {
+ Assertions.assertAll(
+ () -> Assertions.assertEquals(357L, index.getIndexSize().getLongValue()),
+ () -> Assertions.assertEquals(IndexChecksumType.SHA512_128, index.getChunkChecksumType()),
+ () -> Assertions.assertEquals(3, index.getChunkChecksumType().ordinal()),
+ () -> Assertions.assertEquals(17L, index.getChunkCount().getLongValue())
+ );
+ }
+
+}
diff --git a/fileformat/src/test/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderFactoryTest.java b/fileformat/src/test/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderFactoryTest.java
new file mode 100644
index 0000000..54ae762
--- /dev/null
+++ b/fileformat/src/test/java/de/bmarwell/zchunk/fileformat/ZChunkHeaderFactoryTest.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat;
+
+import de.bmarwell.zchunk.compressedint.CompressedInt;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.Set;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class ZChunkHeaderFactoryTest {
+
+ @Test
+ public void testKnownPrefaceFlags() throws IOException {
+ final ByteArrayInputStream in = new ByteArrayInputStream(new byte[]{0b00000001});
+ final CompressedInt ci = ZChunkHeaderFactory.getPrefaceFlagsFromInputStream(in);
+ final Set prefaceFlags = PrefaceFlag.getPrefaceFlags(ci);
+
+ Assertions.assertAll(
+ () -> Assertions.assertEquals(1, prefaceFlags.size()),
+ () -> Assertions.assertEquals(PrefaceFlag.HAS_DATA_STREAMS, prefaceFlags.iterator().next())
+ );
+ }
+
+
+}
diff --git a/fileformat/src/test/java/de/bmarwell/zchunk/fileformat/util/ByteUtilTest.java b/fileformat/src/test/java/de/bmarwell/zchunk/fileformat/util/ByteUtilTest.java
new file mode 100644
index 0000000..cf7db1d
--- /dev/null
+++ b/fileformat/src/test/java/de/bmarwell/zchunk/fileformat/util/ByteUtilTest.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2019, the zchunk-java contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.bmarwell.zchunk.fileformat.util;
+
+import java.math.BigInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class ByteUtilTest {
+
+ @Test
+ public void testDecrease() {
+ final BigInteger testLong = new BigInteger(1, new byte[]{0b00001000});
+
+ final AtomicReference reference = new AtomicReference<>(testLong);
+ final boolean decrease = ByteUtils.decrease(reference, (long) 0b00001000);
+
+ Assertions.assertAll(
+ () -> Assertions.assertTrue(decrease),
+ () -> Assertions.assertEquals(0L, reference.get().longValueExact())
+ );
+ }
+
+ @Test
+ public void testDecrease_other() {
+ final BigInteger testLong = new BigInteger(1, new byte[]{0b00001000});
+
+ final AtomicReference reference = new AtomicReference<>(testLong);
+ final boolean decrease = ByteUtils.decrease(reference, (long) 0b00000001);
+
+ // nothing was decreased, the testlong should still have its old value.
+ Assertions.assertAll(
+ () -> Assertions.assertFalse(decrease),
+ () -> Assertions.assertEquals(testLong.longValueExact(), reference.get().longValueExact())
+ );
+ }
+}
diff --git a/fileformat/src/test/resources/logging.properties b/fileformat/src/test/resources/logging.properties
new file mode 100644
index 0000000..7230d09
--- /dev/null
+++ b/fileformat/src/test/resources/logging.properties
@@ -0,0 +1,26 @@
+#
+# Copyright 2019, the zchunk-java contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+handlers=java.util.logging.ConsoleHandler
+# default log level
+.level=FINE
+# console handler settings
+java.util.logging.ConsoleHandler.level=ALL
+java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
+java.util.logging.SimpleFormatter.format=[%1$tFT%1$tT.%1$tLZ] [%4$-6s] %2$s - %5$s%6$s%n
+# class specific settings
+de.bmarwell.zchunk.fileformat.level=FINER
+# 3rd party
+org.junit.platform.level=INFO
diff --git a/fileformat/src/test/resources/testfiles/LICENSE.dict.fodt.zck b/fileformat/src/test/resources/testfiles/LICENSE.dict.fodt.zck
new file mode 100644
index 0000000..d82fc4c
Binary files /dev/null and b/fileformat/src/test/resources/testfiles/LICENSE.dict.fodt.zck differ
diff --git a/fileformat/src/test/resources/testfiles/LICENSE.dict.fodt.zck.invalid b/fileformat/src/test/resources/testfiles/LICENSE.dict.fodt.zck.invalid
new file mode 100644
index 0000000..d1f328c
Binary files /dev/null and b/fileformat/src/test/resources/testfiles/LICENSE.dict.fodt.zck.invalid differ
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..076c5f5
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,114 @@
+
+
+
+
+ 4.0.0
+
+ de.bmarwell.zchunk
+ zchunk-parent
+ 1.0.0-SNAPSHOT
+
+ pom
+
+
+ compressedint
+
+ compression/compression-api
+
+ fileformat
+
+
+
+ UTF-8
+
+
+ 2.7.5
+
+ 5.4.0
+
+
+
+
+
+
+ org.immutables
+ value
+ ${dependency.immutables.version}
+ provided
+
+
+
+
+ org.checkerframework
+ checker-qual
+ 2.8.1
+ provided
+
+
+
+
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ ${dependency.junit-jupiter.version}
+ test
+
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ ${dependency.junit-jupiter.version}
+ test
+
+
+
+
+
+
+
+
+ maven-compiler-plugin
+ 3.8.0
+
+ 1.8
+ 1.8
+ 1.8
+
+
+
+
+
+
+ maven-surefire-plugin
+ 3.0.0-M3
+
+
+ ${project.build.testOutputDirectory}/logging.properties
+ EN
+
+
+
+
+
+
+
+