From a76119bb6564faf8562e4a9e0a3030d4c5f957f4 Mon Sep 17 00:00:00 2001 From: Benjamin Marwell Date: Sat, 1 Jun 2019 09:27:58 +0200 Subject: [PATCH] Basic unzck owrks. - Readme updates. --- README.md | 10 ++- .../io/github/zchunk/app/ZChunkFilename.java | 9 +++ .../io/github/zchunk/app/commands/Unzck.java | 66 +++++++++++++++++-- .../github/zchunk/app/commands/UnzckTest.java | 26 ++++++++ .../zchunk/fileformat/ZChunkHeaderIndex.java | 6 +- 5 files changed, 108 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index efcdf88..ad157eb 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,21 @@ downloaded is, in fact, the file you wanted. ## What is the goal of this project? +The goal is to create a pure java implementation of the zchunk file format. +This way, java programs or Android apps will be able to use the zchunk file format. -## What dependencies do I need? +## Which dependencies do I need? **zchunk-java** can be used *without any* transitives dependencies/libraries. -However, to have support for compression, you should use the arteifact `zchunk-all` (TBD), which also +However, to have support for compression, you should use the arteifact `zchunk-bundle-lib` (TBD), which also pulls in support for `zstd` compression and maybe other compressions later. +The command line application (`zchunk-app`) is another matter. It uses `picocli` for parsing command lines, +but other than that, it does not have any new dependencies. Since `picocli` is bundled with the app in an +executable `one-jar`, there is no manual copying of dependencies. + ## Runtime Requirements and usage diff --git a/app/src/main/java/io/github/zchunk/app/ZChunkFilename.java b/app/src/main/java/io/github/zchunk/app/ZChunkFilename.java index 4990ce9..99b8319 100644 --- a/app/src/main/java/io/github/zchunk/app/ZChunkFilename.java +++ b/app/src/main/java/io/github/zchunk/app/ZChunkFilename.java @@ -41,4 +41,13 @@ public static File getDictFile(final File zchunkFile) { return new File(newName); } + public static File getNormalFile(final File zchunkFile) { + if (!zchunkFile.getName().endsWith(".zck")) { + throw new UnsupportedOperationException("Cannot acquire target file name"); + } + + final String newName = zchunkFile.getName().replaceAll("^(.*)\\.zck$", "$1"); + + return new File(newName); + } } diff --git a/app/src/main/java/io/github/zchunk/app/commands/Unzck.java b/app/src/main/java/io/github/zchunk/app/commands/Unzck.java index d949146..51ede6a 100644 --- a/app/src/main/java/io/github/zchunk/app/commands/Unzck.java +++ b/app/src/main/java/io/github/zchunk/app/commands/Unzck.java @@ -20,14 +20,19 @@ import io.github.zchunk.app.err.UncompressException; import io.github.zchunk.fileformat.ZChunk; import io.github.zchunk.fileformat.ZChunkFile; +import io.github.zchunk.fileformat.ZChunkHeader; +import io.github.zchunk.fileformat.ZChunkHeaderChunkInfo; +import io.github.zchunk.fileformat.ZChunkHeaderIndex; import io.github.zchunk.fileformat.util.IOUtil; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.util.SortedSet; import java.util.StringJoiner; import java.util.concurrent.Callable; +import java.util.logging.Logger; import org.checkerframework.checker.nullness.qual.Nullable; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -38,6 +43,8 @@ mixinStandardHelpOptions = true) public class Unzck implements Callable { + private static final Logger LOG = Logger.getLogger(Unzck.class.getCanonicalName()); + @Option(names = {"-c", "--stdout"}) private boolean toStdOut; @@ -54,16 +61,64 @@ public class Unzck implements Callable { @Override public Integer call() { final ZChunkFile zChunkFile = ZChunk.fromFile(this.inputFile); + if (this.dictOnly) { return decompressDict(zChunkFile); } - return -1; + return decompressFile(zChunkFile); + } + + private int decompressFile(final ZChunkFile zChunkFile) { + final File target = getTargetFile(); + + try (final FileOutputStream fileOutputStream = new FileOutputStream(target)) { + final File targetDir = target.getAbsoluteFile().getParentFile(); + if (null == targetDir) { + throw new IllegalStateException("TargetDir Parent is null: [" + target.getAbsolutePath() + "]."); + } + targetDir.mkdirs(); + target.createNewFile(); + + final ZChunkHeader zChunkFileHeader = zChunkFile.getHeader(); + final ZChunkHeaderIndex zChunkHeaderIndex = zChunkFileHeader.getIndex(); + if (zChunkHeaderIndex.getDictLength().getIntValue() == 0) { + throw new UnsupportedOperationException("TODO: uncompress without dict"); + } + + final byte[] decompressedDict = ZChunk.getDecompressedDict(zChunkFileHeader, this.inputFile); + + uncompressChunks(fileOutputStream, zChunkFileHeader, decompressedDict); + + } catch (final FileNotFoundException fnfe) { + throw new UncompressException("Unable to create parent dir or file: [" + target.getAbsolutePath() + "].", fnfe); + } catch (final IOException ex) { + throw new UncompressException("Unable to write file: [" + target.getAbsolutePath() + "].", ex); + } + + return 0; + } + + private void uncompressChunks(final FileOutputStream fileOutputStream, + final ZChunkHeader zChunkFileHeader, + final byte[] decompressedDict) throws IOException { + final SortedSet chunks = zChunkFileHeader.getIndex().getChunkInfoSortedByIndex(); + + // TODO: This can be optimized using random access file and parallel writing. + for (final ZChunkHeaderChunkInfo chunk : chunks) { + LOG.finest("Working on chunk [" + chunk + "]."); + final InputStream decompressedChunk = ZChunk.getDecompressedChunk( + zChunkFileHeader, + this.inputFile, + decompressedDict, + chunk.getCurrentIndex()); + IOUtil.copy(decompressedChunk, fileOutputStream); + } } private int decompressDict(final ZChunkFile zChunkFile) { final File target = getTargetFile(); - try { + try (final FileOutputStream fileOutputStream = new FileOutputStream(target)) { final File targetDir = target.getAbsoluteFile().getParentFile(); if (null == targetDir) { throw new IllegalStateException("TargetDir Parent is null: [" + target.getAbsolutePath() + "]."); @@ -71,7 +126,6 @@ private int decompressDict(final ZChunkFile zChunkFile) { targetDir.mkdirs(); target.createNewFile(); final InputStream decompressedDictStream = ZChunk.getDecompressedDictStream(zChunkFile.getHeader(), this.inputFile); - final FileOutputStream fileOutputStream = new FileOutputStream(target); final int copied = IOUtil.copy(decompressedDictStream, fileOutputStream); } catch (final FileNotFoundException fnfe) { @@ -88,7 +142,11 @@ private File getTargetFile() { return this.outputFile; } - return ZChunkFilename.getDictFile(this.inputFile); + if (this.dictOnly) { + return ZChunkFilename.getDictFile(this.inputFile); + } + + return ZChunkFilename.getNormalFile(this.inputFile); } public boolean isToStdOut() { diff --git a/app/src/test/java/io/github/zchunk/app/commands/UnzckTest.java b/app/src/test/java/io/github/zchunk/app/commands/UnzckTest.java index 4059cbe..a20cc90 100644 --- a/app/src/test/java/io/github/zchunk/app/commands/UnzckTest.java +++ b/app/src/test/java/io/github/zchunk/app/commands/UnzckTest.java @@ -53,7 +53,33 @@ public void testUnzckDict() throws NoSuchAlgorithmException, IOException { final byte[] bytes = ChecksumUtil.calculateFileChecksum(targetFile, md5); final String foundMd5 = new BigInteger(1, bytes).toString(16); + // gotten by running the original unzck and then md5sum. Assertions.assertEquals("e051cbdf211c13bead2009e49b3317f5", foundMd5); } + @Test + public void testUnzckFile() throws NoSuchAlgorithmException, IOException { + final ClassLoader classLoader = getClass().getClassLoader(); + final String pathToFiles = classLoader.getResource("files").getFile(); + + final File input = new File(pathToFiles, "LICENSE.dict.fodt.zck"); + Assertions.assertTrue(input.exists()); + Assertions.assertTrue(input.isFile()); + Assertions.assertTrue(input.canRead()); + + LOG.finer("File found: " + input.getAbsolutePath()); + final File targetFile = new File(pathToFiles, "LICENSE.dict.fodt"); + final Unzck unzck = new Unzck(); + unzck.setInputFile(input); + unzck.setOutputFile(targetFile); + unzck.call(); + + final MessageDigest md5 = MessageDigest.getInstance("md5"); + final byte[] bytes = ChecksumUtil.calculateFileChecksum(targetFile, md5); + final String foundMd5 = new BigInteger(1, bytes).toString(16); + + // gotten by running the original unzck and then md5sum. + Assertions.assertEquals("92236dfc074fa2db49a6345f71b51b9e", foundMd5); + } + } diff --git a/fileformat/src/main/java/io/github/zchunk/fileformat/ZChunkHeaderIndex.java b/fileformat/src/main/java/io/github/zchunk/fileformat/ZChunkHeaderIndex.java index bc787a8..afadc8f 100644 --- a/fileformat/src/main/java/io/github/zchunk/fileformat/ZChunkHeaderIndex.java +++ b/fileformat/src/main/java/io/github/zchunk/fileformat/ZChunkHeaderIndex.java @@ -20,7 +20,7 @@ import io.github.zchunk.fileformat.util.ByteUtils; import java.util.Map; import java.util.Optional; -import java.util.Set; +import java.util.SortedSet; import java.util.StringJoiner; import java.util.TreeSet; import java.util.function.Supplier; @@ -89,8 +89,8 @@ public IndexChecksumType getChunkChecksumType() { public abstract Map getChunkInfo(); @Value.Lazy - public Set getChunkInfoSortedByIndex() { - final Supplier> IndexSortedList = () -> new TreeSet<>(ZChunkHeaderChunkInfo.INDEX_COMPARATOR); + public SortedSet getChunkInfoSortedByIndex() { + final Supplier> IndexSortedList = () -> new TreeSet<>(ZChunkHeaderChunkInfo.INDEX_COMPARATOR); return getChunkInfo().values() .stream()