diff --git a/.travis.yml b/.travis.yml index dc1ea51..036b4d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,6 @@ install: stages: - validations - test - - java11 jobs: include: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 91e35e8..dc3ec89 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,7 @@ Before creating pull requests, please tick these checkboxes in your mind: - [ ] If this is a PR for a issue, please name it issues/# (e.g. issues/3). - [ ] Before commiting and creating the PR, please execute: - [ ] ./mvnw clean install -DskipTests=true -Pcheckstyle,checker - - [ ] ./mvnw -T4 clean install javadoc:jar sources:jar + - [ ] ./mvnw -T4 clean install javadoc:jar source:jar If it doesn't compile, please fix the errors first. Thank you :) diff --git a/app/pom.xml b/app/pom.xml new file mode 100644 index 0000000..d3cc7fc --- /dev/null +++ b/app/pom.xml @@ -0,0 +1,48 @@ + + + + zchunk-parent + io.github.zchunk + 1.0.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + zchunk-app + + + + io.github.zchunk + zchunk-bundle-lib + 1.0.0-SNAPSHOT + + + + + info.picocli + picocli + + + + org.checkerframework + checker-qual + provided + + + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + + + diff --git a/app/src/main/java/io/github/zchunk/app/ZChunk.java b/app/src/main/java/io/github/zchunk/app/ZChunk.java new file mode 100644 index 0000000..8b8feb1 --- /dev/null +++ b/app/src/main/java/io/github/zchunk/app/ZChunk.java @@ -0,0 +1,100 @@ +/* + * Copyright 2018 The zchunk-java contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.zchunk.app; + +import io.github.zchunk.app.commands.Unzck; +import java.util.concurrent.Callable; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ParameterException; +import picocli.CommandLine.ParseResult; +import picocli.CommandLine.UnmatchedArgumentException; + +@Command( + name = "ZChunk", + subcommands = { + Unzck.class + }, + version = "1.0" +) +public class ZChunk implements Callable { + + @Option(names = {"-v", "--verbose"}, description = "Be verbose.") + private boolean verbose; + + @Option(names = {"-h", "-?", "--help", "--usage"}, usageHelp = true, description = "display a help message") + private boolean helpRequested; + + @Option(names = {"-V", "--version"}, versionHelp = true) + private boolean showVersion; + + + + /* + Usage: unzck [OPTION...] + unzck - Decompress a zchunk file + + -c, --stdout Direct output to stdout + --dict Only extract the dictionary + -v, --verbose Increase verbosity (can be specified more than + once for debugging) + -?, --help Give this help list + --usage Give a short usage message + -V, --version Show program version + */ + + public static int main(final String[] args) { + final CommandLine cmd = new CommandLine(new ZChunk()); + + try { + final ParseResult parseResult = cmd.parseArgs(args); + + if (cmd.isUsageHelpRequested()) { + cmd.usage(cmd.getOut()); + return cmd.getCommandSpec().exitCodeOnUsageHelp(); + } + + if (cmd.isVersionHelpRequested()) { + cmd.printVersionHelp(cmd.getOut()); + return cmd.getCommandSpec().exitCodeOnVersionHelp(); + } + + } catch (final ParameterException ex) { + cmd.getErr().println(ex.getMessage()); + if (!UnmatchedArgumentException.printSuggestions(ex, cmd.getErr())) { + ex.getCommandLine().usage(cmd.getErr()); + } + return cmd.getCommandSpec().exitCodeOnInvalidInput(); + } + + try { + return cmd.execute(args); + } catch (final RuntimeException ex) { + // exception occurred in business logic + ex.printStackTrace(cmd.getErr()); + return cmd.getCommandSpec().exitCodeOnExecutionException(); + } + + } + + @Override + public Integer call() throws Exception { + + return -1; + } +} diff --git a/app/src/main/java/io/github/zchunk/app/ZChunkFilename.java b/app/src/main/java/io/github/zchunk/app/ZChunkFilename.java new file mode 100644 index 0000000..4990ce9 --- /dev/null +++ b/app/src/main/java/io/github/zchunk/app/ZChunkFilename.java @@ -0,0 +1,44 @@ +/* + * Copyright 2018 The zchunk-java contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.zchunk.app; + +import java.io.File; + +public final class ZChunkFilename { + + private ZChunkFilename() { + // util class + } + + /** + * Returns a new file for the current directory replacing .zck with .zdict. + * + * @param zchunkFile + * the input file. + * @return the file name without directory. + */ + public static File getDictFile(final File zchunkFile) { + if (!zchunkFile.getName().endsWith(".zck")) { + throw new UnsupportedOperationException("Cannot acquire target file name"); + } + + final String newName = zchunkFile.getName().replaceAll("^(.*)\\.zck$", "$1.zdict"); + + return new File(newName); + } + +} diff --git a/app/src/main/java/io/github/zchunk/app/commands/Unzck.java b/app/src/main/java/io/github/zchunk/app/commands/Unzck.java new file mode 100644 index 0000000..d949146 --- /dev/null +++ b/app/src/main/java/io/github/zchunk/app/commands/Unzck.java @@ -0,0 +1,136 @@ +/* + * Copyright 2018 The zchunk-java contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.zchunk.app.commands; + +import io.github.zchunk.app.ZChunkFilename; +import io.github.zchunk.app.err.UncompressException; +import io.github.zchunk.fileformat.ZChunk; +import io.github.zchunk.fileformat.ZChunkFile; +import io.github.zchunk.fileformat.util.IOUtil; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.StringJoiner; +import java.util.concurrent.Callable; +import org.checkerframework.checker.nullness.qual.Nullable; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +@Command(description = "Unpacks the completely downloaded zck file.", + name = "unzck", + mixinStandardHelpOptions = true) +public class Unzck implements Callable { + + @Option(names = {"-c", "--stdout"}) + private boolean toStdOut; + + @Option(names = {"--dict"}) + private boolean dictOnly; + + @Option(names = {"-o"}) + private @Nullable File outputFile; + + @Parameters(arity = "1", paramLabel = "FILE") + @SuppressWarnings(value = {"initialization.fields.uninitialized", "dereference.of.nullable"}) + private File inputFile; + + @Override + public Integer call() { + final ZChunkFile zChunkFile = ZChunk.fromFile(this.inputFile); + if (this.dictOnly) { + return decompressDict(zChunkFile); + } + + return -1; + } + + private int decompressDict(final ZChunkFile zChunkFile) { + final File target = getTargetFile(); + try { + final File targetDir = target.getAbsoluteFile().getParentFile(); + if (null == targetDir) { + throw new IllegalStateException("TargetDir Parent is null: [" + target.getAbsolutePath() + "]."); + } + targetDir.mkdirs(); + target.createNewFile(); + final InputStream decompressedDictStream = ZChunk.getDecompressedDictStream(zChunkFile.getHeader(), this.inputFile); + final FileOutputStream fileOutputStream = new FileOutputStream(target); + final int copied = IOUtil.copy(decompressedDictStream, fileOutputStream); + + } catch (final FileNotFoundException fnfe) { + throw new UncompressException("Unable to create parent dir or file: [" + target.getAbsolutePath() + "].", fnfe); + } catch (final IOException ex) { + throw new UncompressException("Unable to write file: [" + target.getAbsolutePath() + "].", ex); + } + + return 0; + } + + private File getTargetFile() { + if (null != this.outputFile) { + return this.outputFile; + } + + return ZChunkFilename.getDictFile(this.inputFile); + } + + public boolean isToStdOut() { + return this.toStdOut; + } + + public void setToStdOut(final boolean toStdOut) { + this.toStdOut = toStdOut; + } + + public boolean isDictOnly() { + return this.dictOnly; + } + + public void setDictOnly(final boolean dictOnly) { + this.dictOnly = dictOnly; + } + + public @Nullable File getOutputFile() { + return this.outputFile; + } + + public void setOutputFile(final @Nullable File outputFile) { + this.outputFile = outputFile; + } + + public File getInputFile() { + return this.inputFile; + } + + public void setInputFile(final File inputFile) { + this.inputFile = inputFile; + } + + + @Override + public String toString() { + return new StringJoiner(", ", Unzck.class.getSimpleName() + "[", "]") + .add("toStdOut=" + this.toStdOut) + .add("dictOnly=" + this.dictOnly) + .add("outputFile=" + this.outputFile) + .add("inputFile=" + this.inputFile) + .toString(); + } +} diff --git a/app/src/main/java/io/github/zchunk/app/err/UncompressException.java b/app/src/main/java/io/github/zchunk/app/err/UncompressException.java new file mode 100644 index 0000000..256a8ee --- /dev/null +++ b/app/src/main/java/io/github/zchunk/app/err/UncompressException.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 The zchunk-java contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.zchunk.app.err; + +import java.util.StringJoiner; + +public class UncompressException extends RuntimeException { + + private static final long serialVersionUID = -6570432692413525167L; + + public UncompressException(final String message, final Throwable cause) { + super(message, cause); + } + + @Override + public String toString() { + return new StringJoiner(", ", UncompressException.class.getSimpleName() + "[", "]") + .add("super='" + super.toString() + "'") + .toString(); + } +} diff --git a/app/src/test/java/io/github/zchunk/app/ZChunkTest.java b/app/src/test/java/io/github/zchunk/app/ZChunkTest.java new file mode 100644 index 0000000..1981419 --- /dev/null +++ b/app/src/test/java/io/github/zchunk/app/ZChunkTest.java @@ -0,0 +1,48 @@ +/* + * Copyright 2018 The zchunk-java contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.zchunk.app; + +import org.junit.jupiter.api.Test; + +public class ZChunkTest { + + @Test + public void testHelp() { + ZChunk.main(new String[]{"-h"}); + } + + @Test + public void testHelpUnpack() { + ZChunk.main(new String[]{"-h", "unzck"}); + } + + @Test + public void testUnpackHelp() { + ZChunk.main(new String[]{"unzck", "-h"}); + } + + @Test + public void testNothing() { + ZChunk.main(new String[]{}); + } + + @Test + public void testUnzck() { + ZChunk.main(new String[]{"unzck"}); + } + +} diff --git a/app/src/test/java/io/github/zchunk/app/commands/UnzckTest.java b/app/src/test/java/io/github/zchunk/app/commands/UnzckTest.java new file mode 100644 index 0000000..4059cbe --- /dev/null +++ b/app/src/test/java/io/github/zchunk/app/commands/UnzckTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 The zchunk-java contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.zchunk.app.commands; + +import io.github.zchunk.fileformat.util.ChecksumUtil; +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.logging.Logger; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class UnzckTest { + + private static final Logger LOG = Logger.getLogger(UnzckTest.class.getCanonicalName()); + + @Test + public void testUnzckDict() throws NoSuchAlgorithmException, IOException { + final ClassLoader classLoader = getClass().getClassLoader(); + final String pathToFiles = classLoader.getResource("files").getFile(); + + final File input = new File(pathToFiles, "LICENSE.dict.fodt.zck"); + Assertions.assertTrue(input.exists()); + Assertions.assertTrue(input.isFile()); + Assertions.assertTrue(input.canRead()); + + LOG.finer("File found: " + input.getAbsolutePath()); + final File targetFile = new File(pathToFiles, "LICENSE.dict.fodt.zdict"); + + final Unzck unzck = new Unzck(); + unzck.setInputFile(input); + unzck.setOutputFile(targetFile); + unzck.setDictOnly(true); + unzck.call(); + + final MessageDigest md5 = MessageDigest.getInstance("md5"); + final byte[] bytes = ChecksumUtil.calculateFileChecksum(targetFile, md5); + final String foundMd5 = new BigInteger(1, bytes).toString(16); + + Assertions.assertEquals("e051cbdf211c13bead2009e49b3317f5", foundMd5); + } + +} diff --git a/app/src/test/resources/files/LICENSE.dict.fodt.zck b/app/src/test/resources/files/LICENSE.dict.fodt.zck new file mode 100644 index 0000000..d82fc4c Binary files /dev/null and b/app/src/test/resources/files/LICENSE.dict.fodt.zck differ diff --git a/app/src/test/resources/logging.properties b/app/src/test/resources/logging.properties new file mode 100644 index 0000000..3519e09 --- /dev/null +++ b/app/src/test/resources/logging.properties @@ -0,0 +1,26 @@ +# +# Copyright 2019, the zchunk-java contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +handlers=java.util.logging.ConsoleHandler +# default log level +.level=FINE +# console handler settings +java.util.logging.ConsoleHandler.level=ALL +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=[%1$tFT%1$tT.%1$tLZ] [%4$-6s] %2$s - %5$s%6$s%n +# class specific settings +io.github.zchunk.app.level=FINER +# 3rd party +org.junit.platform.level=INFO diff --git a/compression/compression-api/src/main/java/io/github/zchunk/compression/api/CompressionAlgorithmFactory.java b/compression/compression-api/src/main/java/io/github/zchunk/compression/api/CompressionAlgorithmFactory.java index 74daf00..9bd4b3e 100644 --- a/compression/compression-api/src/main/java/io/github/zchunk/compression/api/CompressionAlgorithmFactory.java +++ b/compression/compression-api/src/main/java/io/github/zchunk/compression/api/CompressionAlgorithmFactory.java @@ -109,10 +109,10 @@ public List loadImplementations(@UnderInitialization Resou final ServiceLoader load = ServiceLoader.load(CompressionAlgorithm.class); final Logger logger = Logger.getLogger("io.github.zchunk"); - logger.info("ServiceLoader: " + load); + logger.finest("ServiceLoader: " + load); return StreamSupport.stream(load.spliterator(), false) - .peek(clazz2 -> logger.info("Class: " + clazz2.getClass())) + .peek(clazz2 -> logger.finer("Class: " + clazz2.getClass())) .collect(Collectors.toList()); } diff --git a/fileformat/src/main/java/io/github/zchunk/fileformat/ZChunk.java b/fileformat/src/main/java/io/github/zchunk/fileformat/ZChunk.java index 12efe3d..48b670d 100644 --- a/fileformat/src/main/java/io/github/zchunk/fileformat/ZChunk.java +++ b/fileformat/src/main/java/io/github/zchunk/fileformat/ZChunk.java @@ -96,7 +96,23 @@ public static byte[] getDecompressedDict(final ZChunkHeader header, final File i final String message = String.format("Unable to read dictionary at offset [%d] from file [%s].", offset, input.getAbsolutePath()); throw new IllegalArgumentException(message); } + } + + public static InputStream getDecompressedDictStream(final ZChunkHeader header, final File input) { + final long offset = OffsetUtil.getDictOffset(header); + final CompressionAlgorithm compressionAlgorithm = header.getPreface().getCompressionAlgorithm(); + final BiFunction decompressor = compressionAlgorithm.getOutputStreamSupplier(); + try { + final FileInputStream fis = new FileInputStream(input); + final InputStream decompressedStream = decompressor.apply(fis, new byte[0]); + fis.skip(offset); + + return new BoundedInputStream(decompressedStream, header.getIndex().getUncompressedDictLength().getIntValue()); + } catch (final IOException ioEx) { + final String message = String.format("Unable to read dictionary at offset [%d] from file [%s].", offset, input.getAbsolutePath()); + throw new IllegalArgumentException(message); + } } public static InputStream getDecompressedChunk(final ZChunkHeader header, diff --git a/fileformat/src/main/java/io/github/zchunk/fileformat/util/ChecksumUtil.java b/fileformat/src/main/java/io/github/zchunk/fileformat/util/ChecksumUtil.java index c9190ac..6c2b8e7 100644 --- a/fileformat/src/main/java/io/github/zchunk/fileformat/util/ChecksumUtil.java +++ b/fileformat/src/main/java/io/github/zchunk/fileformat/util/ChecksumUtil.java @@ -230,4 +230,18 @@ private static boolean chunkIsValid(final ZChunkHeaderChunkInfo chunk, final ZCh return false; } } + + public static byte[] calculateFileChecksum(final File input, final MessageDigest digest) throws IOException { + final byte[] buffer = new byte[BUFFER_SIZE]; + + try (final FileInputStream fis = new FileInputStream(input)) { + int readCount; + + while ((readCount = fis.read(buffer)) != -1) { + digest.update(buffer, 0, readCount); + } + + return digest.digest(); + } + } } diff --git a/fileformat/src/main/java/io/github/zchunk/fileformat/util/IOUtil.java b/fileformat/src/main/java/io/github/zchunk/fileformat/util/IOUtil.java new file mode 100644 index 0000000..e6d81b9 --- /dev/null +++ b/fileformat/src/main/java/io/github/zchunk/fileformat/util/IOUtil.java @@ -0,0 +1,50 @@ +/* + * Copyright 2018 The zchunk-java contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.zchunk.fileformat.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public final class IOUtil { + + private static final int BUFFER_SIZE = 1024; + private static final int EOF = -1; + + private IOUtil() { + // util class + } + + public static int copy(final InputStream in, final OutputStream out) throws IOException { + final byte[] buffer = new byte[BUFFER_SIZE]; + int readCount; + int totalWritten = 0; + + while ((readCount = in.read(buffer)) != EOF) { + out.write(buffer, 0, readCount); + totalWritten += readCount; + + if (readCount < BUFFER_SIZE) { + // end reached. + break; + } + } + + return totalWritten; + } + +} diff --git a/pom.xml b/pom.xml index 1050c67..956852f 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ compression/compression-zstd fileformat - + app bundle/lib @@ -72,6 +72,13 @@ provided + + + info.picocli + picocli + 4.0.0-alpha-3 + +