From df1d7013b437c69132c6984f213229a423ef0e2a Mon Sep 17 00:00:00 2001 From: Joergen Ibsen Date: Sun, 20 Sep 2015 10:47:16 +0200 Subject: [PATCH 1/5] Add BriefLZ plugin --- .gitmodules | 3 + plugins/CMakeLists.txt | 1 + plugins/brieflz/CMakeLists.txt | 7 ++ plugins/brieflz/brieflz | 1 + plugins/brieflz/brieflz.md | 17 +++++ plugins/brieflz/squash-brieflz.c | 127 +++++++++++++++++++++++++++++++ plugins/brieflz/squash.ini | 3 + 7 files changed, 159 insertions(+) create mode 100644 plugins/brieflz/CMakeLists.txt create mode 160000 plugins/brieflz/brieflz create mode 100644 plugins/brieflz/brieflz.md create mode 100644 plugins/brieflz/squash-brieflz.c create mode 100644 plugins/brieflz/squash.ini diff --git a/.gitmodules b/.gitmodules index 05c70f5..a5f1dd8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -80,3 +80,6 @@ path = plugins/zlib-ng/zlib-ng url = https://github.com/Dead2/zlib-ng.git branch = develop +[submodule "plugins/brieflz/brieflz"] + path = plugins/brieflz/brieflz + url = https://github.com/jibsen/brieflz.git diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index ceea231..cc49ca7 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -1,4 +1,5 @@ set (plugins_available + brieflz brotli bsc bzip2 diff --git a/plugins/brieflz/CMakeLists.txt b/plugins/brieflz/CMakeLists.txt new file mode 100644 index 0000000..17aeb8c --- /dev/null +++ b/plugins/brieflz/CMakeLists.txt @@ -0,0 +1,7 @@ +set (brieflz_sources + squash-brieflz.c + brieflz/brieflz.c + brieflz/depack.c) + +squash_plugin_add (brieflz brieflz_sources) +squash_plugin_add_include_directories (brieflz brieflz) diff --git a/plugins/brieflz/brieflz b/plugins/brieflz/brieflz new file mode 160000 index 0000000..bcaa6a1 --- /dev/null +++ b/plugins/brieflz/brieflz @@ -0,0 +1 @@ +Subproject commit bcaa6a1ee7ccf005512b5c23aa92b40cf75f9ed1 diff --git a/plugins/brieflz/brieflz.md b/plugins/brieflz/brieflz.md new file mode 100644 index 0000000..0665459 --- /dev/null +++ b/plugins/brieflz/brieflz.md @@ -0,0 +1,17 @@ +# BriefLZ Plugin # + +BriefLZ is a small, fast Lempel-Ziv compression library. + +For more information about BriefLZ, see +https://github.com/jibsen/brieflz + +## Codecs ## + +- **brieflz** — Raw BriefLZ data. + +## License ## + +The BriefLZ plugin is licensed under the [MIT +License](http://opensource.org/licenses/MIT), and BriefLZ +is licensed under the [zlib +license](http://opensource.org/licenses/Zlib). diff --git a/plugins/brieflz/squash-brieflz.c b/plugins/brieflz/squash-brieflz.c new file mode 100644 index 0000000..43e6df8 --- /dev/null +++ b/plugins/brieflz/squash-brieflz.c @@ -0,0 +1,127 @@ +/* Copyright (c) 2015 The Squash Authors + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Authors: + * Evan Nemerson + * Joergen Ibsen + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include + +SQUASH_PLUGIN_EXPORT +SquashStatus squash_plugin_init_codec (SquashCodec* codec, SquashCodecImpl* impl); + +static size_t +squash_brieflz_get_max_compressed_size (SquashCodec* codec, size_t uncompressed_length) { + return (size_t) blz_max_packed_size ((unsigned long) uncompressed_length) + 4; +} + +static SquashStatus +squash_brieflz_decompress_buffer (SquashCodec* codec, + size_t* decompressed_length, + uint8_t decompressed[SQUASH_ARRAY_PARAM(*decompressed_length)], + size_t compressed_length, + const uint8_t compressed[SQUASH_ARRAY_PARAM(compressed_length)], + SquashOptions* options) { + unsigned long original_size; + unsigned long size; + + original_size = (unsigned long) compressed[0] + | ((unsigned long) compressed[1] << 8) + | ((unsigned long) compressed[2] << 16) + | ((unsigned long) compressed[3] << 24); + + if (original_size > *decompressed_length) { + return squash_error (SQUASH_BUFFER_FULL); + } + + size = blz_depack (compressed + 4, decompressed, original_size); + + if (size != original_size) { + return squash_error (SQUASH_FAILED); + } + + *decompressed_length = (size_t) size; + + return SQUASH_OK; +} + +static SquashStatus +squash_brieflz_compress_buffer (SquashCodec* codec, + size_t* compressed_length, + uint8_t compressed[SQUASH_ARRAY_PARAM(*compressed_length)], + size_t uncompressed_length, + const uint8_t uncompressed[SQUASH_ARRAY_PARAM(uncompressed_length)], + SquashOptions* options) { + unsigned long size; + void *workmem = NULL; + + if ((unsigned long) *compressed_length + < squash_brieflz_get_max_compressed_size (codec, uncompressed_length)) { + return squash_error (SQUASH_BUFFER_FULL); + } + + workmem = malloc (blz_workmem_size ((unsigned long) uncompressed_length)); + + if (workmem == NULL) { + return squash_error (SQUASH_MEMORY); + } + + compressed[0] = (uint8_t) uncompressed_length; + compressed[1] = (uint8_t) (uncompressed_length >> 8); + compressed[2] = (uint8_t) (uncompressed_length >> 16); + compressed[3] = (uint8_t) (uncompressed_length >> 24); + + size = blz_pack (uncompressed, compressed + 4, + (unsigned long) uncompressed_length, + workmem); + + free(workmem); + + *compressed_length = (size_t) size + 4; + + return SQUASH_OK; +} + +SquashStatus +squash_plugin_init_codec (SquashCodec* codec, SquashCodecImpl* impl) { + const char* name = squash_codec_get_name (codec); + + if (strcmp ("brieflz", name) == 0) { + impl->get_max_compressed_size = squash_brieflz_get_max_compressed_size; + impl->decompress_buffer = squash_brieflz_decompress_buffer; + impl->compress_buffer_unsafe = squash_brieflz_compress_buffer; + } else { + return squash_error (SQUASH_UNABLE_TO_LOAD); + } + + return SQUASH_OK; +} diff --git a/plugins/brieflz/squash.ini b/plugins/brieflz/squash.ini new file mode 100644 index 0000000..e1d5c44 --- /dev/null +++ b/plugins/brieflz/squash.ini @@ -0,0 +1,3 @@ +license=zlib + +[brieflz] From b0fe174870e85169cbaf65e648f85442d8150f19 Mon Sep 17 00:00:00 2001 From: Joergen Ibsen Date: Sun, 20 Sep 2015 15:16:52 +0200 Subject: [PATCH 2/5] Check compressed size before reading original size --- plugins/brieflz/squash-brieflz.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/brieflz/squash-brieflz.c b/plugins/brieflz/squash-brieflz.c index 43e6df8..07907e7 100644 --- a/plugins/brieflz/squash-brieflz.c +++ b/plugins/brieflz/squash-brieflz.c @@ -54,6 +54,10 @@ squash_brieflz_decompress_buffer (SquashCodec* codec, unsigned long original_size; unsigned long size; + if (compressed_length < 4) { + return squash_error (SQUASH_FAILED); + } + original_size = (unsigned long) compressed[0] | ((unsigned long) compressed[1] << 8) | ((unsigned long) compressed[2] << 16) From 7fb3aadb83943e69fe9a575f1eafa8f03afaf51b Mon Sep 17 00:00:00 2001 From: Joergen Ibsen Date: Sun, 20 Sep 2015 15:19:34 +0200 Subject: [PATCH 3/5] Implement get_uncompressed_size --- plugins/brieflz/squash-brieflz.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/plugins/brieflz/squash-brieflz.c b/plugins/brieflz/squash-brieflz.c index 07907e7..e984a1f 100644 --- a/plugins/brieflz/squash-brieflz.c +++ b/plugins/brieflz/squash-brieflz.c @@ -44,6 +44,20 @@ squash_brieflz_get_max_compressed_size (SquashCodec* codec, size_t uncompressed_ return (size_t) blz_max_packed_size ((unsigned long) uncompressed_length) + 4; } +static size_t +squash_brieflz_get_uncompressed_size (SquashCodec* codec, + size_t compressed_length, + const uint8_t compressed[SQUASH_ARRAY_PARAM(compressed_length)]) { + if (compressed_length < 4) { + return 0; + } + + return (size_t) compressed[0] + | ((size_t) compressed[1] << 8) + | ((size_t) compressed[2] << 16) + | ((size_t) compressed[3] << 24); +} + static SquashStatus squash_brieflz_decompress_buffer (SquashCodec* codec, size_t* decompressed_length, @@ -120,6 +134,7 @@ squash_plugin_init_codec (SquashCodec* codec, SquashCodecImpl* impl) { const char* name = squash_codec_get_name (codec); if (strcmp ("brieflz", name) == 0) { + impl->get_uncompressed_size = squash_brieflz_get_uncompressed_size; impl->get_max_compressed_size = squash_brieflz_get_max_compressed_size; impl->decompress_buffer = squash_brieflz_decompress_buffer; impl->compress_buffer_unsafe = squash_brieflz_compress_buffer; From be00e999ab5ecc0a5c45736a9be9dd93f187c998 Mon Sep 17 00:00:00 2001 From: Joergen Ibsen Date: Sun, 20 Sep 2015 15:24:26 +0200 Subject: [PATCH 4/5] Use safe decompression function --- plugins/brieflz/CMakeLists.txt | 2 +- plugins/brieflz/squash-brieflz.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/brieflz/CMakeLists.txt b/plugins/brieflz/CMakeLists.txt index 17aeb8c..421902e 100644 --- a/plugins/brieflz/CMakeLists.txt +++ b/plugins/brieflz/CMakeLists.txt @@ -1,7 +1,7 @@ set (brieflz_sources squash-brieflz.c brieflz/brieflz.c - brieflz/depack.c) + brieflz/depacks.c) squash_plugin_add (brieflz brieflz_sources) squash_plugin_add_include_directories (brieflz brieflz) diff --git a/plugins/brieflz/squash-brieflz.c b/plugins/brieflz/squash-brieflz.c index e984a1f..8411a89 100644 --- a/plugins/brieflz/squash-brieflz.c +++ b/plugins/brieflz/squash-brieflz.c @@ -81,7 +81,8 @@ squash_brieflz_decompress_buffer (SquashCodec* codec, return squash_error (SQUASH_BUFFER_FULL); } - size = blz_depack (compressed + 4, decompressed, original_size); + size = blz_depack_safe (compressed + 4, (unsigned long) compressed_length - 4, + decompressed, original_size); if (size != original_size) { return squash_error (SQUASH_FAILED); From aca992d8bb3aeb6e0d1722e174bd5a22a26b4d82 Mon Sep 17 00:00:00 2001 From: Joergen Ibsen Date: Sun, 20 Sep 2015 23:44:34 +0200 Subject: [PATCH 5/5] Store original size as 64-bit variable-length integer --- plugins/brieflz/squash-brieflz.c | 125 ++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 28 deletions(-) diff --git a/plugins/brieflz/squash-brieflz.c b/plugins/brieflz/squash-brieflz.c index 8411a89..f568474 100644 --- a/plugins/brieflz/squash-brieflz.c +++ b/plugins/brieflz/squash-brieflz.c @@ -39,23 +39,87 @@ SQUASH_PLUGIN_EXPORT SquashStatus squash_plugin_init_codec (SquashCodec* codec, SquashCodecImpl* impl); +static size_t +read_varuint64 (const uint8_t *p, size_t p_size, uint64_t *v) { + uint64_t n = 0; + size_t i; + + for (i = 0; i < 8 && i < p_size && *p > 0x7F; ++i) { + n = (n << 7) | (*p++ & 0x7F); + } + + if (i == p_size) { + return 0; + } + else if (i == 8) { + n = (n << 8) | *p; + } + else { + n = (n << 7) | *p; + } + + *v = n; + + return i + 1; +} + +static size_t +write_varuint64 (uint8_t *p, size_t p_size, uint64_t v) { + uint8_t buf[10]; + size_t i; + size_t j; + + if (v & 0xFF00000000000000ULL) { + if (p_size < 9) { + return 0; + } + + p[8] = (uint8_t) v; + v >>= 8; + + i = 7; + + for (j = 0; j < 8; ++j) { + p[i--] = (uint8_t) ((v & 0x7F) | 0x80); + v >>= 7; + } + + return 9; + } + + i = 0; + + buf[i++] = (uint8_t) (v & 0x7F); + v >>= 7; + + while (v > 0) { + buf[i++] = (uint8_t) ((v & 0x7F) | 0x80); + v >>= 7; + } + + if (i > p_size) { + return 0; + } + + for (j = 0; j < i; ++j) { + p[j] = buf[i - j - 1]; + } + + return i; +} + static size_t squash_brieflz_get_max_compressed_size (SquashCodec* codec, size_t uncompressed_length) { - return (size_t) blz_max_packed_size ((unsigned long) uncompressed_length) + 4; + return (size_t) blz_max_packed_size ((unsigned long) uncompressed_length) + 9; } static size_t squash_brieflz_get_uncompressed_size (SquashCodec* codec, size_t compressed_length, const uint8_t compressed[SQUASH_ARRAY_PARAM(compressed_length)]) { - if (compressed_length < 4) { - return 0; - } + uint64_t v; - return (size_t) compressed[0] - | ((size_t) compressed[1] << 8) - | ((size_t) compressed[2] << 16) - | ((size_t) compressed[3] << 24); + return read_varuint64 (compressed, compressed_length, &v) == 0 ? 0 : v; } static SquashStatus @@ -65,30 +129,31 @@ squash_brieflz_decompress_buffer (SquashCodec* codec, size_t compressed_length, const uint8_t compressed[SQUASH_ARRAY_PARAM(compressed_length)], SquashOptions* options) { - unsigned long original_size; - unsigned long size; + const uint8_t *src = compressed; + uint64_t original_size; + size_t size; - if (compressed_length < 4) { + size = read_varuint64 (src, compressed_length, &original_size); + + if (size == 0) { return squash_error (SQUASH_FAILED); } - original_size = (unsigned long) compressed[0] - | ((unsigned long) compressed[1] << 8) - | ((unsigned long) compressed[2] << 16) - | ((unsigned long) compressed[3] << 24); + src += size; + compressed_length -= size; if (original_size > *decompressed_length) { return squash_error (SQUASH_BUFFER_FULL); } - size = blz_depack_safe (compressed + 4, (unsigned long) compressed_length - 4, - decompressed, original_size); + size = blz_depack_safe (src, (unsigned long) compressed_length, + decompressed, (unsigned long) original_size); if (size != original_size) { return squash_error (SQUASH_FAILED); } - *decompressed_length = (size_t) size; + *decompressed_length = size; return SQUASH_OK; } @@ -100,32 +165,36 @@ squash_brieflz_compress_buffer (SquashCodec* codec, size_t uncompressed_length, const uint8_t uncompressed[SQUASH_ARRAY_PARAM(uncompressed_length)], SquashOptions* options) { - unsigned long size; + uint8_t *dst = compressed; void *workmem = NULL; + size_t size; if ((unsigned long) *compressed_length < squash_brieflz_get_max_compressed_size (codec, uncompressed_length)) { return squash_error (SQUASH_BUFFER_FULL); } + size = write_varuint64 (dst, *compressed_length, uncompressed_length); + + if (size == 0) { + return squash_error (SQUASH_BUFFER_FULL); + } + + dst += size; + workmem = malloc (blz_workmem_size ((unsigned long) uncompressed_length)); if (workmem == NULL) { return squash_error (SQUASH_MEMORY); } - compressed[0] = (uint8_t) uncompressed_length; - compressed[1] = (uint8_t) (uncompressed_length >> 8); - compressed[2] = (uint8_t) (uncompressed_length >> 16); - compressed[3] = (uint8_t) (uncompressed_length >> 24); - - size = blz_pack (uncompressed, compressed + 4, - (unsigned long) uncompressed_length, - workmem); + size += blz_pack (uncompressed, dst, + (unsigned long) uncompressed_length, + workmem); free(workmem); - *compressed_length = (size_t) size + 4; + *compressed_length = size; return SQUASH_OK; }