Skip to content

Commit

Permalink
[WIP] First test of liblzma executable filters
Browse files Browse the repository at this point in the history
- See issue #75
- Filter maximum is 4, so up to 3 filters + LZMA2 can be used
- X86+PPC+ARM could be a good default
- TODO: Command line parameter so set or disable filters

- Changed VS settings, so unfortunately there are some tab/space changes in here, too.
  • Loading branch information
schnaader committed Dec 26, 2017
1 parent d92cc2d commit 1cfd08a
Showing 1 changed file with 64 additions and 22 deletions.
86 changes: 64 additions & 22 deletions contrib/liblzma/compress_easy_mt.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,24 @@ bool init_decoder(lzma_stream *strm)
return check(ret);
}

lzma_filter get_filter(lzma_vli filter_id) {
lzma_filter result;
result.id = filter_id;
result.options = NULL;
return result;
}

bool init_encoder_mt(lzma_stream *strm, int threads, uint64_t max_memory, uint64_t &memory_usage, uint64_t &block_size)
{
// The threaded encoder takes the options as pointer to
// a lzma_mt structure.
lzma_mt mt;

// No flags are needed.
mt .flags = 0;
mt.flags = 0;

// Let liblzma determine a sane block size.
mt .block_size = 0;
mt.block_size = 0;

// Use no timeout for lzma_code() calls by setting timeout
// to zero. That is, sometimes lzma_code() might block for
Expand All @@ -64,32 +71,67 @@ bool init_encoder_mt(lzma_stream *strm, int threads, uint64_t max_memory, uint64
// needing updates, specify a timeout in milliseconds here.
// See the documentation of lzma_mt in lzma/container.h for
// information how to choose a reasonable timeout.
mt .timeout = 110;

// To use a preset, filters must be set to NULL.
mt.filters = NULL;

mt.timeout = 110;
mt.check = LZMA_CHECK_CRC32;

mt.threads = threads;

uint64_t preset_memory_usage;
int preset_to_use = 1; // Always use at least preset 1, even if it exceeds the given memory limit
for (int preset = 2; preset <= 9; preset++) {
mt.preset = preset;
preset_memory_usage = lzma_stream_encoder_mt_memusage(&mt, &block_size);
if (preset_memory_usage > max_memory) break;
memory_usage = preset_memory_usage;
preset_to_use = preset;
}

mt.preset = preset_to_use;

uint64_t preset_memory_usage;
int preset_to_use = 1; // Always use at least preset 1, even if it exceeds the given memory limit
for (int preset = 2; preset <= 9; preset++) {
lzma_options_lzma opt_lzma2;
if (lzma_lzma_preset(&opt_lzma2, preset)) {
fprintf(stderr, "Unsupported preset, possibly a bug\n");
return false;
}

lzma_filter filterLzma2;
filterLzma2.id = LZMA_FILTER_LZMA2;
filterLzma2.options = &opt_lzma2;

// the executable filters we use don't change memory usage, so we just ignore them here
// and set them below
const lzma_filter filters[] = {
filterLzma2,
get_filter(LZMA_VLI_UNKNOWN)
};

mt.filters = filters;

preset_memory_usage = lzma_stream_encoder_mt_memusage(&mt, &block_size);
if (preset_memory_usage > max_memory) break;
memory_usage = preset_memory_usage;
preset_to_use = preset;
}

lzma_options_lzma opt_lzma2;
if (lzma_lzma_preset(&opt_lzma2, preset_to_use)) {
fprintf(stderr, "Unsupported preset, possibly a bug\n");
return false;
}

lzma_filter filterLzma2;
filterLzma2.id = LZMA_FILTER_LZMA2;
filterLzma2.options = &opt_lzma2;

const lzma_filter filters[] = {
get_filter(LZMA_FILTER_X86),
get_filter(LZMA_FILTER_POWERPC),
//get_filter(LZMA_FILTER_IA64),
get_filter(LZMA_FILTER_ARM),
//get_filter(LZMA_FILTER_ARMTHUMB),
//get_filter(LZMA_FILTER_SPARC),
filterLzma2,
get_filter(LZMA_VLI_UNKNOWN)
};

mt.filters = filters;

// Initialize the threaded encoder.
lzma_ret ret = lzma_stream_encoder_mt(strm, &mt);

// Determine memory usage and block size
memory_usage = lzma_stream_encoder_mt_memusage(&mt, &block_size);
// Determine memory usage and block size
memory_usage = lzma_stream_encoder_mt_memusage(&mt, &block_size);

return check(ret);
return check(ret);
}

0 comments on commit 1cfd08a

Please # to comment.