xz: Make -T0 use multithreaded mode on single-core systems.

The main problem withi the old behavior is that the compressed
output is different on single-core systems vs. multicore systems.
This commit fixes it by making -T0 one thread in multithreaded mode
on single-core systems.

The downside of this is that it uses more memory. However, if
--memlimit-compress is used, xz can (thanks to the previous commit)
drop to the single-threaded mode still.
This commit is contained in:
Lasse Collin 2022-04-14 12:59:09 +03:00
parent 898faa9728
commit 0adc13bfe3
3 changed files with 27 additions and 9 deletions

View File

@ -211,7 +211,7 @@ coder_set_compression_settings(void)
} }
} }
if (hardware_threads_get() > 1) { if (hardware_threads_is_mt()) {
message(V_WARNING, _("Switching to single-threaded " message(V_WARNING, _("Switching to single-threaded "
"mode due to --flush-timeout")); "mode due to --flush-timeout"));
hardware_threads_set(1); hardware_threads_set(1);
@ -225,7 +225,7 @@ coder_set_compression_settings(void)
if (opt_mode == MODE_COMPRESS) { if (opt_mode == MODE_COMPRESS) {
#ifdef HAVE_ENCODERS #ifdef HAVE_ENCODERS
# ifdef MYTHREAD_ENABLED # ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && hardware_threads_get() > 1) { if (opt_format == FORMAT_XZ && hardware_threads_is_mt()) {
mt_options.threads = hardware_threads_get(); mt_options.threads = hardware_threads_get();
mt_options.block_size = opt_block_size; mt_options.block_size = opt_block_size;
mt_options.check = check; mt_options.check = check;
@ -278,7 +278,7 @@ coder_set_compression_settings(void)
#ifdef HAVE_ENCODERS #ifdef HAVE_ENCODERS
# ifdef MYTHREAD_ENABLED # ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && mt_options.threads > 1) { if (opt_format == FORMAT_XZ && hardware_threads_is_mt()) {
// Try to reduce the number of threads before // Try to reduce the number of threads before
// adjusting the compression settings down. // adjusting the compression settings down.
while (mt_options.threads > 1) { while (mt_options.threads > 1) {
@ -469,7 +469,7 @@ coder_init(file_pair *pair)
case FORMAT_XZ: case FORMAT_XZ:
# ifdef MYTHREAD_ENABLED # ifdef MYTHREAD_ENABLED
if (hardware_threads_get() > 1) if (hardware_threads_is_mt())
ret = lzma_stream_encoder_mt( ret = lzma_stream_encoder_mt(
&strm, &mt_options); &strm, &mt_options);
else else
@ -619,7 +619,7 @@ split_block(uint64_t *block_remaining,
{ {
if (*next_block_remaining > 0) { if (*next_block_remaining > 0) {
// The Block at *list_pos has previously been split up. // The Block at *list_pos has previously been split up.
assert(hardware_threads_get() == 1); assert(!hardware_threads_is_mt());
assert(opt_block_size > 0); assert(opt_block_size > 0);
assert(opt_block_list != NULL); assert(opt_block_list != NULL);
@ -647,7 +647,7 @@ split_block(uint64_t *block_remaining,
// If in single-threaded mode, split up the Block if needed. // If in single-threaded mode, split up the Block if needed.
// This is not needed in multi-threaded mode because liblzma // This is not needed in multi-threaded mode because liblzma
// will do this due to how threaded encoding works. // will do this due to how threaded encoding works.
if (hardware_threads_get() == 1 && opt_block_size > 0 if (!hardware_threads_is_mt() && opt_block_size > 0
&& *block_remaining > opt_block_size) { && *block_remaining > opt_block_size) {
*next_block_remaining *next_block_remaining
= *block_remaining - opt_block_size; = *block_remaining - opt_block_size;
@ -707,7 +707,7 @@ coder_normal(file_pair *pair)
// --block-size doesn't do anything here in threaded mode, // --block-size doesn't do anything here in threaded mode,
// because the threaded encoder will take care of splitting // because the threaded encoder will take care of splitting
// to fixed-sized Blocks. // to fixed-sized Blocks.
if (hardware_threads_get() == 1 && opt_block_size > 0) if (!hardware_threads_is_mt() && opt_block_size > 0)
block_remaining = opt_block_size; block_remaining = opt_block_size;
// If --block-list was used, start with the first size. // If --block-list was used, start with the first size.
@ -721,7 +721,7 @@ coder_normal(file_pair *pair)
// mode the size info isn't written into Block Headers. // mode the size info isn't written into Block Headers.
if (opt_block_list != NULL) { if (opt_block_list != NULL) {
if (block_remaining < opt_block_list[list_pos]) { if (block_remaining < opt_block_list[list_pos]) {
assert(hardware_threads_get() == 1); assert(!hardware_threads_is_mt());
next_block_remaining = opt_block_list[list_pos] next_block_remaining = opt_block_list[list_pos]
- block_remaining; - block_remaining;
} else { } else {
@ -785,7 +785,7 @@ coder_normal(file_pair *pair)
} else { } else {
// Start a new Block after LZMA_FULL_BARRIER. // Start a new Block after LZMA_FULL_BARRIER.
if (opt_block_list == NULL) { if (opt_block_list == NULL) {
assert(hardware_threads_get() == 1); assert(!hardware_threads_is_mt());
assert(opt_block_size > 0); assert(opt_block_size > 0);
block_remaining = opt_block_size; block_remaining = opt_block_size;
} else { } else {

View File

@ -17,6 +17,10 @@
/// the --threads=NUM command line option. /// the --threads=NUM command line option.
static uint32_t threads_max = 1; static uint32_t threads_max = 1;
/// True when the number of threads is automatically determined based
/// on the available hardware threads.
static bool threads_are_automatic = false;
/// Memory usage limit for compression /// Memory usage limit for compression
static uint64_t memlimit_compress = 0; static uint64_t memlimit_compress = 0;
@ -48,6 +52,8 @@ hardware_threads_set(uint32_t n)
{ {
if (n == 0) { if (n == 0) {
// Automatic number of threads was requested. // Automatic number of threads was requested.
threads_are_automatic = true;
// If threading support was enabled at build time, // If threading support was enabled at build time,
// use the number of available CPU cores. Otherwise // use the number of available CPU cores. Otherwise
// use one thread since disabling threading support // use one thread since disabling threading support
@ -61,6 +67,7 @@ hardware_threads_set(uint32_t n)
#endif #endif
} else { } else {
threads_max = n; threads_max = n;
threads_are_automatic = false;
} }
return; return;
@ -74,6 +81,13 @@ hardware_threads_get(void)
} }
extern bool
hardware_threads_is_mt(void)
{
return threads_max > 1 || threads_are_automatic;
}
extern void extern void
hardware_memlimit_set(uint64_t new_memlimit, hardware_memlimit_set(uint64_t new_memlimit,
bool set_compress, bool set_decompress, bool set_mtdec, bool set_compress, bool set_decompress, bool set_mtdec,

View File

@ -21,6 +21,10 @@ extern void hardware_threads_set(uint32_t threadlimit);
/// Get the maximum number of worker threads. /// Get the maximum number of worker threads.
extern uint32_t hardware_threads_get(void); extern uint32_t hardware_threads_get(void);
/// Returns true if multithreaded mode should be used for .xz compression.
/// This can be true even if the number of threads is one.
extern bool hardware_threads_is_mt(void);
/// Set the memory usage limit. There are separate limits for compression, /// Set the memory usage limit. There are separate limits for compression,
/// decompression (also includes --list), and multithreaded decompression. /// decompression (also includes --list), and multithreaded decompression.