xz: Add --memlimit-mt-decompress along with a default limit value.

--memlimit-mt-decompress allows specifying the limit for
multithreaded decompression. This matches memlimit_threading in
liblzma. This limit can only affect the number of threads being
used; it will never prevent xz from decompressing a file. The
old --memlimit-decompress option is still used at the same time.

If the value of --memlimit-decompress (the default value or
one specified by the user) is less than the value of
--memlimit-mt-decompress , then --memlimit-mt-decompress is
reduced to match --memlimit-decompress.

Man page wasn't updated yet.
This commit is contained in:
Lasse Collin 2022-04-11 22:20:49 +03:00
parent fe87b4cd53
commit cad299008c
5 changed files with 97 additions and 42 deletions

View File

@ -29,10 +29,11 @@ bool opt_ignore_check = false;
const char stdin_filename[] = "(stdin)"; const char stdin_filename[] = "(stdin)";
/// Parse and set the memory usage limit for compression and/or decompression. /// Parse and set the memory usage limit for compression, decompression,
/// and/or multithreaded decompression.
static void static void
parse_memlimit(const char *name, const char *name_percentage, char *str, parse_memlimit(const char *name, const char *name_percentage, char *str,
bool set_compress, bool set_decompress) bool set_compress, bool set_decompress, bool set_mtdec)
{ {
bool is_percentage = false; bool is_percentage = false;
uint64_t value; uint64_t value;
@ -49,8 +50,8 @@ parse_memlimit(const char *name, const char *name_percentage, char *str,
value = str_to_uint64(name, str, 0, UINT64_MAX); value = str_to_uint64(name, str, 0, UINT64_MAX);
} }
hardware_memlimit_set( hardware_memlimit_set(value, set_compress, set_decompress, set_mtdec,
value, set_compress, set_decompress, is_percentage); is_percentage);
return; return;
} }
@ -138,6 +139,7 @@ parse_real(args_info *args, int argc, char **argv)
OPT_BLOCK_LIST, OPT_BLOCK_LIST,
OPT_MEM_COMPRESS, OPT_MEM_COMPRESS,
OPT_MEM_DECOMPRESS, OPT_MEM_DECOMPRESS,
OPT_MEM_MT_DECOMPRESS,
OPT_NO_ADJUST, OPT_NO_ADJUST,
OPT_INFO_MEMORY, OPT_INFO_MEMORY,
OPT_ROBOT, OPT_ROBOT,
@ -176,6 +178,7 @@ parse_real(args_info *args, int argc, char **argv)
{ "block-list", required_argument, NULL, OPT_BLOCK_LIST }, { "block-list", required_argument, NULL, OPT_BLOCK_LIST },
{ "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS }, { "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS },
{ "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS }, { "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS },
{ "memlimit-mt-decompress", required_argument, NULL, OPT_MEM_MT_DECOMPRESS },
{ "memlimit", required_argument, NULL, 'M' }, { "memlimit", required_argument, NULL, 'M' },
{ "memory", required_argument, NULL, 'M' }, // Old alias { "memory", required_argument, NULL, 'M' }, // Old alias
{ "no-adjust", no_argument, NULL, OPT_NO_ADJUST }, { "no-adjust", no_argument, NULL, OPT_NO_ADJUST },
@ -225,20 +228,27 @@ parse_real(args_info *args, int argc, char **argv)
case OPT_MEM_COMPRESS: case OPT_MEM_COMPRESS:
parse_memlimit("memlimit-compress", parse_memlimit("memlimit-compress",
"memlimit-compress%", optarg, "memlimit-compress%", optarg,
true, false); true, false, false);
break; break;
// --memlimit-decompress // --memlimit-decompress
case OPT_MEM_DECOMPRESS: case OPT_MEM_DECOMPRESS:
parse_memlimit("memlimit-decompress", parse_memlimit("memlimit-decompress",
"memlimit-decompress%", optarg, "memlimit-decompress%", optarg,
false, true); false, true, false);
break;
// --memlimit-mt-decompress
case OPT_MEM_MT_DECOMPRESS:
parse_memlimit("memlimit-mt-decompress",
"memlimit-mt-decompress%", optarg,
false, false, true);
break; break;
// --memlimit // --memlimit
case 'M': case 'M':
parse_memlimit("memlimit", "memlimit%", optarg, parse_memlimit("memlimit", "memlimit%", optarg,
true, true); true, true, true);
break; break;
// --suffix // --suffix

View File

@ -524,32 +524,20 @@ coder_init(file_pair *pair)
mt_options.flags = flags; mt_options.flags = flags;
mt_options.threads = hardware_threads_get(); mt_options.threads = hardware_threads_get();
// TODO: Support --memlimit-threading=LIMIT.
mt_options.memlimit_stop mt_options.memlimit_stop
= hardware_memlimit_get(MODE_DECOMPRESS); = hardware_memlimit_get(MODE_DECOMPRESS);
mt_options.memlimit_threading
= mt_options.memlimit_stop;
if (mt_options.threads == 1) { // If single-threaded mode was requested, set the
// Single-threaded mode was requested. Force // memlimit for threading to zero. This forces the
// the decoder to use minimal memory, matching // decoder to use single-threaded mode which matches
// the behavior of lzma_stream_decoder(). // the behavior of lzma_stream_decoder().
mt_options.memlimit_threading = 0;
} else if (mt_options.memlimit_threading
== UINT64_MAX) {
// TODO: Support --memlimit-threading=LIMIT.
// //
// If lzma_physmem() fails, it returns 0 and // Otherwise use the limit for threaded decompression
// we end up with a single thread. // which has a sane default (users are still free to
// // make it insanely high though).
// NOTE: It is assential that we never end up
// with an effectively infinite value in
// memlimit_threading!
mt_options.memlimit_threading mt_options.memlimit_threading
= lzma_physmem() / 4; = mt_options.threads == 1
} ? 0 : hardware_memlimit_mtdec_get();
ret = lzma_stream_decoder_mt(&strm, &mt_options); ret = lzma_stream_decoder_mt(&strm, &mt_options);
# else # else

View File

@ -18,10 +18,26 @@
static uint32_t threads_max = 1; static uint32_t threads_max = 1;
/// Memory usage limit for compression /// Memory usage limit for compression
static uint64_t memlimit_compress; static uint64_t memlimit_compress = 0;
/// Memory usage limit for decompression /// Memory usage limit for decompression
static uint64_t memlimit_decompress; static uint64_t memlimit_decompress = 0;
/// Default memory usage for multithreaded modes:
///
/// - Default value for --memlimit-mt-decompress
///
/// This value is caluclated in hardware_init() and cannot be changed later.
static uint64_t memlimit_mt_default;
/// Memory usage limit for multithreaded decompression. This is a soft limit:
/// if reducing the number of threads to one isn't enough to keep memory
/// usage below this limit, then one thread is used and this limit is ignored.
/// memlimit_decompress is still obeyed.
///
/// This can be set with --memlimit-mt-decompress. The default value for
/// this is memlimit_mt_default.
static uint64_t memlimit_mtdec;
/// Total amount of physical RAM /// Total amount of physical RAM
static uint64_t total_ram; static uint64_t total_ram;
@ -60,7 +76,8 @@ hardware_threads_get(void)
extern void extern void
hardware_memlimit_set(uint64_t new_memlimit, hardware_memlimit_set(uint64_t new_memlimit,
bool set_compress, bool set_decompress, bool is_percentage) bool set_compress, bool set_decompress, bool set_mtdec,
bool is_percentage)
{ {
if (is_percentage) { if (is_percentage) {
assert(new_memlimit > 0); assert(new_memlimit > 0);
@ -110,6 +127,9 @@ hardware_memlimit_set(uint64_t new_memlimit,
if (set_decompress) if (set_decompress)
memlimit_decompress = new_memlimit; memlimit_decompress = new_memlimit;
if (set_mtdec)
memlimit_mtdec = new_memlimit;
return; return;
} }
@ -132,6 +152,23 @@ hardware_memlimit_get(enum operation_mode mode)
} }
extern uint64_t
hardware_memlimit_mtdec_get(void)
{
uint64_t m = memlimit_mtdec != 0
? memlimit_mtdec
: memlimit_mt_default;
// Cap the value to memlimit_decompress if it has been specified.
// This is nice for --info-memory. It wouldn't be needed for liblzma
// since it does this anyway.
if (memlimit_decompress != 0 && m > memlimit_decompress)
m = memlimit_decompress;
return m;
}
/// Helper for hardware_memlimit_show() to print one human-readable info line. /// Helper for hardware_memlimit_show() to print one human-readable info line.
static void static void
memlimit_show(const char *str, size_t str_columns, uint64_t value) memlimit_show(const char *str, size_t str_columns, uint64_t value)
@ -203,7 +240,20 @@ hardware_init(void)
if (total_ram == 0) if (total_ram == 0)
total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024; total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024;
// Set the defaults. // FIXME? There may be better methods to determine the default value.
hardware_memlimit_set(0, true, true, false); // One Linux-specific suggestion is to use MemAvailable from
// /proc/meminfo as the starting point.
memlimit_mt_default = total_ram / 4;
// A too high value may cause 32-bit xz to run out of address space.
// Use a conservative maximum value here. A few typical address space
// sizes with Linux:
// - x86-64 with 32-bit xz: 4 GiB
// - x86: 3 GiB
// - MIPS32: 2 GiB
const size_t mem_ceiling = SIZE_MAX / 3; // About 1365 GiB on 32-bit
if (memlimit_mt_default > mem_ceiling)
memlimit_mt_default = mem_ceiling;
return; return;
} }

View File

@ -22,16 +22,21 @@ extern void hardware_threads_set(uint32_t threadlimit);
extern uint32_t hardware_threads_get(void); extern uint32_t hardware_threads_get(void);
/// Set the memory usage limit. There are separate limits for compression /// Set the memory usage limit. There are separate limits for compression,
/// and decompression (the latter includes also --list), one or both can /// decompression (also includes --list), and multithreaded decompression.
/// be set with a single call to this function. Zero indicates resetting /// Any combination of these can be set with a single call to this function.
/// the limit back to the defaults. The limit can also be set as a percentage /// Zero indicates resetting the limit back to the defaults.
/// of installed RAM; the percentage must be in the range [1, 100]. /// The limit can also be set as a percentage of installed RAM; the
/// percentage must be in the range [1, 100].
extern void hardware_memlimit_set(uint64_t new_memlimit, extern void hardware_memlimit_set(uint64_t new_memlimit,
bool set_compress, bool set_decompress, bool is_percentage); bool set_compress, bool set_decompress, bool set_mtdec,
bool is_percentage);
/// Get the current memory usage limit for compression or decompression. /// Get the current memory usage limit for compression or decompression.
extern uint64_t hardware_memlimit_get(enum operation_mode mode); extern uint64_t hardware_memlimit_get(enum operation_mode mode);
/// Get the current memory usage limit for multithreaded decompression.
extern uint64_t hardware_memlimit_mtdec_get(void);
/// Display the amount of RAM and memory usage limits and exit. /// Display the amount of RAM and memory usage limits and exit.
extern void hardware_memlimit_show(void) lzma_attribute((__noreturn__)); extern void hardware_memlimit_show(void) lzma_attribute((__noreturn__));

View File

@ -1180,9 +1180,11 @@ message_help(bool long_help)
puts(_( // xgettext:no-c-format puts(_( // xgettext:no-c-format
" --memlimit-compress=LIMIT\n" " --memlimit-compress=LIMIT\n"
" --memlimit-decompress=LIMIT\n" " --memlimit-decompress=LIMIT\n"
" --memlimit-mt-decompress=LIMIT\n"
" -M, --memlimit=LIMIT\n" " -M, --memlimit=LIMIT\n"
" set memory usage limit for compression, decompression,\n" " set memory usage limit for compression, decompression,\n"
" or both; LIMIT is in bytes, % of RAM, or 0 for defaults")); " threaded decompression, or all of these; LIMIT is in\n"
" bytes, % of RAM, or 0 for defaults"));
puts(_( puts(_(
" --no-adjust if compression settings exceed the memory usage limit,\n" " --no-adjust if compression settings exceed the memory usage limit,\n"