diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e1499f7..f3416e0a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2198,6 +2198,9 @@ this many MiB of RAM if xz cannot determine the amount at runtime") tuklib_add_definition_if(xz HAVE_OPTRESET) endif() + check_symbol_exists(getrlimit sys/resource.h HAVE_GETRLIMIT) + tuklib_add_definition_if(xz HAVE_GETRLIMIT) + check_symbol_exists(posix_fadvise fcntl.h HAVE_POSIX_FADVISE) tuklib_add_definition_if(xz HAVE_POSIX_FADVISE) diff --git a/configure.ac b/configure.ac index 47e2dc65..3b29cd84 100644 --- a/configure.ac +++ b/configure.ac @@ -1008,8 +1008,8 @@ AC_CHECK_DECL([CLOCK_MONOTONIC], [AC_DEFINE([HAVE_CLOCK_MONOTONIC], [1], # Find the best function to set timestamps. AC_CHECK_FUNCS([futimens futimes futimesat utimes _futime utime], [break]) -# This is nice to have but not mandatory. -AC_CHECK_FUNCS([posix_fadvise]) +# These are nice to have but not mandatory. +AC_CHECK_FUNCS([getrlimit posix_fadvise]) TUKLIB_PROGNAME TUKLIB_INTEGER diff --git a/src/xz/hardware.c b/src/xz/hardware.c index 952652fe..2e921474 100644 --- a/src/xz/hardware.c +++ b/src/xz/hardware.c @@ -11,6 +11,10 @@ #include "private.h" +#ifdef HAVE_GETRLIMIT +# include +#endif + /// Maximum number of worker threads. This can be set with /// the --threads=NUM command line option. @@ -321,6 +325,61 @@ hardware_init(void) // /proc/meminfo as the starting point. memlimit_mt_default = total_ram / 4; +#ifdef HAVE_GETRLIMIT + // Try to set the default multithreaded memory usage limit so that + // we won't exceed resource limits. Exceeding the limits would result + // in allocation failures, which currently make liblzma and xz fail + // (instead of continuing by reducing the number of threads). + const int resources[] = { + RLIMIT_DATA, +# ifdef RLIMIT_AS + RLIMIT_AS, // OpenBSD 7.8 doesn't have RLIMIT_AS. +# endif +# if defined(RLIMIT_VMEM) && RLIMIT_VMEM != RLIMIT_AS + RLIMIT_VMEM, // For Solaris. On FreeBSD this is an alias. +# endif + }; + + // The resource limits cannot be passed to liblzma directly; + // some margin is required: + // - The memory usage limit counts only liblzma's memory usage, + // but xz itself needs some memory (including gettext usage etc.). + // - Memory allocation has some overhead. + // - Address space limit counts code size too. + // + // The following value is a guess based on quick testing on Linux. + const rlim_t margin = 64 << 20; + + for (size_t i = 0; i < ARRAY_SIZE(resources); ++i) { + // RLIM_SAVED_* might be used on some 32-bit OSes + // (AIX at least) when the limit doesn't fit in a 32-bit + // unsigned integer. Thus, for us these are the same thing + // as no limit at all. + struct rlimit rl; + if (getrlimit(resources[i], &rl) == 0 + && rl.rlim_cur != RLIM_INFINITY + && rl.rlim_cur != RLIM_SAVED_CUR + && rl.rlim_cur != RLIM_SAVED_MAX) { + // Subtract the margin from the current resource + // limit, but avoid negative results. Avoid also 0 + // because hardware_memlimit_show() (--info-memory) + // treats it specially. In practice, 1 byte is + // effectively 0 anyway. + // + // SUSv2 and POSIX.1-2024 require rlimit_t to be + // unsigned. A cast is needed to silence a compiler + // warning still because, for historical reasons, + // rlim_t is intentionally signed on FreeBSD 14. + const uint64_t rl_with_margin = rl.rlim_cur > margin + ? (uint64_t)(rl.rlim_cur - margin) : 1; + + // Lower the memory usage limit if needed. + if (memlimit_mt_default > rl_with_margin) + memlimit_mt_default = rl_with_margin; + } + } +#endif + #if SIZE_MAX == UINT32_MAX // A too high value may cause 32-bit xz to run out of address space. // Use a conservative maximum value here. A few typical address space