root
/
xz
mirror of https://git.tukaani.org/xz.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Remove the nowadays unneeded memory limitting malloc() wrapper.

This commit is contained in:
Lasse Collin 2008-11-25 02:37:47 +02:00
parent 69472ee5f0
commit a8368b75cd
7 changed files with 0 additions and 614 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/liblzma/api/Makefile.am
View File

@ -25,7 +25,6 @@ nobase_include_HEADERS = \
lzma/index_hash.h \
lzma/init.h \
lzma/lzma.h \
lzma/memlimit.h \
lzma/simple.h \
lzma/stream_flags.h \
lzma/subblock.h \

1
src/liblzma/api/lzma.h
View File

@ -208,7 +208,6 @@ extern "C" {
#include "lzma/index.h"
#include "lzma/index_hash.h"
#include "lzma/stream_flags.h"
#include "lzma/memlimit.h"
/*
* All subheaders included. Undefine LZMA_H_INTERNAL to prevent applications

207
src/liblzma/api/lzma/memlimit.h
View File

@ -1,207 +0,0 @@
/**
* \file lzma/memlimit.h
* \brief Memory usage limiter
*
* \author Copyright (C) 1999-2006 Igor Pavlov
* \author Copyright (C) 2007 Lasse Collin
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*/
#ifndef LZMA_H_INTERNAL
# error Never include this file directly. Use <lzma.h> instead.
#endif
/**
* \brief Opaque data type used with the memory usage limiting functions
*/
typedef struct lzma_memlimit_s lzma_memlimit;
/**
* \brief Allocates and initializes a new lzma_memlimit structure
*
* It is easy to make liblzma to use huge amounts of memory. This can
* be a problem especially with the decoder, since it a file requiring
* huge amounts of memory to uncompress could allow even a denial of
* service attack if the memory usage wasn't limited.
*
* liblzma provides a set of functions to control memory usage. Pointers
* to these functions can be used in lzma_allocator structure, which makes
* it easy to limit memory usage with liblzma.
*
* The memory limiter functions are not tied to limiting memory usage
* with liblzma itself. You can use them with anything you like.
*
* In multi-threaded applications, only one thread at once may use the same
* lzma_memlimit structure. If there is a need, this limitation may
* be removed in future versions without breaking the libary API/ABI.
*
* \param limit Initial memory usage limit in bytes
*
* \return Pointer to allocated and initialized lzma_memlimit
* structure. On error, NULL is returned. The reason behind
* an error is either that malloc() failed or that the given
* limit was so small that it didn't allow allocating even
* the lzma_memlimit structure itself.
*
* \note Excluding lzma_memlimit_usage(), the functions whose name begin
* lzma_memlimit_ can be used even if lzma_init() hasn't been
* called.
*/
extern lzma_memlimit *lzma_memlimit_create(size_t limit)
lzma_attr_warn_unused_result;
/**
* \brief Sets a new memory usage limit
*
* \param mem Pointer to a lzma_memlimit structure returned
* earlier by lzma_memry_limit_create().
* \param limit New memory usage limit
*
* The new usage limit may be smaller than the amount of memory currently
* allocated via *mem: New allocations will fail until enough memory has
* been freed or a new limit is set, but the existing allocatations will
* stay untouched.
*/
extern void lzma_memlimit_set(lzma_memlimit *mem, size_t limit);
/**
* \brief Gets the current memory usage limit
*/
extern size_t lzma_memlimit_get(const lzma_memlimit *mem)
lzma_attr_pure;
/**
* \brief Gets the amount of currently allocated memory
*
* \note This value includes the sizes of some helper structures,
* thus it will always be larger than the total number of
* bytes allocated via lzma_memlimit_alloc().
*/
extern size_t lzma_memlimit_used(const lzma_memlimit *mem)
lzma_attr_pure;
/**
* \brief Gets the maximum amount of memory required in total
*
* Returns how much memory was or would have been allocated at the same time.
* If lzma_memlimit_alloc() was requested so much memory that the limit
* would have been exceeded or malloc() simply ran out of memory, the
* requested amount is still included to the value returned by
* lzma_memlimit_max(). This may be used as a hint how much bigger memory
* limit would have been needed.
*
* If the clear flag is set, the internal variable holding the maximum
* value is set to the current memory usage (the same value as returned
* by lzma_memlimit_used()).
*
* \note Usually liblzma needs to allocate many chunks of memory, and
* displaying a message like "memory usage limit reached, at
* least 1024 bytes would have been needed" may be confusing,
* because the next allocation could have been e.g. 8 MiB.
*
* \todo The description of this function is unclear.
*/
extern size_t lzma_memlimit_max(lzma_memlimit *mem, lzma_bool clear);
/**
* \brief Checks if memory limit was reached at some point
*
* This function is useful to find out if the reason for LZMA_MEM_ERROR
* was running out of memory or hitting the memory usage limit imposed
* by lzma_memlimit_alloc(). If the clear argument is true, the internal
* flag, that indicates that limit was reached, is cleared.
*/
extern lzma_bool lzma_memlimit_reached(lzma_memlimit *mem, lzma_bool clear);
/**
* \brief Gets the number of allocations owned by the memory limiter
*
* The count does not include the helper structures; if no memory has
* been allocated with lzma_memlimit_alloc() or all memory allocated
* has been freed or detached, this will return zero.
*/
extern size_t lzma_memlimit_count(const lzma_memlimit *mem)
lzma_attr_pure;
/**
* \brief Allocates memory with malloc() if memory limit allows
*
* \param mem Pointer to a lzma_memlimit structure returned
* earlier by lzma_memry_limit_create().
* \param nmemb Number of elements to allocate. While liblzma always
* sets this to one, this function still takes the
* value of nmemb into account to keep the function
* usable with zlib and libbzip2.
* \param size Size of an element.
*
* \return Pointer to memory allocated with malloc(nmemb * size),
* except if nmemb * size == 0 which returns malloc(1).
* On error, NULL is returned.
*
* \note This function assumes that nmemb * size is at maximum of
* SIZE_MAX. If it isn't, an overflow will occur resulting
* invalid amount of memory being allocated.
*/
extern void *lzma_memlimit_alloc(
lzma_memlimit *mem, size_t nmemb, size_t size)
lzma_attr_warn_unused_result;
/**
* \brief Removes the pointer from memory limiting list
*
* \param mem Pointer to a lzma_memlimit structure returned
* earlier by lzma_memry_limit_create().
* \param ptr Pointer returned earlier by lzma_memlimit_alloc().
*
* This function removes ptr from the internal list and decreases the
* counter of used memory accordingly. The ptr itself isn't freed. This is
* useful when Extra Records allocated by liblzma using lzma_memlimit
* are needed by the application and must not be freed when the
* lzma_memlimit structure is destroyed.
*
* It is OK to call this function with ptr that hasn't been allocated with
* lzma_memlimit_alloc(). In that case, this has no effect other than wasting
* a few CPU cycles.
*/
extern void lzma_memlimit_detach(lzma_memlimit *mem, void *ptr);
/**
* \brief Frees memory and updates the memory limit list
*
* This is like lzma_memlimit_detach() but also frees the given pointer.
*/
extern void lzma_memlimit_free(lzma_memlimit *mem, void *ptr);
/**
* \brief Frees the memory allocated for and by the memory usage limiter
*
* \param mem Pointer to memory limiter
* \param free_allocated If this is non-zero, all the memory allocated
* by lzma_memlimit_alloc() using *mem is also
* freed if it hasn't already been freed with
* lzma_memlimit_free(). Usually this should be
* set to true.
*/
extern void lzma_memlimit_end(
lzma_memlimit *mem, lzma_bool free_allocated);

1
src/liblzma/common/Makefile.am
View File

@ -33,7 +33,6 @@ libcommon_la_SOURCES = \
index.c \
index.h \
init.c \
memory_limiter.c \
stream_flags_common.c \
stream_flags_common.h \
vli_size.c

288
src/liblzma/common/memory_limiter.c
View File

@ -1,288 +0,0 @@
///////////////////////////////////////////////////////////////////////////////
//
/// \file memory_limiter.c
/// \brief Limitting memory usage
//
// Copyright (C) 2007 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
///////////////////////////////////////////////////////////////////////////////
#include "common.h"
/// Rounds an unsigned integer upwards to the next multiple.
#define my_ceil(num, multiple) \
((num) + (((multiple) - ((num) % (multiple))) % (multiple)))
/// Add approximated overhead of malloc() to size and round upwards to the
/// next multiple of 2 * sizeof(size_t). I suppose that most malloc()
/// implementations align small allocations this way, but the overhead
/// varies due to several reasons (free lists, mmap() usage etc.).
///
/// This doesn't need to be exact at all. It's enough to take into account
/// that there is some overhead. That way our memory usage count won't be
/// horribly wrong if we are used to allocate lots of small memory chunks.
#define malloc_ceil(size) \
my_ceil((size) + 2 * sizeof(void *), 2 * sizeof(size_t))
typedef struct lzma_memlimit_list_s lzma_memlimit_list;
struct lzma_memlimit_list_s {
lzma_memlimit_list *next;
void *ptr;
size_t size;
};
struct lzma_memlimit_s {
/// List of allocated memory chunks
lzma_memlimit_list *list;
/// Number of bytes currently allocated; this includes the memory
/// needed for the helper structures.
size_t used;
/// Memory usage limit
size_t limit;
/// Maximum amount of memory that have been or would have been needed.
/// That is, this is updated also if memory allocation fails, letting
/// the application check how much memory was tried to be allocated
/// in total.
size_t max;
/// True if lzma_memlimit_alloc() has returned NULL due to memory
/// usage limit.
bool limit_reached;
};
extern LZMA_API lzma_memlimit *
lzma_memlimit_create(size_t limit)
{
const size_t base_size = malloc_ceil(sizeof(lzma_memlimit));
if (limit < base_size)
return NULL;
lzma_memlimit *mem = malloc(sizeof(lzma_memlimit));
if (mem != NULL) {
mem->list = NULL;
mem->used = base_size;
mem->limit = limit;
mem->max = base_size;
mem->limit_reached = false;
}
return mem;
}
extern LZMA_API void
lzma_memlimit_set(lzma_memlimit *mem, size_t limit)
{
mem->limit = limit;
return;
}
extern LZMA_API size_t
lzma_memlimit_get(const lzma_memlimit *mem)
{
return mem->limit;
}
extern LZMA_API size_t
lzma_memlimit_used(const lzma_memlimit *mem)
{
return mem->used;
}
extern LZMA_API size_t
lzma_memlimit_max(lzma_memlimit *mem, lzma_bool clear)
{
const size_t ret = mem->max;
if (clear)
mem->max = mem->used;
return ret;
}
extern LZMA_API lzma_bool
lzma_memlimit_reached(lzma_memlimit *mem, lzma_bool clear)
{
const bool ret = mem->limit_reached;
if (clear)
mem->limit_reached = false;
return ret;
}
extern LZMA_API size_t
lzma_memlimit_count(const lzma_memlimit *mem)
{
// This is slow; we could have a counter in lzma_memlimit
// for fast version. I expect the primary use of this
// function to be limited to easy checking of memory leaks,
// in which this implementation is just fine.
size_t count = 0;
const lzma_memlimit_list *record = mem->list;
while (record != NULL) {
++count;
record = record->next;
}
return count;
}
extern LZMA_API void
lzma_memlimit_end(lzma_memlimit *mem, lzma_bool free_allocated)
{
if (mem == NULL)
return;
lzma_memlimit_list *record = mem->list;
while (record != NULL) {
if (free_allocated)
free(record->ptr);
lzma_memlimit_list *tmp = record;
record = record->next;
free(tmp);
}
free(mem);
return;
}
extern LZMA_API void *
lzma_memlimit_alloc(lzma_memlimit *mem, size_t nmemb, size_t size)
{
// While liblzma always sets nmemb to one, do this multiplication
// to make these functions usable e.g. with zlib and libbzip2.
// Making sure that this doesn't overflow is up to the application.
size *= nmemb;
// Some malloc() implementations return NULL on malloc(0). We like
// to get a non-NULL value.
if (size == 0)
size = 1;
// Calculate how much memory we are going to allocate in reality.
const size_t total_size = malloc_ceil(size)
+ malloc_ceil(sizeof(lzma_memlimit_list));
// Integer overflow protection for total_size and mem->used.
if (total_size <= size || SIZE_MAX - total_size < mem->used) {
mem->max = SIZE_MAX;
mem->limit_reached = true;
return NULL;
}
// Update the maximum memory requirement counter if needed. This
// is updated even if memory allocation would fail or limit would
// be reached.
if (mem->used + total_size > mem->max)
mem->max = mem->used + total_size;
// Check if we would stay in the memory usage limits. We need to
// check also that the current usage is in the limits, because
// the application could have decreased the limit between calls
// to this function.
if (mem->limit < mem->used || mem->limit - mem->used < total_size) {
mem->limit_reached = true;
return NULL;
}
// Allocate separate memory chunks for lzma_memlimit_list and the
// actual requested memory. Optimizing this to use only one
// allocation is not a good idea, because applications may want to
// detach lzma_extra structures that have been allocated with
// lzma_memlimit_alloc().
lzma_memlimit_list *record = malloc(sizeof(lzma_memlimit_list));
void *ptr = malloc(size);
if (record == NULL || ptr == NULL) {
free(record);
free(ptr);
return NULL;
}
// Add the new entry to the beginning of the list. This should be
// more efficient when freeing memory, because usually it is
// "last allocated, first freed".
record->next = mem->list;
record->ptr = ptr;
record->size = total_size;
mem->list = record;
mem->used += total_size;
return ptr;
}
extern LZMA_API void
lzma_memlimit_detach(lzma_memlimit *mem, void *ptr)
{
if (ptr == NULL || mem->list == NULL)
return;
lzma_memlimit_list *record = mem->list;
lzma_memlimit_list *prev = NULL;
while (record->ptr != ptr) {
prev = record;
record = record->next;
if (record == NULL)
return;
}
if (prev != NULL)
prev->next = record->next;
else
mem->list = record->next;
assert(mem->used >= record->size);
mem->used -= record->size;
free(record);
return;
}
extern LZMA_API void
lzma_memlimit_free(lzma_memlimit *mem, void *ptr)
{
if (ptr == NULL)
return;
lzma_memlimit_detach(mem, ptr);
free(ptr);
return;
}

2
tests/Makefile.am
View File

@ -34,7 +34,6 @@ endif
check_PROGRAMS = \
create_compress_files \
test_memlimit \
test_check \
test_stream_flags \
test_filter_flags \
@ -42,7 +41,6 @@ check_PROGRAMS = \
test_index
TESTS = \
test_memlimit \
test_check \
test_stream_flags \
test_filter_flags \

114
tests/test_memlimit.c
View File

@ -1,114 +0,0 @@
///////////////////////////////////////////////////////////////////////////////
//
/// \file test_memlimit.c
/// \brief Tests the memory usage limiter
///
/// \note These tests cannot be done at exact byte count accuracy,
/// because memory limiter takes into account the memory wasted
/// by bookkeeping structures and alignment (padding).
//
// Copyright (C) 2008 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
///////////////////////////////////////////////////////////////////////////////
#include "tests.h"
int
main(void)
{
void *a;
void *b;
lzma_memlimit *mem;
expect((mem = lzma_memlimit_create(1 << 16)) != NULL);
expect(lzma_memlimit_count(mem) == 0);
expect(lzma_memlimit_used(mem) > 0);
expect(lzma_memlimit_used(mem) < 4096);
expect(lzma_memlimit_used(mem) == lzma_memlimit_max(mem, false));
expect(!lzma_memlimit_reached(mem, false));
expect((a = lzma_memlimit_alloc(mem, 1, 4096)) != NULL);
expect(lzma_memlimit_count(mem) == 1);
expect(lzma_memlimit_used(mem) > 4096);
expect(lzma_memlimit_used(mem) < 8192);
expect(lzma_memlimit_used(mem) == lzma_memlimit_max(mem, false));
expect(!lzma_memlimit_reached(mem, false));
expect((b = lzma_memlimit_alloc(mem, 1, 4096)) != NULL);
expect(lzma_memlimit_count(mem) == 2);
expect(lzma_memlimit_used(mem) > 8192);
expect(lzma_memlimit_used(mem) < 12288);
expect(lzma_memlimit_used(mem) == lzma_memlimit_max(mem, false));
expect(!lzma_memlimit_reached(mem, false));
expect((lzma_memlimit_alloc(mem, 1, 1 << 17)) == NULL);
expect(lzma_memlimit_count(mem) == 2);
expect(lzma_memlimit_used(mem) > 8192);
expect(lzma_memlimit_used(mem) < 12288);
expect(lzma_memlimit_used(mem) < lzma_memlimit_max(mem, false));
expect(lzma_memlimit_max(mem, false) > (1 << 17));
expect(lzma_memlimit_reached(mem, false));
lzma_memlimit_free(mem, a);
expect(lzma_memlimit_count(mem) == 1);
expect(lzma_memlimit_used(mem) > 4096);
expect(lzma_memlimit_used(mem) < 8192);
expect(lzma_memlimit_max(mem, true) > (1 << 17));
expect(lzma_memlimit_reached(mem, true));
expect(lzma_memlimit_used(mem) == lzma_memlimit_max(mem, false));
expect(!lzma_memlimit_reached(mem, false));
expect(lzma_memlimit_get(mem) == 1 << 16);
lzma_memlimit_set(mem, 6144);
expect(lzma_memlimit_get(mem) == 6144);
expect(lzma_memlimit_alloc(mem, 1, 4096) == NULL);
expect(lzma_memlimit_max(mem, false) > 8192);
expect(lzma_memlimit_reached(mem, false));
lzma_memlimit_free(mem, b);
expect(lzma_memlimit_count(mem) == 0);
expect(lzma_memlimit_used(mem) > 0);
expect(lzma_memlimit_used(mem) < 4096);
expect((a = lzma_memlimit_alloc(mem, 1, 4096)) != NULL);
expect(lzma_memlimit_count(mem) == 1);
expect(lzma_memlimit_used(mem) > 4096);
expect(lzma_memlimit_used(mem) < 8192);
expect(lzma_memlimit_max(mem, false) > 8192);
expect(lzma_memlimit_reached(mem, false));
expect(lzma_memlimit_max(mem, true) > 8192);
expect(lzma_memlimit_reached(mem, true));
expect(lzma_memlimit_max(mem, true) < 8192);
expect(!lzma_memlimit_reached(mem, true));
lzma_memlimit_detach(mem, a);
free(a);
expect(lzma_memlimit_count(mem) == 0);
lzma_memlimit_set(mem, SIZE_MAX);
expect(lzma_memlimit_alloc(mem, 1, SIZE_MAX - 33) == NULL);
expect(lzma_memlimit_count(mem) == 0);
expect(lzma_memlimit_max(mem, true) == SIZE_MAX);
expect(lzma_memlimit_reached(mem, true));
expect(lzma_memlimit_alloc(mem, 1, SIZE_MAX) == NULL);
expect(lzma_memlimit_count(mem) == 0);
expect(lzma_memlimit_max(mem, false) == SIZE_MAX);
expect(lzma_memlimit_reached(mem, false));
lzma_memlimit_end(mem, true);
return 0;
}