root
/
xz
mirror of https://git.tukaani.org/xz.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Revised the Index handling code. 

This breaks API and ABI but most apps are not affected
since most apps don't use this part of the API. You will
get a compile error if you are using anything that got
broken.

Summary of changes:

  - Ability to store Stream Flags, which are needed
    for random-access reading in multi-Stream files.

  - Separate function to set size of Stream Padding.

  - Iterator structure makes it possible to read the same
    lzma_index from multiple threads at the same time.

  - A lot faster code to locate Blocks.

  - Removed lzma_index_equal() without adding anything
    to replace it. I don't know what it should do exactly
    with the new features and what actually needs this
    function in the first place other than test_index.c,
    which now has its own code to compare lzma_indexes.
This commit is contained in:
Lasse Collin 2009-12-31 22:45:53 +02:00
parent f29997a846
commit 1a7ec87c8e
9 changed files with 1733 additions and 888 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

550
src/liblzma/api/lzma/index.h
View File

@ -1,6 +1,6 @@
/** /**
* \file lzma/index.h * \file lzma/index.h
* \brief Handling of .xz Index lists * \brief Handling of .xz Index and related information
*/ */
/* /*
@ -18,98 +18,295 @@
/** /**
* \brief Opaque data type to hold the Index * \brief Opaque data type to hold the Index(es) and other information
*
* lzma_index often holds just one .xz Index and possibly the Stream Flags
* of the same Stream and size of the Stream Padding field. However,
* multiple lzma_indexes can be concatenated with lzma_index_cat() and then
* there may be information about multiple Streams in the same lzma_index.
*
* Notes about thread safety: Only one thread may modify lzma_index at
* a time. All functions that take non-const pointer to lzma_index
* modify it. As long as no thread is modifying the lzma_index, getting
* information from the same lzma_index can be done from multiple threads
* at the same time with functions that take a const pointer to
* lzma_index or use lzma_index_iter. The same iterator must be used
* only by one thread at a time, of course, but there can be as many
* iterators for the same lzma_index as needed.
*/ */
typedef struct lzma_index_s lzma_index; typedef struct lzma_index_s lzma_index;
/** /**
* \brief Index Record and its location * \brief Iterator to get information about Blocks and Streams
*/ */
typedef struct { typedef struct {
struct {
/** /**
* \brief Total encoded size of a Block including Block Padding * \brief Pointer to Stream Flags
* *
* This value is useful if you need to know the actual size of the * This is NULL if Stream Flags have not been set for
* Block that the Block decoder will read. * this Stream with lzma_index_stream_flags().
*/ */
lzma_vli total_size; const lzma_stream_flags *flags;
const void *reserved_ptr1;
const void *reserved_ptr2;
const void *reserved_ptr3;
/** /**
* \brief Encoded size of a Block excluding Block Padding * \brief Stream number in the lzma_index
* *
* This value is stored in the Index. When doing random-access * The first Stream is 1.
* reading, you should give this value to the Block decoder along
* with uncompressed_size.
*/ */
lzma_vli unpadded_size; lzma_vli number;
/** /**
* \brief Uncompressed Size of a Block * \brief Number of Blocks in the Stream
*
* If this is zero, the block structure below has
* undefined values.
*/
lzma_vli block_count;
/**
* \brief Compressed start offset of this Stream
*
* The offset is relative to the beginning of the lzma_index
* (i.e. usually the beginning of the .xz file).
*/
lzma_vli compressed_offset;
/**
* \brief Uncompressed start offset of this Stream
*
* The offset is relative to the beginning of the lzma_index
* (i.e. usually the beginning of the .xz file).
*/
lzma_vli uncompressed_offset;
/**
* \brief Compressed size of this Stream
*
* This includes all headers except the possible
* Stream Padding after this Stream.
*/
lzma_vli compressed_size;
/**
* \brief Uncompressed size of this Stream
*/ */
lzma_vli uncompressed_size; lzma_vli uncompressed_size;
/** /**
* \brief Compressed offset in the Stream(s) * \brief Size of Stream Padding after this Stream
* *
* This is the offset of the first byte of the Block, that is, * If it hasn't been set with lzma_index_stream_padding(),
* where you need to seek to decode the Block. The offset * this defaults to zero. Stream Padding is always
* is relative to the beginning of the Stream, or if there are * a multiple of four bytes.
* multiple Indexes combined, relative to the beginning of the
* first Stream.
*/ */
lzma_vli stream_offset; lzma_vli padding;
lzma_vli reserved_vli1;
lzma_vli reserved_vli2;
lzma_vli reserved_vli3;
lzma_vli reserved_vli4;
} stream;
struct {
/**
* \brief Block number in the file
*
* The first Block is 1.
*/
lzma_vli number_in_file;
/** /**
* \brief Uncompressed offset * \brief Compressed start offset of this Block
* *
* When doing random-access reading, it is possible that the target * This offset is relative to the beginning of the
* offset is not exactly at Block boundary. One will need to compare * lzma_index (i.e. usually the beginning of the .xz file).
* the target offset against uncompressed_offset, and possibly decode * Normally this is where you should seek in the .xz file
* and throw away some amount of data before reaching the target * to start decompressing this Block.
* offset.
*/ */
lzma_vli uncompressed_offset; lzma_vli compressed_file_offset;
} lzma_index_record; /**
* \brief Uncompressed start offset of this Block
*
* This offset is relative to the beginning of the lzma_index
* (i.e. usually the beginning of the .xz file).
*/
lzma_vli uncompressed_file_offset;
/**
* \brief Block number in this Stream
*
* The first Block is 1.
*/
lzma_vli number_in_stream;
/**
* \brief Compressed start offset of this Block
*
* This offset is relative to the beginning of the Stream
* containing this Block.
*/
lzma_vli compressed_stream_offset;
/**
* \brief Uncompressed start offset of this Block
*
* This offset is relative to the beginning of the Stream
* containing this Block.
*/
lzma_vli uncompressed_stream_offset;
/**
* \brief Uncompressed size of this Block
*
* You should pass this to the Block decoder if you will
* decode this Block.
*
* When doing random-access reading, it is possible that
* the target offset is not exactly at Block boundary. One
* will need to compare the target offset against
* uncompressed_file_offset or uncompressed_stream_offset,
* and possibly decode and throw away some amount of data
* before reaching the target offset.
*/
lzma_vli uncompressed_size;
/**
* \brief Unpadded size of this Block
*
* You should pass this to the Block decoder if you will
* decode this Block.
*/
lzma_vli unpadded_size;
/**
* \brief Total compressed size
*
* This includes all headers and padding in this Block.
* This is useful if you need to know how many bytes
* the Block decoder will actually read.
*/
lzma_vli total_size;
lzma_vli reserved_vli1;
lzma_vli reserved_vli2;
lzma_vli reserved_vli3;
lzma_vli reserved_vli4;
const void *reserved_ptr1;
const void *reserved_ptr2;
const void *reserved_ptr3;
const void *reserved_ptr4;
} block;
/*
* Internal data which is used to store the state of the iterator.
* The exact format may vary between liblzma versions, so don't
* touch these in any way.
*/
union {
const void *p;
size_t s;
lzma_vli v;
} internal[6];
} lzma_index_iter;
/** /**
* \brief Calculate memory usage for Index with given number of Records * \brief Operation mode for lzma_index_iter_next()
*/
typedef enum {
LZMA_INDEX_ITER_ANY = 0,
/**<
* \brief Get the next Block or Stream
*
* Go to the next Block if the current Stream has at least
* one Block left. Otherwise go to the next Stream even if
* it has no Blocks. If the Stream has no Blocks
* (lzma_index_iter.stream.block_count == 0),
* lzma_index_iter.block will have undefined values.
*/
LZMA_INDEX_ITER_STREAM = 1,
/**<
* \brief Get the next Stream
*
* Go to the next Stream even if the current Stream has
* unread Blocks left. If the next Stream has at least one
* Block, the iterator will point to the first Block.
* If there are no Blocks, lzma_index_iter.block will have
* undefined values.
*/
LZMA_INDEX_ITER_BLOCK = 2,
/**<
* \brief Get the next Block
*
* Go to the next Block if the current Stream has at least
* one Block left. If the current Stream has no Blocks left,
* the next Stream with at least one Block is located and
* the iterator will be made to point to the first Block of
* that Stream.
*/
LZMA_INDEX_ITER_NONEMPTY_BLOCK = 3
/**<
* \brief Get the next non-empty Block
*
* This is like LZMA_INDEX_ITER_BLOCK except that it will
* skip Blocks whose Uncompressed Size is zero.
*/
} lzma_index_iter_mode;
/**
* \brief Calculate memory usage of lzma_index
* *
* On disk, the size of the Index field depends on both the number of Records * On disk, the size of the Index field depends on both the number of Records
* stored and how big values the Records store (due to variable-length integer * stored and how big values the Records store (due to variable-length integer
* encoding). When the Index is kept in lzma_index structure, the memory usage * encoding). When the Index is kept in lzma_index structure, the memory usage
* depends only on the number of Records stored in the Index. The size in RAM * depends only on the number of Records/Blocks stored in the Index(es), and
* is almost always a lot bigger than in encoded form on disk. * in case of concatenated lzma_indexes, the number of Streams. The size in
* RAM is almost always significantly bigger than in the encoded form on disk.
* *
* This function calculates an approximate amount of memory needed hold the * This function calculates an approximate amount of memory needed hold
* given number of Records in lzma_index structure. This value may vary * the given number of Streams and Blocks in lzma_index structure. This
* between liblzma versions if the internal implementation is modified. * value may vary between CPU archtectures and also between liblzma versions
* * if the internal implementation is modified.
* If you want to know how much memory an existing lzma_index structure is
* using, use lzma_index_memusage(lzma_index_count(i)).
*/ */
extern LZMA_API(uint64_t) lzma_index_memusage(lzma_vli record_count) extern LZMA_API(uint64_t) lzma_index_memusage(
lzma_vli streams, lzma_vli blocks) lzma_nothrow;
/**
* \brief Calculate the memory usage of an existing lzma_index
*
* This is a shorthand for lzma_index_memusage(lzma_index_stream_count(i),
* lzma_index_block_count(i)).
*/
extern LZMA_API(uint64_t) lzma_index_memused(const lzma_index *i)
lzma_nothrow; lzma_nothrow;
/** /**
* \brief Allocate and initialize a new lzma_index structure * \brief Allocate and initialize a new lzma_index structure
* *
* If i is NULL, a new lzma_index structure is allocated, initialized, * \return On success, a pointer to an empty initialized lzma_index is
* and a pointer to it returned. If allocation fails, NULL is returned. * returned. If allocation fails, NULL is returned.
*
* If i is non-NULL, it is reinitialized and the same pointer returned.
* In this case, return value cannot be NULL or a different pointer than
* the i that was given as an argument.
*/ */
extern LZMA_API(lzma_index *) lzma_index_init( extern LZMA_API(lzma_index *) lzma_index_init(lzma_allocator *allocator)
lzma_index *i, lzma_allocator *allocator) lzma_nothrow; lzma_nothrow;
/** /**
* \brief Deallocate the Index * \brief Deallocate lzma_index
* *
* If i is NULL, this does nothing. * If i is NULL, this does nothing.
*/ */
@ -118,7 +315,7 @@ extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator)
/** /**
* \brief Add a new Record to an Index * \brief Add a new Block to lzma_index
* *
* \param i Pointer to a lzma_index structure * \param i Pointer to a lzma_index structure
* \param allocator Pointer to lzma_allocator, or NULL to * \param allocator Pointer to lzma_allocator, or NULL to
@ -130,7 +327,10 @@ extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator)
* taken directly from lzma_block structure * taken directly from lzma_block structure
* after encoding or decoding the Block. * after encoding or decoding the Block.
* *
* Appending a new Record does not affect the read position. * Appending a new Block does not invalidate iterators. For example,
* if an iterator was pointing to the end of the lzma_index, after
* lzma_index_append() it is possible to read the next Block with
* an existing iterator.
* *
* \return - LZMA_OK * \return - LZMA_OK
* - LZMA_MEM_ERROR * - LZMA_MEM_ERROR
@ -145,9 +345,72 @@ extern LZMA_API(lzma_ret) lzma_index_append(
/** /**
* \brief Get the number of Records * \brief Set the Stream Flags
*
* Set the Stream Flags of the last (and typically the only) Stream
* in lzma_index. This can be useful when reading information from the
* lzma_index, because to decode Blocks, knowing the integrity check type
* is needed.
*
* The given Stream Flags are copied into internal preallocated structure
* in the lzma_index, thus the caller doesn't need to keep the *stream_flags
* available after calling this function.
*
* \return - LZMA_OK
* - LZMA_OPTIONS_ERROR: Unsupported stream_flags->version.
* - LZMA_PROG_ERROR
*/ */
extern LZMA_API(lzma_vli) lzma_index_count(const lzma_index *i) extern LZMA_API(lzma_ret) lzma_index_stream_flags(
lzma_index *i, const lzma_stream_flags *stream_flags)
lzma_nothrow lzma_attr_warn_unused_result;
/**
* \brief Get the types of integrity Checks
*
* If lzma_index_stream_padding() is used to set the Stream Flags for
* every Stream, lzma_index_checks() can be used to get a bitmask to
* indicate which Check types have been used. It can be useful e.g. if
* showing the Check types to the user.
*
* The bitmask is 1 << check_id, e.g. CRC32 is 1 << 1 and SHA-256 is 1 << 10.
*/
extern LZMA_API(uint32_t) lzma_index_checks(const lzma_index *i)
lzma_nothrow lzma_attr_pure;
/**
* \brief Set the amount of Stream Padding
*
* Set the amount of Stream Padding of the last (and typically the only)
* Stream in the lzma_index. This is needed when planning to do random-access
* reading within multiple concatenated Streams.
*
* By default, the amount of Stream Padding is assumed to be zero bytes.
*
* \return - LZMA_OK
* - LZMA_DATA_ERROR: The file size would grow too big.
* - LZMA_PROG_ERROR
*/
extern LZMA_API(lzma_ret) lzma_index_stream_padding(
lzma_index *i, lzma_vli stream_padding)
lzma_nothrow lzma_attr_warn_unused_result;
/**
* \brief Get the number of Streams
*/
extern LZMA_API(lzma_vli) lzma_index_stream_count(const lzma_index *i)
lzma_nothrow lzma_attr_pure;
/**
* \brief Get the number of Blocks
*
* This returns the total number of Blocks in lzma_index. To get number
* of Blocks in individual Streams, use lzma_index_iter.
*/
extern LZMA_API(lzma_vli) lzma_index_block_count(const lzma_index *i)
lzma_nothrow lzma_attr_pure; lzma_nothrow lzma_attr_pure;
@ -160,6 +423,17 @@ extern LZMA_API(lzma_vli) lzma_index_size(const lzma_index *i)
lzma_nothrow lzma_attr_pure; lzma_nothrow lzma_attr_pure;
/**
* \brief Get the total size of the Stream
*
* If multiple lzma_indexes have been combined, this works as if the Blocks
* were in a single Stream. This is useful if you are going to combine
* Blocks from multiple Streams into a single new Stream.
*/
extern LZMA_API(lzma_vli) lzma_index_stream_size(const lzma_index *i)
lzma_nothrow lzma_attr_pure;
/** /**
* \brief Get the total size of the Blocks * \brief Get the total size of the Blocks
* *
@ -170,138 +444,145 @@ extern LZMA_API(lzma_vli) lzma_index_total_size(const lzma_index *i)
lzma_nothrow lzma_attr_pure; lzma_nothrow lzma_attr_pure;
/**
* \brief Get the total size of the Stream
*
* If multiple Indexes have been combined, this works as if the Blocks
* were in a single Stream.
*/
extern LZMA_API(lzma_vli) lzma_index_stream_size(const lzma_index *i)
lzma_nothrow lzma_attr_pure;
/** /**
* \brief Get the total size of the file * \brief Get the total size of the file
* *
* When no Indexes have been combined with lzma_index_cat(), this function is * When no lzma_indexes have been combined with lzma_index_cat() and there is
* identical to lzma_index_stream_size(). If multiple Indexes have been * no Stream Padding, this function is identical to lzma_index_stream_size().
* combined, this includes also the headers of each separate Stream and the * If multiple lzma_indexes have been combined, this includes also the headers
* possible Stream Padding fields. * of each separate Stream and the possible Stream Padding fields.
*/ */
extern LZMA_API(lzma_vli) lzma_index_file_size(const lzma_index *i) extern LZMA_API(lzma_vli) lzma_index_file_size(const lzma_index *i)
lzma_nothrow lzma_attr_pure; lzma_nothrow lzma_attr_pure;
/** /**
* \brief Get the uncompressed size of the Stream * \brief Get the uncompressed size of the file
*/ */
extern LZMA_API(lzma_vli) lzma_index_uncompressed_size(const lzma_index *i) extern LZMA_API(lzma_vli) lzma_index_uncompressed_size(const lzma_index *i)
lzma_nothrow lzma_attr_pure; lzma_nothrow lzma_attr_pure;
/** /**
* \brief Get the next Record from the Index * \brief Initialize an iterator
*
* \param iter Pointer to a lzma_index_iter structure
* \param i lzma_index to which the iterator will be associated
*
* This function associates the iterator with the given lzma_index, and calls
* lzma_index_iter_rewind() on the iterator.
*
* This function doesn't allocate any memory, thus there is no
* lzma_index_iter_end(). The iterator is valid as long as the
* associated lzma_index is valid, that is, until lzma_index_end() or
* using it as source in lzma_index_cat(). Specifically, lzma_index doesn't
* become invalid if new Blocks are added to it with lzma_index_append() or
* if it is used as the destionation in lzma_index_cat().
*
* It is safe to make copies of an initialized lzma_index_iter, for example,
* to easily restart reading at some particular position.
*/ */
extern LZMA_API(lzma_bool) lzma_index_read( extern LZMA_API(void) lzma_index_iter_init(
lzma_index *i, lzma_index_record *record) lzma_index_iter *iter, const lzma_index *i) lzma_nothrow;
/**
* \brief Rewind the iterator
*
* Rewind the iterator so that next call to lzma_index_iter_next() will
* return the first Block or Stream.
*/
extern LZMA_API(void) lzma_index_iter_rewind(lzma_index_iter *iter)
lzma_nothrow;
/**
* \brief Get the next Block or Stream
*
* \param iter Iterator initialized with lzma_index_iter_init()
* \param mode Specify what kind of information the caller wants
* to get. See lzma_index_iter_mode for details.
*
* \return If next Block or Stream matching the mode was found, *iter
* is updated and this function returns false. If no Block or
* Stream matching the mode is found, *iter is not modified
* and this function returns true. If mode is set to an unknown
* value, *iter is not modified and this function returns true.
*/
extern LZMA_API(lzma_bool) lzma_index_iter_next(
lzma_index_iter *iter, lzma_index_iter_mode mode)
lzma_nothrow lzma_attr_warn_unused_result; lzma_nothrow lzma_attr_warn_unused_result;
/** /**
* \brief Rewind the Index * \brief Locate a Block
* *
* Rewind the Index so that next call to lzma_index_read() will return the * If it is possible to seek in the .xz file, it is possible to parse
* first Record. * the Index field(s) and use lzma_index_iter_locate() to do random-access
*/ * reading with granularity of Block size.
extern LZMA_API(void) lzma_index_rewind(lzma_index *i) lzma_nothrow;
/**
* \brief Locate a Record
* *
* When the Index is available, it is possible to do random-access reading * \param iter Iterator that was earlier initialized with
* with granularity of Block size. * lzma_index_iter_init().
*
* \param i Pointer to lzma_index structure
* \param record Pointer to a structure to hold the search results
* \param target Uncompressed target offset which the caller would * \param target Uncompressed target offset which the caller would
* like to locate from the Stream * like to locate from the Stream
* *
* If the target is smaller than the uncompressed size of the Stream (can be * If the target is smaller than the uncompressed size of the Stream (can be
* checked with lzma_index_uncompressed_size()): * checked with lzma_index_uncompressed_size()):
* - Information about the Record containing the requested uncompressed * - Information about the Stream and Block containing the requested
* offset is stored into *record. * uncompressed offset is stored into *iter.
* - Read offset will be adjusted so that calling lzma_index_read() can be * - Internal state of the iterator is adjusted so that
* used to read subsequent Records. * lzma_index_iter_next() can be used to read subsequent Blocks or Streams.
* - This function returns false. * - This function returns false.
* *
* If target is greater than the uncompressed size of the Stream, *record * If target is greater than the uncompressed size of the Stream, *iter
* and the read position are not modified, and this function returns true. * is not modified, and this function returns true.
*/ */
extern LZMA_API(lzma_bool) lzma_index_locate( extern LZMA_API(lzma_bool) lzma_index_iter_locate(
lzma_index *i, lzma_index_record *record, lzma_vli target) lzma_index_iter *iter, lzma_vli target) lzma_nothrow;
lzma_nothrow;
/** /**
* \brief Concatenate Indexes of two Streams * \brief Concatenate lzma_indexes
* *
* Concatenating Indexes is useful when doing random-access reading in * Concatenating lzma_indexes is useful when doing random-access reading in
* multi-Stream .xz file, or when combining multiple Streams into single * multi-Stream .xz file, or when combining multiple Streams into single
* Stream. * Stream.
* *
* \param dest Destination Index after which src is appended * \param dest lzma_index after which src is appended
* \param src Source Index. If this function succeeds, the * \param src lzma_index to be appeneded after dest. If this
* memory allocated for src is freed or moved to * function succeeds, the memory allocated for src
* be part of dest. * is freed or moved to be part of dest, and all
* iterators pointing to src will become invalid.
* \param allocator Custom memory allocator; can be NULL to use * \param allocator Custom memory allocator; can be NULL to use
* malloc() and free(). * malloc() and free().
* \param padding Size of the Stream Padding field between Streams.
* This must be a multiple of four.
* *
* \return - LZMA_OK: Indexes concatenated successfully. src is now * \return - LZMA_OK: lzma_indexes were concatenated successfully.
* a dangling pointer. * src is now a dangling pointer.
* - LZMA_DATA_ERROR: *dest would grow too big. * - LZMA_DATA_ERROR: *dest would grow too big.
* - LZMA_MEM_ERROR * - LZMA_MEM_ERROR
* - LZMA_PROG_ERROR * - LZMA_PROG_ERROR
*/ */
extern LZMA_API(lzma_ret) lzma_index_cat(lzma_index *lzma_restrict dest, extern LZMA_API(lzma_ret) lzma_index_cat(lzma_index *lzma_restrict dest,
lzma_index *lzma_restrict src, lzma_index *lzma_restrict src,
lzma_allocator *allocator, lzma_vli padding) lzma_allocator *allocator)
lzma_nothrow lzma_attr_warn_unused_result; lzma_nothrow lzma_attr_warn_unused_result;
/** /**
* \brief Duplicate an Index list * \brief Duplicate lzma_index
* *
* Makes an identical copy of the Index. Also the read position is copied. * \return A copy of the lzma_index, or NULL if memory allocation failed.
*
* \return A copy of the Index, or NULL if memory allocation failed.
*/ */
extern LZMA_API(lzma_index *) lzma_index_dup( extern LZMA_API(lzma_index *) lzma_index_dup(
const lzma_index *i, lzma_allocator *allocator) const lzma_index *i, lzma_allocator *allocator)
lzma_nothrow lzma_attr_warn_unused_result; lzma_nothrow lzma_attr_warn_unused_result;
/**
* \brief Compare if two Index lists are identical
*
* Read positions are not compared.
*
* \return True if *a and *b are equal, false otherwise.
*/
extern LZMA_API(lzma_bool) lzma_index_equal(
const lzma_index *a, const lzma_index *b)
lzma_nothrow lzma_attr_pure;
/** /**
* \brief Initialize .xz Index encoder * \brief Initialize .xz Index encoder
* *
* \param strm Pointer to properly prepared lzma_stream * \param strm Pointer to properly prepared lzma_stream
* \param i Pointer to lzma_index which should be encoded. * \param i Pointer to lzma_index which should be encoded.
* The read position will be at the end of the Index
* after lzma_code() has returned LZMA_STREAM_END.
* *
* The only valid action value for lzma_code() is LZMA_RUN. * The only valid action value for lzma_code() is LZMA_RUN.
* *
@ -309,7 +590,8 @@ extern LZMA_API(lzma_bool) lzma_index_equal(
* - LZMA_MEM_ERROR * - LZMA_MEM_ERROR
* - LZMA_PROG_ERROR * - LZMA_PROG_ERROR
*/ */
extern LZMA_API(lzma_ret) lzma_index_encoder(lzma_stream *strm, lzma_index *i) extern LZMA_API(lzma_ret) lzma_index_encoder(
lzma_stream *strm, const lzma_index *i)
lzma_nothrow lzma_attr_warn_unused_result; lzma_nothrow lzma_attr_warn_unused_result;
@ -322,10 +604,10 @@ extern LZMA_API(lzma_ret) lzma_index_encoder(lzma_stream *strm, lzma_index *i)
* set *i to NULL (the old value is ignored). If * set *i to NULL (the old value is ignored). If
* decoding succeeds (lzma_code() returns * decoding succeeds (lzma_code() returns
* LZMA_STREAM_END), *i will be set to point * LZMA_STREAM_END), *i will be set to point
* to the decoded Index, which the application * to a new lzma_index, which the application
* has to later free with lzma_index_end(). * has to later free with lzma_index_end().
* \param memlimit How much memory the resulting Index is allowed * \param memlimit How much memory the resulting lzma_index is
* to require. * allowed to require.
* *
* The only valid action value for lzma_code() is LZMA_RUN. * The only valid action value for lzma_code() is LZMA_RUN.
* *
@ -349,9 +631,7 @@ extern LZMA_API(lzma_ret) lzma_index_decoder(
/** /**
* \brief Single-call .xz Index encoder * \brief Single-call .xz Index encoder
* *
* \param i Index to be encoded. The read position will be at * \param i lzma_index to be encoded
* the end of the Index if encoding succeeds, or at
* unspecified position in case an error occurs.
* \param out Beginning of the output buffer * \param out Beginning of the output buffer
* \param out_pos The next byte will be written to out[*out_pos]. * \param out_pos The next byte will be written to out[*out_pos].
* *out_pos is updated only if encoding succeeds. * *out_pos is updated only if encoding succeeds.
@ -367,23 +647,23 @@ extern LZMA_API(lzma_ret) lzma_index_decoder(
* \note This function doesn't take allocator argument since all * \note This function doesn't take allocator argument since all
* the internal data is allocated on stack. * the internal data is allocated on stack.
*/ */
extern LZMA_API(lzma_ret) lzma_index_buffer_encode(lzma_index *i, extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i,
uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow;
/** /**
* \brief Single-call .xz Index decoder * \brief Single-call .xz Index decoder
* *
* \param i If decoding succeeds, *i will point to the * \param i If decoding succeeds, *i will point to a new
* decoded Index, which the application has to * lzma_index, which the application has to
* later free with lzma_index_end(). If an error * later free with lzma_index_end(). If an error
* occurs, *i will be NULL. The old value of *i * occurs, *i will be NULL. The old value of *i
* is always ignored and thus doesn't need to be * is always ignored and thus doesn't need to be
* initialized by the caller. * initialized by the caller.
* \param memlimit Pointer to how much memory the resulting Index * \param memlimit Pointer to how much memory the resulting
* is allowed to require. The value pointed by * lzma_index is allowed to require. The value
* this pointer is modified if and only if * pointed by this pointer is modified if and only
* LZMA_MEMLIMIT_ERROR is returned. * if LZMA_MEMLIMIT_ERROR is returned.
* \param allocator Pointer to lzma_allocator, or NULL to use malloc() * \param allocator Pointer to lzma_allocator, or NULL to use malloc()
* \param in Beginning of the input buffer * \param in Beginning of the input buffer
* \param in_pos The next byte will be read from in[*in_pos]. * \param in_pos The next byte will be read from in[*in_pos].

1613
src/liblzma/common/index.c
View File

File diff suppressed because it is too large Load Diff

6
src/liblzma/common/index.h
View File

@ -28,6 +28,12 @@
extern uint32_t lzma_index_padding_size(const lzma_index *i); extern uint32_t lzma_index_padding_size(const lzma_index *i);
/// Set for how many Records to allocate memory the next time
/// lzma_index_append() needs to allocate space for a new Record.
/// This is used only by the Index decoder.
extern void lzma_index_prealloc(lzma_index *i, lzma_vli records);
/// Round the variable-length integer to the next multiple of four. /// Round the variable-length integer to the next multiple of four.
static inline lzma_vli static inline lzma_vli
vli_ceil4(lzma_vli vli) vli_ceil4(lzma_vli vli)

12
src/liblzma/common/index_decoder.c
View File

@ -95,11 +95,15 @@ index_decode(lzma_coder *coder, lzma_allocator *allocator,
// Fall through // Fall through
case SEQ_MEMUSAGE: case SEQ_MEMUSAGE:
if (lzma_index_memusage(coder->count) > coder->memlimit) { if (lzma_index_memusage(1, coder->count) > coder->memlimit) {
ret = LZMA_MEMLIMIT_ERROR; ret = LZMA_MEMLIMIT_ERROR;
goto out; goto out;
} }
// Tell the Index handling code how many Records this
// Index has to allow it to allocate memory more efficiently.
lzma_index_prealloc(coder->index, coder->count);
ret = LZMA_OK; ret = LZMA_OK;
coder->sequence = coder->count == 0 coder->sequence = coder->count == 0
? SEQ_PADDING_INIT : SEQ_UNPADDED; ? SEQ_PADDING_INIT : SEQ_UNPADDED;
@ -214,7 +218,7 @@ static lzma_ret
index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage,
uint64_t *old_memlimit, uint64_t new_memlimit) uint64_t *old_memlimit, uint64_t new_memlimit)
{ {
*memusage = lzma_index_memusage(coder->count); *memusage = lzma_index_memusage(1, coder->count);
if (new_memlimit != 0 && new_memlimit < *memusage) if (new_memlimit != 0 && new_memlimit < *memusage)
return LZMA_MEMLIMIT_ERROR; return LZMA_MEMLIMIT_ERROR;
@ -238,7 +242,7 @@ index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator,
*i = NULL; *i = NULL;
// We always allocate a new lzma_index. // We always allocate a new lzma_index.
coder->index = lzma_index_init(NULL, allocator); coder->index = lzma_index_init(allocator);
if (coder->index == NULL) if (coder->index == NULL)
return LZMA_MEM_ERROR; return LZMA_MEM_ERROR;
@ -329,7 +333,7 @@ lzma_index_buffer_decode(
} else if (ret == LZMA_MEMLIMIT_ERROR) { } else if (ret == LZMA_MEMLIMIT_ERROR) {
// Tell the caller how much memory would have // Tell the caller how much memory would have
// been needed. // been needed.
*memlimit = lzma_index_memusage(coder.count); *memlimit = lzma_index_memusage(1, coder.count);
} }
} }

36
src/liblzma/common/index_encoder.c
View File

@ -26,12 +26,11 @@ struct lzma_coder_s {
SEQ_CRC32, SEQ_CRC32,
} sequence; } sequence;
/// Index given to us to encode. Note that we modify it in sense that /// Index being encoded
/// we read it, and read position is tracked in lzma_index structure. const lzma_index *index;
lzma_index *index;
/// The current Index Record being encoded /// Iterator for the Index being encoded
lzma_index_record record; lzma_index_iter iter;
/// Position in integers /// Position in integers
size_t pos; size_t pos;
@ -69,8 +68,8 @@ index_encode(lzma_coder *coder,
break; break;
case SEQ_COUNT: { case SEQ_COUNT: {
const lzma_vli index_count = lzma_index_count(coder->index); const lzma_vli count = lzma_index_block_count(coder->index);
ret = lzma_vli_encode(index_count, &coder->pos, ret = lzma_vli_encode(count, &coder->pos,
out, out_pos, out_size); out, out_pos, out_size);
if (ret != LZMA_STREAM_END) if (ret != LZMA_STREAM_END)
goto out; goto out;
@ -82,7 +81,8 @@ index_encode(lzma_coder *coder,
} }
case SEQ_NEXT: case SEQ_NEXT:
if (lzma_index_read(coder->index, &coder->record)) { if (lzma_index_iter_next(
&coder->iter, LZMA_INDEX_ITER_BLOCK)) {
// Get the size of the Index Padding field. // Get the size of the Index Padding field.
coder->pos = lzma_index_padding_size(coder->index); coder->pos = lzma_index_padding_size(coder->index);
assert(coder->pos <= 3); assert(coder->pos <= 3);
@ -90,12 +90,6 @@ index_encode(lzma_coder *coder,
break; break;
} }
// Unpadded Size must be within valid limits.
if (coder->record.unpadded_size < UNPADDED_SIZE_MIN
|| coder->record.unpadded_size
> UNPADDED_SIZE_MAX)
return LZMA_PROG_ERROR;
coder->sequence = SEQ_UNPADDED; coder->sequence = SEQ_UNPADDED;
// Fall through // Fall through
@ -103,8 +97,8 @@ index_encode(lzma_coder *coder,
case SEQ_UNPADDED: case SEQ_UNPADDED:
case SEQ_UNCOMPRESSED: { case SEQ_UNCOMPRESSED: {
const lzma_vli size = coder->sequence == SEQ_UNPADDED const lzma_vli size = coder->sequence == SEQ_UNPADDED
? coder->record.unpadded_size ? coder->iter.block.unpadded_size
: coder->record.uncompressed_size; : coder->iter.block.uncompressed_size;
ret = lzma_vli_encode(size, &coder->pos, ret = lzma_vli_encode(size, &coder->pos,
out, out_pos, out_size); out, out_pos, out_size);
@ -172,9 +166,9 @@ index_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
static void static void
index_encoder_reset(lzma_coder *coder, lzma_index *i) index_encoder_reset(lzma_coder *coder, const lzma_index *i)
{ {
lzma_index_rewind(i); lzma_index_iter_init(&coder->iter, i);
coder->sequence = SEQ_INDICATOR; coder->sequence = SEQ_INDICATOR;
coder->index = i; coder->index = i;
@ -187,7 +181,7 @@ index_encoder_reset(lzma_coder *coder, lzma_index *i)
extern lzma_ret extern lzma_ret
lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
lzma_index *i) const lzma_index *i)
{ {
lzma_next_coder_init(&lzma_index_encoder_init, next, allocator); lzma_next_coder_init(&lzma_index_encoder_init, next, allocator);
@ -210,7 +204,7 @@ lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
extern LZMA_API(lzma_ret) extern LZMA_API(lzma_ret)
lzma_index_encoder(lzma_stream *strm, lzma_index *i) lzma_index_encoder(lzma_stream *strm, const lzma_index *i)
{ {
lzma_next_strm_init(lzma_index_encoder_init, strm, i); lzma_next_strm_init(lzma_index_encoder_init, strm, i);
@ -221,7 +215,7 @@ lzma_index_encoder(lzma_stream *strm, lzma_index *i)
extern LZMA_API(lzma_ret) extern LZMA_API(lzma_ret)
lzma_index_buffer_encode(lzma_index *i, lzma_index_buffer_encode(const lzma_index *i,
uint8_t *out, size_t *out_pos, size_t out_size) uint8_t *out, size_t *out_pos, size_t out_size)
{ {
// Validate the arugments. // Validate the arugments.

2
src/liblzma/common/index_encoder.h
View File

@ -17,7 +17,7 @@
extern lzma_ret lzma_index_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_index_encoder_init(lzma_next_coder *next,
lzma_allocator *allocator, lzma_index *i); lzma_allocator *allocator, const lzma_index *i);
#endif #endif

6
src/liblzma/common/stream_buffer_encoder.c
View File

@ -94,11 +94,11 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check,
// Index // Index
{ {
// Create an Index with one Record. // Create an Index with one Record.
lzma_index *i = lzma_index_init(NULL, NULL); lzma_index *i = lzma_index_init(allocator);
if (i == NULL) if (i == NULL)
return LZMA_MEM_ERROR; return LZMA_MEM_ERROR;
lzma_ret ret = lzma_index_append(i, NULL, lzma_ret ret = lzma_index_append(i, allocator,
lzma_block_unpadded_size(&block), lzma_block_unpadded_size(&block),
block.uncompressed_size); block.uncompressed_size);
@ -111,7 +111,7 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check,
stream_flags.backward_size = lzma_index_size(i); stream_flags.backward_size = lzma_index_size(i);
} }
lzma_index_end(i, NULL); lzma_index_end(i, allocator);
if (ret != LZMA_OK) if (ret != LZMA_OK)
return ret; return ret;

3
src/liblzma/common/stream_encoder.c
View File

@ -292,7 +292,8 @@ lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
next->coder->filters[0].id = LZMA_VLI_UNKNOWN; next->coder->filters[0].id = LZMA_VLI_UNKNOWN;
// Initialize the Index // Initialize the Index
next->coder->index = lzma_index_init(next->coder->index, allocator); lzma_index_end(next->coder->index, allocator);
next->coder->index = lzma_index_init(allocator);
if (next->coder->index == NULL) if (next->coder->index == NULL)
return LZMA_MEM_ERROR; return LZMA_MEM_ERROR;

359
tests/test_index.c
View File

@ -21,7 +21,7 @@
static lzma_index * static lzma_index *
create_empty(void) create_empty(void)
{ {
lzma_index *i = lzma_index_init(NULL, NULL); lzma_index *i = lzma_index_init(NULL);
expect(i != NULL); expect(i != NULL);
return i; return i;
} }
@ -30,7 +30,7 @@ create_empty(void)
static lzma_index * static lzma_index *
create_small(void) create_small(void)
{ {
lzma_index *i = lzma_index_init(NULL, NULL); lzma_index *i = lzma_index_init(NULL);
expect(i != NULL); expect(i != NULL);
expect(lzma_index_append(i, NULL, 101, 555) == LZMA_OK); expect(lzma_index_append(i, NULL, 101, 555) == LZMA_OK);
expect(lzma_index_append(i, NULL, 602, 777) == LZMA_OK); expect(lzma_index_append(i, NULL, 602, 777) == LZMA_OK);
@ -42,7 +42,7 @@ create_small(void)
static lzma_index * static lzma_index *
create_big(void) create_big(void)
{ {
lzma_index *i = lzma_index_init(NULL, NULL); lzma_index *i = lzma_index_init(NULL);
expect(i != NULL); expect(i != NULL);
lzma_vli total_size = 0; lzma_vli total_size = 0;
@ -58,7 +58,7 @@ create_big(void)
uncompressed_size += n; uncompressed_size += n;
} }
expect(lzma_index_count(i) == BIG_COUNT); expect(lzma_index_block_count(i) == BIG_COUNT);
expect(lzma_index_total_size(i) == total_size); expect(lzma_index_total_size(i) == total_size);
expect(lzma_index_uncompressed_size(i) == uncompressed_size); expect(lzma_index_uncompressed_size(i) == uncompressed_size);
expect(lzma_index_total_size(i) + lzma_index_size(i) expect(lzma_index_total_size(i) + lzma_index_size(i)
@ -69,6 +69,60 @@ create_big(void)
} }
static bool
is_equal(const lzma_index *a, const lzma_index *b)
{
// Compare only the Stream and Block sizes and offsets.
lzma_index_iter ra, rb;
lzma_index_iter_init(&ra, a);
lzma_index_iter_init(&rb, b);
while (true) {
bool reta = lzma_index_iter_next(&ra, LZMA_INDEX_ITER_ANY);
bool retb = lzma_index_iter_next(&rb, LZMA_INDEX_ITER_ANY);
if (reta)
return !(reta ^ retb);
if (ra.stream.number != rb.stream.number
|| ra.stream.block_count
!= rb.stream.block_count
|| ra.stream.compressed_offset
!= rb.stream.compressed_offset
|| ra.stream.uncompressed_offset
!= rb.stream.uncompressed_offset
|| ra.stream.compressed_size
!= rb.stream.compressed_size
|| ra.stream.uncompressed_size
!= rb.stream.uncompressed_size
|| ra.stream.padding
!= rb.stream.padding)
return false;
if (ra.stream.block_count == 0)
continue;
if (ra.block.number_in_file != rb.block.number_in_file
|| ra.block.compressed_file_offset
!= rb.block.compressed_file_offset
|| ra.block.uncompressed_file_offset
!= rb.block.uncompressed_file_offset
|| ra.block.number_in_stream
!= rb.block.number_in_stream
|| ra.block.compressed_stream_offset
!= rb.block.compressed_stream_offset
|| ra.block.uncompressed_stream_offset
!= rb.block.uncompressed_stream_offset
|| ra.block.uncompressed_size
!= rb.block.uncompressed_size
|| ra.block.unpadded_size
!= rb.block.unpadded_size
|| ra.block.total_size
!= rb.block.total_size)
return false;
}
}
static void static void
test_equal(void) test_equal(void)
{ {
@ -77,13 +131,13 @@ test_equal(void)
lzma_index *c = create_big(); lzma_index *c = create_big();
expect(a && b && c); expect(a && b && c);
expect(lzma_index_equal(a, a)); expect(is_equal(a, a));
expect(lzma_index_equal(b, b)); expect(is_equal(b, b));
expect(lzma_index_equal(c, c)); expect(is_equal(c, c));
expect(!lzma_index_equal(a, b)); expect(!is_equal(a, b));
expect(!lzma_index_equal(a, c)); expect(!is_equal(a, c));
expect(!lzma_index_equal(b, c)); expect(!is_equal(b, c));
lzma_index_end(a, NULL); lzma_index_end(a, NULL);
lzma_index_end(b, NULL); lzma_index_end(b, NULL);
@ -111,6 +165,7 @@ test_copy(const lzma_index *i)
{ {
lzma_index *d = lzma_index_dup(i, NULL); lzma_index *d = lzma_index_dup(i, NULL);
expect(d != NULL); expect(d != NULL);
expect(is_equal(i, d));
lzma_index_end(d, NULL); lzma_index_end(d, NULL);
} }
@ -118,7 +173,8 @@ test_copy(const lzma_index *i)
static void static void
test_read(lzma_index *i) test_read(lzma_index *i)
{ {
lzma_index_record record; lzma_index_iter r;
lzma_index_iter_init(&r, i);
// Try twice so we see that rewinding works. // Try twice so we see that rewinding works.
for (size_t j = 0; j < 2; ++j) { for (size_t j = 0; j < 2; ++j) {
@ -128,25 +184,26 @@ test_read(lzma_index *i)
lzma_vli uncompressed_offset = 0; lzma_vli uncompressed_offset = 0;
uint32_t count = 0; uint32_t count = 0;
while (!lzma_index_read(i, &record)) { while (!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)) {
++count; ++count;
total_size += record.total_size; total_size += r.block.total_size;
uncompressed_size += record.uncompressed_size; uncompressed_size += r.block.uncompressed_size;
expect(record.stream_offset == stream_offset); expect(r.block.compressed_file_offset
expect(record.uncompressed_offset == stream_offset);
expect(r.block.uncompressed_file_offset
== uncompressed_offset); == uncompressed_offset);
stream_offset += record.total_size; stream_offset += r.block.total_size;
uncompressed_offset += record.uncompressed_size; uncompressed_offset += r.block.uncompressed_size;
} }
expect(lzma_index_total_size(i) == total_size); expect(lzma_index_total_size(i) == total_size);
expect(lzma_index_uncompressed_size(i) == uncompressed_size); expect(lzma_index_uncompressed_size(i) == uncompressed_size);
expect(lzma_index_count(i) == count); expect(lzma_index_block_count(i) == count);
lzma_index_rewind(i); lzma_index_iter_rewind(&r);
} }
} }
@ -171,7 +228,7 @@ test_code(lzma_index *i)
expect(d == NULL); expect(d == NULL);
succeed(decoder_loop(&strm, buf, index_size)); succeed(decoder_loop(&strm, buf, index_size));
expect(lzma_index_equal(i, d)); expect(is_equal(i, d));
lzma_index_end(d, NULL); lzma_index_end(d, NULL);
lzma_end(&strm); lzma_end(&strm);
@ -179,11 +236,11 @@ test_code(lzma_index *i)
// Decode with hashing // Decode with hashing
lzma_index_hash *h = lzma_index_hash_init(NULL, NULL); lzma_index_hash *h = lzma_index_hash_init(NULL, NULL);
expect(h != NULL); expect(h != NULL);
lzma_index_rewind(i); lzma_index_iter r;
lzma_index_record r; lzma_index_iter_init(&r, i);
while (!lzma_index_read(i, &r)) while (!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK))
expect(lzma_index_hash_append(h, r.unpadded_size, expect(lzma_index_hash_append(h, r.block.unpadded_size,
r.uncompressed_size) == LZMA_OK); r.block.uncompressed_size) == LZMA_OK);
size_t pos = 0; size_t pos = 0;
while (pos < index_size - 1) while (pos < index_size - 1)
expect(lzma_index_hash_decode(h, buf, &pos, pos + 1) expect(lzma_index_hash_decode(h, buf, &pos, pos + 1)
@ -213,7 +270,7 @@ test_code(lzma_index *i)
succeed(lzma_index_buffer_decode(&d, &memlimit, NULL, buf, &buf_pos, succeed(lzma_index_buffer_decode(&d, &memlimit, NULL, buf, &buf_pos,
index_size + 1)); index_size + 1));
expect(buf_pos == index_size + 1); expect(buf_pos == index_size + 1);
expect(lzma_index_equal(i, d)); expect(is_equal(i, d));
lzma_index_end(d, NULL); lzma_index_end(d, NULL);
@ -234,71 +291,99 @@ static void
test_cat(void) test_cat(void)
{ {
lzma_index *a, *b, *c; lzma_index *a, *b, *c;
lzma_index_record r; lzma_index_iter r;
// Empty Indexes // Empty Indexes
a = create_empty(); a = create_empty();
b = create_empty(); b = create_empty();
expect(lzma_index_cat(a, b, NULL, 0) == LZMA_OK); expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_count(a) == 0); expect(lzma_index_block_count(a) == 0);
expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8); expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8);
expect(lzma_index_file_size(a) expect(lzma_index_file_size(a)
== 2 * (2 * LZMA_STREAM_HEADER_SIZE + 8)); == 2 * (2 * LZMA_STREAM_HEADER_SIZE + 8));
expect(lzma_index_read(a, &r)); lzma_index_iter_init(&r, a);
expect(lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK));
b = create_empty(); b = create_empty();
expect(lzma_index_cat(a, b, NULL, 0) == LZMA_OK); expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_count(a) == 0); expect(lzma_index_block_count(a) == 0);
expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8); expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8);
expect(lzma_index_file_size(a) expect(lzma_index_file_size(a)
== 3 * (2 * LZMA_STREAM_HEADER_SIZE + 8)); == 3 * (2 * LZMA_STREAM_HEADER_SIZE + 8));
b = create_empty(); b = create_empty();
c = create_empty(); c = create_empty();
expect(lzma_index_cat(b, c, NULL, 4) == LZMA_OK); expect(lzma_index_stream_padding(b, 4) == LZMA_OK);
expect(lzma_index_count(b) == 0); expect(lzma_index_cat(b, c, NULL) == LZMA_OK);
expect(lzma_index_block_count(b) == 0);
expect(lzma_index_stream_size(b) == 2 * LZMA_STREAM_HEADER_SIZE + 8); expect(lzma_index_stream_size(b) == 2 * LZMA_STREAM_HEADER_SIZE + 8);
expect(lzma_index_file_size(b) expect(lzma_index_file_size(b)
== 2 * (2 * LZMA_STREAM_HEADER_SIZE + 8) + 4); == 2 * (2 * LZMA_STREAM_HEADER_SIZE + 8) + 4);
expect(lzma_index_cat(a, b, NULL, 8) == LZMA_OK); expect(lzma_index_stream_padding(a, 8) == LZMA_OK);
expect(lzma_index_count(a) == 0); expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_block_count(a) == 0);
expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8); expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8);
expect(lzma_index_file_size(a) expect(lzma_index_file_size(a)
== 5 * (2 * LZMA_STREAM_HEADER_SIZE + 8) + 4 + 8); == 5 * (2 * LZMA_STREAM_HEADER_SIZE + 8) + 4 + 8);
expect(lzma_index_read(a, &r)); expect(lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK));
lzma_index_rewind(a); lzma_index_iter_rewind(&r);
expect(lzma_index_read(a, &r)); expect(lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK));
lzma_index_end(a, NULL); lzma_index_end(a, NULL);
// Small Indexes // Small Indexes
a = create_small(); a = create_small();
lzma_vli stream_size = lzma_index_stream_size(a); lzma_vli stream_size = lzma_index_stream_size(a);
lzma_index_iter_init(&r, a);
for (int i = SMALL_COUNT; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
b = create_small(); b = create_small();
expect(lzma_index_cat(a, b, NULL, 4) == LZMA_OK); expect(lzma_index_stream_padding(a, 4) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_file_size(a) == stream_size * 2 + 4); expect(lzma_index_file_size(a) == stream_size * 2 + 4);
expect(lzma_index_stream_size(a) > stream_size); expect(lzma_index_stream_size(a) > stream_size);
expect(lzma_index_stream_size(a) < stream_size * 2); expect(lzma_index_stream_size(a) < stream_size * 2);
for (int i = SMALL_COUNT; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_iter_rewind(&r);
for (int i = SMALL_COUNT * 2; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
b = create_small(); b = create_small();
c = create_small(); c = create_small();
expect(lzma_index_cat(b, c, NULL, 8) == LZMA_OK); expect(lzma_index_stream_padding(b, 8) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL, 12) == LZMA_OK); expect(lzma_index_cat(b, c, NULL) == LZMA_OK);
expect(lzma_index_stream_padding(a, 12) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_file_size(a) == stream_size * 4 + 4 + 8 + 12); expect(lzma_index_file_size(a) == stream_size * 4 + 4 + 8 + 12);
expect(lzma_index_count(a) == SMALL_COUNT * 4); expect(lzma_index_block_count(a) == SMALL_COUNT * 4);
for (int i = SMALL_COUNT * 2; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_iter_rewind(&r);
for (int i = SMALL_COUNT * 4; i >= 0; --i) for (int i = SMALL_COUNT * 4; i >= 0; --i)
expect(!lzma_index_read(a, &r) ^ (i == 0)); expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_end(a, NULL); lzma_index_end(a, NULL);
// Mix of empty and small // Mix of empty and small
a = create_empty(); a = create_empty();
b = create_small(); b = create_small();
expect(lzma_index_cat(a, b, NULL, 4) == LZMA_OK); expect(lzma_index_stream_padding(a, 4) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
lzma_index_iter_init(&r, a);
for (int i = SMALL_COUNT; i >= 0; --i) for (int i = SMALL_COUNT; i >= 0; --i)
expect(!lzma_index_read(a, &r) ^ (i == 0)); expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_end(a, NULL); lzma_index_end(a, NULL);
@ -306,19 +391,24 @@ test_cat(void)
a = create_big(); a = create_big();
stream_size = lzma_index_stream_size(a); stream_size = lzma_index_stream_size(a);
b = create_big(); b = create_big();
expect(lzma_index_cat(a, b, NULL, 4) == LZMA_OK); expect(lzma_index_stream_padding(a, 4) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_file_size(a) == stream_size * 2 + 4); expect(lzma_index_file_size(a) == stream_size * 2 + 4);
expect(lzma_index_stream_size(a) > stream_size); expect(lzma_index_stream_size(a) > stream_size);
expect(lzma_index_stream_size(a) < stream_size * 2); expect(lzma_index_stream_size(a) < stream_size * 2);
b = create_big(); b = create_big();
c = create_big(); c = create_big();
expect(lzma_index_cat(b, c, NULL, 8) == LZMA_OK); expect(lzma_index_stream_padding(b, 8) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL, 12) == LZMA_OK); expect(lzma_index_cat(b, c, NULL) == LZMA_OK);
expect(lzma_index_stream_padding(a, 12) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_file_size(a) == stream_size * 4 + 4 + 8 + 12); expect(lzma_index_file_size(a) == stream_size * 4 + 4 + 8 + 12);
lzma_index_iter_init(&r, a);
for (int i = BIG_COUNT * 4; i >= 0; --i) for (int i = BIG_COUNT * 4; i >= 0; --i)
expect(!lzma_index_read(a, &r) ^ (i == 0)); expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_end(a, NULL); lzma_index_end(a, NULL);
} }
@ -327,102 +417,113 @@ test_cat(void)
static void static void
test_locate(void) test_locate(void)
{ {
lzma_index_record r; lzma_index *i = lzma_index_init(NULL);
lzma_index *i = lzma_index_init(NULL, NULL);
expect(i != NULL); expect(i != NULL);
lzma_index_iter r;
lzma_index_iter_init(&r, i);
// Cannot locate anything from an empty Index. // Cannot locate anything from an empty Index.
expect(lzma_index_locate(i, &r, 0)); expect(lzma_index_iter_locate(&r, 0));
expect(lzma_index_locate(i, &r, 555)); expect(lzma_index_iter_locate(&r, 555));
// One empty Record: nothing is found since there's no uncompressed // One empty Record: nothing is found since there's no uncompressed
// data. // data.
expect(lzma_index_append(i, NULL, 16, 0) == LZMA_OK); expect(lzma_index_append(i, NULL, 16, 0) == LZMA_OK);
expect(lzma_index_locate(i, &r, 0)); expect(lzma_index_iter_locate(&r, 0));
// Non-empty Record and we can find something. // Non-empty Record and we can find something.
expect(lzma_index_append(i, NULL, 32, 5) == LZMA_OK); expect(lzma_index_append(i, NULL, 32, 5) == LZMA_OK);
expect(!lzma_index_locate(i, &r, 0)); expect(!lzma_index_iter_locate(&r, 0));
expect(r.total_size == 32); expect(r.block.total_size == 32);
expect(r.uncompressed_size == 5); expect(r.block.uncompressed_size == 5);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 16); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 0); == LZMA_STREAM_HEADER_SIZE + 16);
expect(r.block.uncompressed_file_offset == 0);
// Still cannot find anything past the end. // Still cannot find anything past the end.
expect(lzma_index_locate(i, &r, 5)); expect(lzma_index_iter_locate(&r, 5));
// Add the third Record. // Add the third Record.
expect(lzma_index_append(i, NULL, 40, 11) == LZMA_OK); expect(lzma_index_append(i, NULL, 40, 11) == LZMA_OK);
expect(!lzma_index_locate(i, &r, 0)); expect(!lzma_index_iter_locate(&r, 0));
expect(r.total_size == 32); expect(r.block.total_size == 32);
expect(r.uncompressed_size == 5); expect(r.block.uncompressed_size == 5);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 16); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 0); == LZMA_STREAM_HEADER_SIZE + 16);
expect(r.block.uncompressed_file_offset == 0);
expect(!lzma_index_read(i, &r)); expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK));
expect(r.total_size == 40); expect(r.block.total_size == 40);
expect(r.uncompressed_size == 11); expect(r.block.uncompressed_size == 11);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 16 + 32); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 5); == LZMA_STREAM_HEADER_SIZE + 16 + 32);
expect(r.block.uncompressed_file_offset == 5);
expect(!lzma_index_locate(i, &r, 2)); expect(!lzma_index_iter_locate(&r, 2));
expect(r.total_size == 32); expect(r.block.total_size == 32);
expect(r.uncompressed_size == 5); expect(r.block.uncompressed_size == 5);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 16); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 0); == LZMA_STREAM_HEADER_SIZE + 16);
expect(r.block.uncompressed_file_offset == 0);
expect(!lzma_index_locate(i, &r, 5)); expect(!lzma_index_iter_locate(&r, 5));
expect(r.total_size == 40); expect(r.block.total_size == 40);
expect(r.uncompressed_size == 11); expect(r.block.uncompressed_size == 11);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 16 + 32); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 5); == LZMA_STREAM_HEADER_SIZE + 16 + 32);
expect(r.block.uncompressed_file_offset == 5);
expect(!lzma_index_locate(i, &r, 5 + 11 - 1)); expect(!lzma_index_iter_locate(&r, 5 + 11 - 1));
expect(r.total_size == 40); expect(r.block.total_size == 40);
expect(r.uncompressed_size == 11); expect(r.block.uncompressed_size == 11);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 16 + 32); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 5); == LZMA_STREAM_HEADER_SIZE + 16 + 32);
expect(r.block.uncompressed_file_offset == 5);
expect(lzma_index_locate(i, &r, 5 + 11)); expect(lzma_index_iter_locate(&r, 5 + 11));
expect(lzma_index_locate(i, &r, 5 + 15)); expect(lzma_index_iter_locate(&r, 5 + 15));
// Large Index // Large Index
i = lzma_index_init(i, NULL); lzma_index_end(i, NULL);
i = lzma_index_init(NULL);
expect(i != NULL); expect(i != NULL);
lzma_index_iter_init(&r, i);
for (size_t n = 4; n <= 4 * 5555; n += 4) for (size_t n = 4; n <= 4 * 5555; n += 4)
expect(lzma_index_append(i, NULL, n + 8, n) == LZMA_OK); expect(lzma_index_append(i, NULL, n + 8, n) == LZMA_OK);
expect(lzma_index_count(i) == 5555); expect(lzma_index_block_count(i) == 5555);
// First Record // First Record
expect(!lzma_index_locate(i, &r, 0)); expect(!lzma_index_iter_locate(&r, 0));
expect(r.total_size == 4 + 8); expect(r.block.total_size == 4 + 8);
expect(r.uncompressed_size == 4); expect(r.block.uncompressed_size == 4);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE);
expect(r.uncompressed_offset == 0); expect(r.block.uncompressed_file_offset == 0);
expect(!lzma_index_locate(i, &r, 3)); expect(!lzma_index_iter_locate(&r, 3));
expect(r.total_size == 4 + 8); expect(r.block.total_size == 4 + 8);
expect(r.uncompressed_size == 4); expect(r.block.uncompressed_size == 4);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE);
expect(r.uncompressed_offset == 0); expect(r.block.uncompressed_file_offset == 0);
// Second Record // Second Record
expect(!lzma_index_locate(i, &r, 4)); expect(!lzma_index_iter_locate(&r, 4));
expect(r.total_size == 2 * 4 + 8); expect(r.block.total_size == 2 * 4 + 8);
expect(r.uncompressed_size == 2 * 4); expect(r.block.uncompressed_size == 2 * 4);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 4 + 8); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 4); == LZMA_STREAM_HEADER_SIZE + 4 + 8);
expect(r.block.uncompressed_file_offset == 4);
// Last Record // Last Record
expect(!lzma_index_locate(i, &r, lzma_index_uncompressed_size(i) - 1)); expect(!lzma_index_iter_locate(
expect(r.total_size == 4 * 5555 + 8); &r, lzma_index_uncompressed_size(i) - 1));
expect(r.uncompressed_size == 4 * 5555); expect(r.block.total_size == 4 * 5555 + 8);
expect(r.stream_offset == lzma_index_total_size(i) expect(r.block.uncompressed_size == 4 * 5555);
expect(r.block.compressed_file_offset == lzma_index_total_size(i)
+ LZMA_STREAM_HEADER_SIZE - 4 * 5555 - 8); + LZMA_STREAM_HEADER_SIZE - 4 * 5555 - 8);
expect(r.uncompressed_offset expect(r.block.uncompressed_file_offset
== lzma_index_uncompressed_size(i) - 4 * 5555); == lzma_index_uncompressed_size(i) - 4 * 5555);
// Allocation chunk boundaries. See INDEX_GROUP_SIZE in // Allocation chunk boundaries. See INDEX_GROUP_SIZE in
@ -439,48 +540,50 @@ test_locate(void)
} }
while (n < start + 2 * radius) { while (n < start + 2 * radius) {
expect(!lzma_index_locate(i, &r, ubase + n * 4)); expect(!lzma_index_iter_locate(&r, ubase + n * 4));
expect(r.stream_offset == tbase + n * 4 + 8 expect(r.block.compressed_file_offset == tbase + n * 4 + 8
+ LZMA_STREAM_HEADER_SIZE); + LZMA_STREAM_HEADER_SIZE);
expect(r.uncompressed_offset == ubase + n * 4); expect(r.block.uncompressed_file_offset == ubase + n * 4);
tbase += n * 4 + 8; tbase += n * 4 + 8;
ubase += n * 4; ubase += n * 4;
++n; ++n;
expect(r.total_size == n * 4 + 8); expect(r.block.total_size == n * 4 + 8);
expect(r.uncompressed_size == n * 4); expect(r.block.uncompressed_size == n * 4);
} }
// Do it also backwards since lzma_index_locate() uses relative search. // Do it also backwards.
while (n > start) { while (n > start) {
expect(!lzma_index_locate(i, &r, ubase + (n - 1) * 4)); expect(!lzma_index_iter_locate(&r, ubase + (n - 1) * 4));
expect(r.total_size == n * 4 + 8); expect(r.block.total_size == n * 4 + 8);
expect(r.uncompressed_size == n * 4); expect(r.block.uncompressed_size == n * 4);
--n; --n;
tbase -= n * 4 + 8; tbase -= n * 4 + 8;
ubase -= n * 4; ubase -= n * 4;
expect(r.stream_offset == tbase + n * 4 + 8 expect(r.block.compressed_file_offset == tbase + n * 4 + 8
+ LZMA_STREAM_HEADER_SIZE); + LZMA_STREAM_HEADER_SIZE);
expect(r.uncompressed_offset == ubase + n * 4); expect(r.block.uncompressed_file_offset == ubase + n * 4);
} }
// Test locating in concatend Index. // Test locating in concatend Index.
i = lzma_index_init(i, NULL); lzma_index_end(i, NULL);
i = lzma_index_init(NULL);
expect(i != NULL); expect(i != NULL);
lzma_index_iter_init(&r, i);
for (n = 0; n < group_multiple; ++n) for (n = 0; n < group_multiple; ++n)
expect(lzma_index_append(i, NULL, 8, 0) == LZMA_OK); expect(lzma_index_append(i, NULL, 8, 0) == LZMA_OK);
expect(lzma_index_append(i, NULL, 16, 1) == LZMA_OK); expect(lzma_index_append(i, NULL, 16, 1) == LZMA_OK);
expect(!lzma_index_locate(i, &r, 0)); expect(!lzma_index_iter_locate(&r, 0));
expect(r.total_size == 16); expect(r.block.total_size == 16);
expect(r.uncompressed_size == 1); expect(r.block.uncompressed_size == 1);
expect(r.stream_offset expect(r.block.compressed_file_offset
== LZMA_STREAM_HEADER_SIZE + group_multiple * 8); == LZMA_STREAM_HEADER_SIZE + group_multiple * 8);
expect(r.uncompressed_offset == 0); expect(r.block.uncompressed_file_offset == 0);
lzma_index_end(i, NULL); lzma_index_end(i, NULL);
} }