Revised the Index handling code.

This breaks API and ABI but most apps are not affected
since most apps don't use this part of the API. You will
get a compile error if you are using anything that got
broken.

Summary of changes:

  - Ability to store Stream Flags, which are needed
    for random-access reading in multi-Stream files.

  - Separate function to set size of Stream Padding.

  - Iterator structure makes it possible to read the same
    lzma_index from multiple threads at the same time.

  - A lot faster code to locate Blocks.

  - Removed lzma_index_equal() without adding anything
    to replace it. I don't know what it should do exactly
    with the new features and what actually needs this
    function in the first place other than test_index.c,
    which now has its own code to compare lzma_indexes.
This commit is contained in:
Lasse Collin 2009-12-31 22:45:53 +02:00
parent f29997a846
commit 1a7ec87c8e
9 changed files with 1733 additions and 888 deletions

View File

@ -1,6 +1,6 @@
/** /**
* \file lzma/index.h * \file lzma/index.h
* \brief Handling of .xz Index lists * \brief Handling of .xz Index and related information
*/ */
/* /*
@ -18,98 +18,295 @@
/** /**
* \brief Opaque data type to hold the Index * \brief Opaque data type to hold the Index(es) and other information
*
* lzma_index often holds just one .xz Index and possibly the Stream Flags
* of the same Stream and size of the Stream Padding field. However,
* multiple lzma_indexes can be concatenated with lzma_index_cat() and then
* there may be information about multiple Streams in the same lzma_index.
*
* Notes about thread safety: Only one thread may modify lzma_index at
* a time. All functions that take non-const pointer to lzma_index
* modify it. As long as no thread is modifying the lzma_index, getting
* information from the same lzma_index can be done from multiple threads
* at the same time with functions that take a const pointer to
* lzma_index or use lzma_index_iter. The same iterator must be used
* only by one thread at a time, of course, but there can be as many
* iterators for the same lzma_index as needed.
*/ */
typedef struct lzma_index_s lzma_index; typedef struct lzma_index_s lzma_index;
/** /**
* \brief Index Record and its location * \brief Iterator to get information about Blocks and Streams
*/ */
typedef struct { typedef struct {
struct {
/** /**
* \brief Total encoded size of a Block including Block Padding * \brief Pointer to Stream Flags
* *
* This value is useful if you need to know the actual size of the * This is NULL if Stream Flags have not been set for
* Block that the Block decoder will read. * this Stream with lzma_index_stream_flags().
*/ */
lzma_vli total_size; const lzma_stream_flags *flags;
const void *reserved_ptr1;
const void *reserved_ptr2;
const void *reserved_ptr3;
/** /**
* \brief Encoded size of a Block excluding Block Padding * \brief Stream number in the lzma_index
* *
* This value is stored in the Index. When doing random-access * The first Stream is 1.
* reading, you should give this value to the Block decoder along
* with uncompressed_size.
*/ */
lzma_vli unpadded_size; lzma_vli number;
/** /**
* \brief Uncompressed Size of a Block * \brief Number of Blocks in the Stream
*
* If this is zero, the block structure below has
* undefined values.
*/
lzma_vli block_count;
/**
* \brief Compressed start offset of this Stream
*
* The offset is relative to the beginning of the lzma_index
* (i.e. usually the beginning of the .xz file).
*/
lzma_vli compressed_offset;
/**
* \brief Uncompressed start offset of this Stream
*
* The offset is relative to the beginning of the lzma_index
* (i.e. usually the beginning of the .xz file).
*/
lzma_vli uncompressed_offset;
/**
* \brief Compressed size of this Stream
*
* This includes all headers except the possible
* Stream Padding after this Stream.
*/
lzma_vli compressed_size;
/**
* \brief Uncompressed size of this Stream
*/ */
lzma_vli uncompressed_size; lzma_vli uncompressed_size;
/** /**
* \brief Compressed offset in the Stream(s) * \brief Size of Stream Padding after this Stream
* *
* This is the offset of the first byte of the Block, that is, * If it hasn't been set with lzma_index_stream_padding(),
* where you need to seek to decode the Block. The offset * this defaults to zero. Stream Padding is always
* is relative to the beginning of the Stream, or if there are * a multiple of four bytes.
* multiple Indexes combined, relative to the beginning of the
* first Stream.
*/ */
lzma_vli stream_offset; lzma_vli padding;
lzma_vli reserved_vli1;
lzma_vli reserved_vli2;
lzma_vli reserved_vli3;
lzma_vli reserved_vli4;
} stream;
struct {
/**
* \brief Block number in the file
*
* The first Block is 1.
*/
lzma_vli number_in_file;
/** /**
* \brief Uncompressed offset * \brief Compressed start offset of this Block
* *
* When doing random-access reading, it is possible that the target * This offset is relative to the beginning of the
* offset is not exactly at Block boundary. One will need to compare * lzma_index (i.e. usually the beginning of the .xz file).
* the target offset against uncompressed_offset, and possibly decode * Normally this is where you should seek in the .xz file
* and throw away some amount of data before reaching the target * to start decompressing this Block.
* offset.
*/ */
lzma_vli uncompressed_offset; lzma_vli compressed_file_offset;
} lzma_index_record; /**
* \brief Uncompressed start offset of this Block
*
* This offset is relative to the beginning of the lzma_index
* (i.e. usually the beginning of the .xz file).
*/
lzma_vli uncompressed_file_offset;
/**
* \brief Block number in this Stream
*
* The first Block is 1.
*/
lzma_vli number_in_stream;
/**
* \brief Compressed start offset of this Block
*
* This offset is relative to the beginning of the Stream
* containing this Block.
*/
lzma_vli compressed_stream_offset;
/**
* \brief Uncompressed start offset of this Block
*
* This offset is relative to the beginning of the Stream
* containing this Block.
*/
lzma_vli uncompressed_stream_offset;
/**
* \brief Uncompressed size of this Block
*
* You should pass this to the Block decoder if you will
* decode this Block.
*
* When doing random-access reading, it is possible that
* the target offset is not exactly at Block boundary. One
* will need to compare the target offset against
* uncompressed_file_offset or uncompressed_stream_offset,
* and possibly decode and throw away some amount of data
* before reaching the target offset.
*/
lzma_vli uncompressed_size;
/**
* \brief Unpadded size of this Block
*
* You should pass this to the Block decoder if you will
* decode this Block.
*/
lzma_vli unpadded_size;
/**
* \brief Total compressed size
*
* This includes all headers and padding in this Block.
* This is useful if you need to know how many bytes
* the Block decoder will actually read.
*/
lzma_vli total_size;
lzma_vli reserved_vli1;
lzma_vli reserved_vli2;
lzma_vli reserved_vli3;
lzma_vli reserved_vli4;
const void *reserved_ptr1;
const void *reserved_ptr2;
const void *reserved_ptr3;
const void *reserved_ptr4;
} block;
/*
* Internal data which is used to store the state of the iterator.
* The exact format may vary between liblzma versions, so don't
* touch these in any way.
*/
union {
const void *p;
size_t s;
lzma_vli v;
} internal[6];
} lzma_index_iter;
/** /**
* \brief Calculate memory usage for Index with given number of Records * \brief Operation mode for lzma_index_iter_next()
*/
typedef enum {
LZMA_INDEX_ITER_ANY = 0,
/**<
* \brief Get the next Block or Stream
*
* Go to the next Block if the current Stream has at least
* one Block left. Otherwise go to the next Stream even if
* it has no Blocks. If the Stream has no Blocks
* (lzma_index_iter.stream.block_count == 0),
* lzma_index_iter.block will have undefined values.
*/
LZMA_INDEX_ITER_STREAM = 1,
/**<
* \brief Get the next Stream
*
* Go to the next Stream even if the current Stream has
* unread Blocks left. If the next Stream has at least one
* Block, the iterator will point to the first Block.
* If there are no Blocks, lzma_index_iter.block will have
* undefined values.
*/
LZMA_INDEX_ITER_BLOCK = 2,
/**<
* \brief Get the next Block
*
* Go to the next Block if the current Stream has at least
* one Block left. If the current Stream has no Blocks left,
* the next Stream with at least one Block is located and
* the iterator will be made to point to the first Block of
* that Stream.
*/
LZMA_INDEX_ITER_NONEMPTY_BLOCK = 3
/**<
* \brief Get the next non-empty Block
*
* This is like LZMA_INDEX_ITER_BLOCK except that it will
* skip Blocks whose Uncompressed Size is zero.
*/
} lzma_index_iter_mode;
/**
* \brief Calculate memory usage of lzma_index
* *
* On disk, the size of the Index field depends on both the number of Records * On disk, the size of the Index field depends on both the number of Records
* stored and how big values the Records store (due to variable-length integer * stored and how big values the Records store (due to variable-length integer
* encoding). When the Index is kept in lzma_index structure, the memory usage * encoding). When the Index is kept in lzma_index structure, the memory usage
* depends only on the number of Records stored in the Index. The size in RAM * depends only on the number of Records/Blocks stored in the Index(es), and
* is almost always a lot bigger than in encoded form on disk. * in case of concatenated lzma_indexes, the number of Streams. The size in
* RAM is almost always significantly bigger than in the encoded form on disk.
* *
* This function calculates an approximate amount of memory needed hold the * This function calculates an approximate amount of memory needed hold
* given number of Records in lzma_index structure. This value may vary * the given number of Streams and Blocks in lzma_index structure. This
* between liblzma versions if the internal implementation is modified. * value may vary between CPU archtectures and also between liblzma versions
* * if the internal implementation is modified.
* If you want to know how much memory an existing lzma_index structure is
* using, use lzma_index_memusage(lzma_index_count(i)).
*/ */
extern LZMA_API(uint64_t) lzma_index_memusage(lzma_vli record_count) extern LZMA_API(uint64_t) lzma_index_memusage(
lzma_vli streams, lzma_vli blocks) lzma_nothrow;
/**
* \brief Calculate the memory usage of an existing lzma_index
*
* This is a shorthand for lzma_index_memusage(lzma_index_stream_count(i),
* lzma_index_block_count(i)).
*/
extern LZMA_API(uint64_t) lzma_index_memused(const lzma_index *i)
lzma_nothrow; lzma_nothrow;
/** /**
* \brief Allocate and initialize a new lzma_index structure * \brief Allocate and initialize a new lzma_index structure
* *
* If i is NULL, a new lzma_index structure is allocated, initialized, * \return On success, a pointer to an empty initialized lzma_index is
* and a pointer to it returned. If allocation fails, NULL is returned. * returned. If allocation fails, NULL is returned.
*
* If i is non-NULL, it is reinitialized and the same pointer returned.
* In this case, return value cannot be NULL or a different pointer than
* the i that was given as an argument.
*/ */
extern LZMA_API(lzma_index *) lzma_index_init( extern LZMA_API(lzma_index *) lzma_index_init(lzma_allocator *allocator)
lzma_index *i, lzma_allocator *allocator) lzma_nothrow; lzma_nothrow;
/** /**
* \brief Deallocate the Index * \brief Deallocate lzma_index
* *
* If i is NULL, this does nothing. * If i is NULL, this does nothing.
*/ */
@ -118,7 +315,7 @@ extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator)
/** /**
* \brief Add a new Record to an Index * \brief Add a new Block to lzma_index
* *
* \param i Pointer to a lzma_index structure * \param i Pointer to a lzma_index structure
* \param allocator Pointer to lzma_allocator, or NULL to * \param allocator Pointer to lzma_allocator, or NULL to
@ -130,7 +327,10 @@ extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator)
* taken directly from lzma_block structure * taken directly from lzma_block structure
* after encoding or decoding the Block. * after encoding or decoding the Block.
* *
* Appending a new Record does not affect the read position. * Appending a new Block does not invalidate iterators. For example,
* if an iterator was pointing to the end of the lzma_index, after
* lzma_index_append() it is possible to read the next Block with
* an existing iterator.
* *
* \return - LZMA_OK * \return - LZMA_OK
* - LZMA_MEM_ERROR * - LZMA_MEM_ERROR
@ -145,9 +345,72 @@ extern LZMA_API(lzma_ret) lzma_index_append(
/** /**
* \brief Get the number of Records * \brief Set the Stream Flags
*
* Set the Stream Flags of the last (and typically the only) Stream
* in lzma_index. This can be useful when reading information from the
* lzma_index, because to decode Blocks, knowing the integrity check type
* is needed.
*
* The given Stream Flags are copied into internal preallocated structure
* in the lzma_index, thus the caller doesn't need to keep the *stream_flags
* available after calling this function.
*
* \return - LZMA_OK
* - LZMA_OPTIONS_ERROR: Unsupported stream_flags->version.
* - LZMA_PROG_ERROR
*/ */
extern LZMA_API(lzma_vli) lzma_index_count(const lzma_index *i) extern LZMA_API(lzma_ret) lzma_index_stream_flags(
lzma_index *i, const lzma_stream_flags *stream_flags)
lzma_nothrow lzma_attr_warn_unused_result;
/**
* \brief Get the types of integrity Checks
*
* If lzma_index_stream_padding() is used to set the Stream Flags for
* every Stream, lzma_index_checks() can be used to get a bitmask to
* indicate which Check types have been used. It can be useful e.g. if
* showing the Check types to the user.
*
* The bitmask is 1 << check_id, e.g. CRC32 is 1 << 1 and SHA-256 is 1 << 10.
*/
extern LZMA_API(uint32_t) lzma_index_checks(const lzma_index *i)
lzma_nothrow lzma_attr_pure;
/**
* \brief Set the amount of Stream Padding
*
* Set the amount of Stream Padding of the last (and typically the only)
* Stream in the lzma_index. This is needed when planning to do random-access
* reading within multiple concatenated Streams.
*
* By default, the amount of Stream Padding is assumed to be zero bytes.
*
* \return - LZMA_OK
* - LZMA_DATA_ERROR: The file size would grow too big.
* - LZMA_PROG_ERROR
*/
extern LZMA_API(lzma_ret) lzma_index_stream_padding(
lzma_index *i, lzma_vli stream_padding)
lzma_nothrow lzma_attr_warn_unused_result;
/**
* \brief Get the number of Streams
*/
extern LZMA_API(lzma_vli) lzma_index_stream_count(const lzma_index *i)
lzma_nothrow lzma_attr_pure;
/**
* \brief Get the number of Blocks
*
* This returns the total number of Blocks in lzma_index. To get number
* of Blocks in individual Streams, use lzma_index_iter.
*/
extern LZMA_API(lzma_vli) lzma_index_block_count(const lzma_index *i)
lzma_nothrow lzma_attr_pure; lzma_nothrow lzma_attr_pure;
@ -160,6 +423,17 @@ extern LZMA_API(lzma_vli) lzma_index_size(const lzma_index *i)
lzma_nothrow lzma_attr_pure; lzma_nothrow lzma_attr_pure;
/**
* \brief Get the total size of the Stream
*
* If multiple lzma_indexes have been combined, this works as if the Blocks
* were in a single Stream. This is useful if you are going to combine
* Blocks from multiple Streams into a single new Stream.
*/
extern LZMA_API(lzma_vli) lzma_index_stream_size(const lzma_index *i)
lzma_nothrow lzma_attr_pure;
/** /**
* \brief Get the total size of the Blocks * \brief Get the total size of the Blocks
* *
@ -170,138 +444,145 @@ extern LZMA_API(lzma_vli) lzma_index_total_size(const lzma_index *i)
lzma_nothrow lzma_attr_pure; lzma_nothrow lzma_attr_pure;
/**
* \brief Get the total size of the Stream
*
* If multiple Indexes have been combined, this works as if the Blocks
* were in a single Stream.
*/
extern LZMA_API(lzma_vli) lzma_index_stream_size(const lzma_index *i)
lzma_nothrow lzma_attr_pure;
/** /**
* \brief Get the total size of the file * \brief Get the total size of the file
* *
* When no Indexes have been combined with lzma_index_cat(), this function is * When no lzma_indexes have been combined with lzma_index_cat() and there is
* identical to lzma_index_stream_size(). If multiple Indexes have been * no Stream Padding, this function is identical to lzma_index_stream_size().
* combined, this includes also the headers of each separate Stream and the * If multiple lzma_indexes have been combined, this includes also the headers
* possible Stream Padding fields. * of each separate Stream and the possible Stream Padding fields.
*/ */
extern LZMA_API(lzma_vli) lzma_index_file_size(const lzma_index *i) extern LZMA_API(lzma_vli) lzma_index_file_size(const lzma_index *i)
lzma_nothrow lzma_attr_pure; lzma_nothrow lzma_attr_pure;
/** /**
* \brief Get the uncompressed size of the Stream * \brief Get the uncompressed size of the file
*/ */
extern LZMA_API(lzma_vli) lzma_index_uncompressed_size(const lzma_index *i) extern LZMA_API(lzma_vli) lzma_index_uncompressed_size(const lzma_index *i)
lzma_nothrow lzma_attr_pure; lzma_nothrow lzma_attr_pure;
/** /**
* \brief Get the next Record from the Index * \brief Initialize an iterator
*
* \param iter Pointer to a lzma_index_iter structure
* \param i lzma_index to which the iterator will be associated
*
* This function associates the iterator with the given lzma_index, and calls
* lzma_index_iter_rewind() on the iterator.
*
* This function doesn't allocate any memory, thus there is no
* lzma_index_iter_end(). The iterator is valid as long as the
* associated lzma_index is valid, that is, until lzma_index_end() or
* using it as source in lzma_index_cat(). Specifically, lzma_index doesn't
* become invalid if new Blocks are added to it with lzma_index_append() or
* if it is used as the destionation in lzma_index_cat().
*
* It is safe to make copies of an initialized lzma_index_iter, for example,
* to easily restart reading at some particular position.
*/ */
extern LZMA_API(lzma_bool) lzma_index_read( extern LZMA_API(void) lzma_index_iter_init(
lzma_index *i, lzma_index_record *record) lzma_index_iter *iter, const lzma_index *i) lzma_nothrow;
/**
* \brief Rewind the iterator
*
* Rewind the iterator so that next call to lzma_index_iter_next() will
* return the first Block or Stream.
*/
extern LZMA_API(void) lzma_index_iter_rewind(lzma_index_iter *iter)
lzma_nothrow;
/**
* \brief Get the next Block or Stream
*
* \param iter Iterator initialized with lzma_index_iter_init()
* \param mode Specify what kind of information the caller wants
* to get. See lzma_index_iter_mode for details.
*
* \return If next Block or Stream matching the mode was found, *iter
* is updated and this function returns false. If no Block or
* Stream matching the mode is found, *iter is not modified
* and this function returns true. If mode is set to an unknown
* value, *iter is not modified and this function returns true.
*/
extern LZMA_API(lzma_bool) lzma_index_iter_next(
lzma_index_iter *iter, lzma_index_iter_mode mode)
lzma_nothrow lzma_attr_warn_unused_result; lzma_nothrow lzma_attr_warn_unused_result;
/** /**
* \brief Rewind the Index * \brief Locate a Block
* *
* Rewind the Index so that next call to lzma_index_read() will return the * If it is possible to seek in the .xz file, it is possible to parse
* first Record. * the Index field(s) and use lzma_index_iter_locate() to do random-access
*/ * reading with granularity of Block size.
extern LZMA_API(void) lzma_index_rewind(lzma_index *i) lzma_nothrow;
/**
* \brief Locate a Record
* *
* When the Index is available, it is possible to do random-access reading * \param iter Iterator that was earlier initialized with
* with granularity of Block size. * lzma_index_iter_init().
*
* \param i Pointer to lzma_index structure
* \param record Pointer to a structure to hold the search results
* \param target Uncompressed target offset which the caller would * \param target Uncompressed target offset which the caller would
* like to locate from the Stream * like to locate from the Stream
* *
* If the target is smaller than the uncompressed size of the Stream (can be * If the target is smaller than the uncompressed size of the Stream (can be
* checked with lzma_index_uncompressed_size()): * checked with lzma_index_uncompressed_size()):
* - Information about the Record containing the requested uncompressed * - Information about the Stream and Block containing the requested
* offset is stored into *record. * uncompressed offset is stored into *iter.
* - Read offset will be adjusted so that calling lzma_index_read() can be * - Internal state of the iterator is adjusted so that
* used to read subsequent Records. * lzma_index_iter_next() can be used to read subsequent Blocks or Streams.
* - This function returns false. * - This function returns false.
* *
* If target is greater than the uncompressed size of the Stream, *record * If target is greater than the uncompressed size of the Stream, *iter
* and the read position are not modified, and this function returns true. * is not modified, and this function returns true.
*/ */
extern LZMA_API(lzma_bool) lzma_index_locate( extern LZMA_API(lzma_bool) lzma_index_iter_locate(
lzma_index *i, lzma_index_record *record, lzma_vli target) lzma_index_iter *iter, lzma_vli target) lzma_nothrow;
lzma_nothrow;
/** /**
* \brief Concatenate Indexes of two Streams * \brief Concatenate lzma_indexes
* *
* Concatenating Indexes is useful when doing random-access reading in * Concatenating lzma_indexes is useful when doing random-access reading in
* multi-Stream .xz file, or when combining multiple Streams into single * multi-Stream .xz file, or when combining multiple Streams into single
* Stream. * Stream.
* *
* \param dest Destination Index after which src is appended * \param dest lzma_index after which src is appended
* \param src Source Index. If this function succeeds, the * \param src lzma_index to be appeneded after dest. If this
* memory allocated for src is freed or moved to * function succeeds, the memory allocated for src
* be part of dest. * is freed or moved to be part of dest, and all
* iterators pointing to src will become invalid.
* \param allocator Custom memory allocator; can be NULL to use * \param allocator Custom memory allocator; can be NULL to use
* malloc() and free(). * malloc() and free().
* \param padding Size of the Stream Padding field between Streams.
* This must be a multiple of four.
* *
* \return - LZMA_OK: Indexes concatenated successfully. src is now * \return - LZMA_OK: lzma_indexes were concatenated successfully.
* a dangling pointer. * src is now a dangling pointer.
* - LZMA_DATA_ERROR: *dest would grow too big. * - LZMA_DATA_ERROR: *dest would grow too big.
* - LZMA_MEM_ERROR * - LZMA_MEM_ERROR
* - LZMA_PROG_ERROR * - LZMA_PROG_ERROR
*/ */
extern LZMA_API(lzma_ret) lzma_index_cat(lzma_index *lzma_restrict dest, extern LZMA_API(lzma_ret) lzma_index_cat(lzma_index *lzma_restrict dest,
lzma_index *lzma_restrict src, lzma_index *lzma_restrict src,
lzma_allocator *allocator, lzma_vli padding) lzma_allocator *allocator)
lzma_nothrow lzma_attr_warn_unused_result; lzma_nothrow lzma_attr_warn_unused_result;
/** /**
* \brief Duplicate an Index list * \brief Duplicate lzma_index
* *
* Makes an identical copy of the Index. Also the read position is copied. * \return A copy of the lzma_index, or NULL if memory allocation failed.
*
* \return A copy of the Index, or NULL if memory allocation failed.
*/ */
extern LZMA_API(lzma_index *) lzma_index_dup( extern LZMA_API(lzma_index *) lzma_index_dup(
const lzma_index *i, lzma_allocator *allocator) const lzma_index *i, lzma_allocator *allocator)
lzma_nothrow lzma_attr_warn_unused_result; lzma_nothrow lzma_attr_warn_unused_result;
/**
* \brief Compare if two Index lists are identical
*
* Read positions are not compared.
*
* \return True if *a and *b are equal, false otherwise.
*/
extern LZMA_API(lzma_bool) lzma_index_equal(
const lzma_index *a, const lzma_index *b)
lzma_nothrow lzma_attr_pure;
/** /**
* \brief Initialize .xz Index encoder * \brief Initialize .xz Index encoder
* *
* \param strm Pointer to properly prepared lzma_stream * \param strm Pointer to properly prepared lzma_stream
* \param i Pointer to lzma_index which should be encoded. * \param i Pointer to lzma_index which should be encoded.
* The read position will be at the end of the Index
* after lzma_code() has returned LZMA_STREAM_END.
* *
* The only valid action value for lzma_code() is LZMA_RUN. * The only valid action value for lzma_code() is LZMA_RUN.
* *
@ -309,7 +590,8 @@ extern LZMA_API(lzma_bool) lzma_index_equal(
* - LZMA_MEM_ERROR * - LZMA_MEM_ERROR
* - LZMA_PROG_ERROR * - LZMA_PROG_ERROR
*/ */
extern LZMA_API(lzma_ret) lzma_index_encoder(lzma_stream *strm, lzma_index *i) extern LZMA_API(lzma_ret) lzma_index_encoder(
lzma_stream *strm, const lzma_index *i)
lzma_nothrow lzma_attr_warn_unused_result; lzma_nothrow lzma_attr_warn_unused_result;
@ -322,10 +604,10 @@ extern LZMA_API(lzma_ret) lzma_index_encoder(lzma_stream *strm, lzma_index *i)
* set *i to NULL (the old value is ignored). If * set *i to NULL (the old value is ignored). If
* decoding succeeds (lzma_code() returns * decoding succeeds (lzma_code() returns
* LZMA_STREAM_END), *i will be set to point * LZMA_STREAM_END), *i will be set to point
* to the decoded Index, which the application * to a new lzma_index, which the application
* has to later free with lzma_index_end(). * has to later free with lzma_index_end().
* \param memlimit How much memory the resulting Index is allowed * \param memlimit How much memory the resulting lzma_index is
* to require. * allowed to require.
* *
* The only valid action value for lzma_code() is LZMA_RUN. * The only valid action value for lzma_code() is LZMA_RUN.
* *
@ -349,9 +631,7 @@ extern LZMA_API(lzma_ret) lzma_index_decoder(
/** /**
* \brief Single-call .xz Index encoder * \brief Single-call .xz Index encoder
* *
* \param i Index to be encoded. The read position will be at * \param i lzma_index to be encoded
* the end of the Index if encoding succeeds, or at
* unspecified position in case an error occurs.
* \param out Beginning of the output buffer * \param out Beginning of the output buffer
* \param out_pos The next byte will be written to out[*out_pos]. * \param out_pos The next byte will be written to out[*out_pos].
* *out_pos is updated only if encoding succeeds. * *out_pos is updated only if encoding succeeds.
@ -367,23 +647,23 @@ extern LZMA_API(lzma_ret) lzma_index_decoder(
* \note This function doesn't take allocator argument since all * \note This function doesn't take allocator argument since all
* the internal data is allocated on stack. * the internal data is allocated on stack.
*/ */
extern LZMA_API(lzma_ret) lzma_index_buffer_encode(lzma_index *i, extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i,
uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow;
/** /**
* \brief Single-call .xz Index decoder * \brief Single-call .xz Index decoder
* *
* \param i If decoding succeeds, *i will point to the * \param i If decoding succeeds, *i will point to a new
* decoded Index, which the application has to * lzma_index, which the application has to
* later free with lzma_index_end(). If an error * later free with lzma_index_end(). If an error
* occurs, *i will be NULL. The old value of *i * occurs, *i will be NULL. The old value of *i
* is always ignored and thus doesn't need to be * is always ignored and thus doesn't need to be
* initialized by the caller. * initialized by the caller.
* \param memlimit Pointer to how much memory the resulting Index * \param memlimit Pointer to how much memory the resulting
* is allowed to require. The value pointed by * lzma_index is allowed to require. The value
* this pointer is modified if and only if * pointed by this pointer is modified if and only
* LZMA_MEMLIMIT_ERROR is returned. * if LZMA_MEMLIMIT_ERROR is returned.
* \param allocator Pointer to lzma_allocator, or NULL to use malloc() * \param allocator Pointer to lzma_allocator, or NULL to use malloc()
* \param in Beginning of the input buffer * \param in Beginning of the input buffer
* \param in_pos The next byte will be read from in[*in_pos]. * \param in_pos The next byte will be read from in[*in_pos].

File diff suppressed because it is too large Load Diff

View File

@ -28,6 +28,12 @@
extern uint32_t lzma_index_padding_size(const lzma_index *i); extern uint32_t lzma_index_padding_size(const lzma_index *i);
/// Set for how many Records to allocate memory the next time
/// lzma_index_append() needs to allocate space for a new Record.
/// This is used only by the Index decoder.
extern void lzma_index_prealloc(lzma_index *i, lzma_vli records);
/// Round the variable-length integer to the next multiple of four. /// Round the variable-length integer to the next multiple of four.
static inline lzma_vli static inline lzma_vli
vli_ceil4(lzma_vli vli) vli_ceil4(lzma_vli vli)

View File

@ -95,11 +95,15 @@ index_decode(lzma_coder *coder, lzma_allocator *allocator,
// Fall through // Fall through
case SEQ_MEMUSAGE: case SEQ_MEMUSAGE:
if (lzma_index_memusage(coder->count) > coder->memlimit) { if (lzma_index_memusage(1, coder->count) > coder->memlimit) {
ret = LZMA_MEMLIMIT_ERROR; ret = LZMA_MEMLIMIT_ERROR;
goto out; goto out;
} }
// Tell the Index handling code how many Records this
// Index has to allow it to allocate memory more efficiently.
lzma_index_prealloc(coder->index, coder->count);
ret = LZMA_OK; ret = LZMA_OK;
coder->sequence = coder->count == 0 coder->sequence = coder->count == 0
? SEQ_PADDING_INIT : SEQ_UNPADDED; ? SEQ_PADDING_INIT : SEQ_UNPADDED;
@ -214,7 +218,7 @@ static lzma_ret
index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage,
uint64_t *old_memlimit, uint64_t new_memlimit) uint64_t *old_memlimit, uint64_t new_memlimit)
{ {
*memusage = lzma_index_memusage(coder->count); *memusage = lzma_index_memusage(1, coder->count);
if (new_memlimit != 0 && new_memlimit < *memusage) if (new_memlimit != 0 && new_memlimit < *memusage)
return LZMA_MEMLIMIT_ERROR; return LZMA_MEMLIMIT_ERROR;
@ -238,7 +242,7 @@ index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator,
*i = NULL; *i = NULL;
// We always allocate a new lzma_index. // We always allocate a new lzma_index.
coder->index = lzma_index_init(NULL, allocator); coder->index = lzma_index_init(allocator);
if (coder->index == NULL) if (coder->index == NULL)
return LZMA_MEM_ERROR; return LZMA_MEM_ERROR;
@ -329,7 +333,7 @@ lzma_index_buffer_decode(
} else if (ret == LZMA_MEMLIMIT_ERROR) { } else if (ret == LZMA_MEMLIMIT_ERROR) {
// Tell the caller how much memory would have // Tell the caller how much memory would have
// been needed. // been needed.
*memlimit = lzma_index_memusage(coder.count); *memlimit = lzma_index_memusage(1, coder.count);
} }
} }

View File

@ -26,12 +26,11 @@ struct lzma_coder_s {
SEQ_CRC32, SEQ_CRC32,
} sequence; } sequence;
/// Index given to us to encode. Note that we modify it in sense that /// Index being encoded
/// we read it, and read position is tracked in lzma_index structure. const lzma_index *index;
lzma_index *index;
/// The current Index Record being encoded /// Iterator for the Index being encoded
lzma_index_record record; lzma_index_iter iter;
/// Position in integers /// Position in integers
size_t pos; size_t pos;
@ -69,8 +68,8 @@ index_encode(lzma_coder *coder,
break; break;
case SEQ_COUNT: { case SEQ_COUNT: {
const lzma_vli index_count = lzma_index_count(coder->index); const lzma_vli count = lzma_index_block_count(coder->index);
ret = lzma_vli_encode(index_count, &coder->pos, ret = lzma_vli_encode(count, &coder->pos,
out, out_pos, out_size); out, out_pos, out_size);
if (ret != LZMA_STREAM_END) if (ret != LZMA_STREAM_END)
goto out; goto out;
@ -82,7 +81,8 @@ index_encode(lzma_coder *coder,
} }
case SEQ_NEXT: case SEQ_NEXT:
if (lzma_index_read(coder->index, &coder->record)) { if (lzma_index_iter_next(
&coder->iter, LZMA_INDEX_ITER_BLOCK)) {
// Get the size of the Index Padding field. // Get the size of the Index Padding field.
coder->pos = lzma_index_padding_size(coder->index); coder->pos = lzma_index_padding_size(coder->index);
assert(coder->pos <= 3); assert(coder->pos <= 3);
@ -90,12 +90,6 @@ index_encode(lzma_coder *coder,
break; break;
} }
// Unpadded Size must be within valid limits.
if (coder->record.unpadded_size < UNPADDED_SIZE_MIN
|| coder->record.unpadded_size
> UNPADDED_SIZE_MAX)
return LZMA_PROG_ERROR;
coder->sequence = SEQ_UNPADDED; coder->sequence = SEQ_UNPADDED;
// Fall through // Fall through
@ -103,8 +97,8 @@ index_encode(lzma_coder *coder,
case SEQ_UNPADDED: case SEQ_UNPADDED:
case SEQ_UNCOMPRESSED: { case SEQ_UNCOMPRESSED: {
const lzma_vli size = coder->sequence == SEQ_UNPADDED const lzma_vli size = coder->sequence == SEQ_UNPADDED
? coder->record.unpadded_size ? coder->iter.block.unpadded_size
: coder->record.uncompressed_size; : coder->iter.block.uncompressed_size;
ret = lzma_vli_encode(size, &coder->pos, ret = lzma_vli_encode(size, &coder->pos,
out, out_pos, out_size); out, out_pos, out_size);
@ -172,9 +166,9 @@ index_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
static void static void
index_encoder_reset(lzma_coder *coder, lzma_index *i) index_encoder_reset(lzma_coder *coder, const lzma_index *i)
{ {
lzma_index_rewind(i); lzma_index_iter_init(&coder->iter, i);
coder->sequence = SEQ_INDICATOR; coder->sequence = SEQ_INDICATOR;
coder->index = i; coder->index = i;
@ -187,7 +181,7 @@ index_encoder_reset(lzma_coder *coder, lzma_index *i)
extern lzma_ret extern lzma_ret
lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
lzma_index *i) const lzma_index *i)
{ {
lzma_next_coder_init(&lzma_index_encoder_init, next, allocator); lzma_next_coder_init(&lzma_index_encoder_init, next, allocator);
@ -210,7 +204,7 @@ lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
extern LZMA_API(lzma_ret) extern LZMA_API(lzma_ret)
lzma_index_encoder(lzma_stream *strm, lzma_index *i) lzma_index_encoder(lzma_stream *strm, const lzma_index *i)
{ {
lzma_next_strm_init(lzma_index_encoder_init, strm, i); lzma_next_strm_init(lzma_index_encoder_init, strm, i);
@ -221,7 +215,7 @@ lzma_index_encoder(lzma_stream *strm, lzma_index *i)
extern LZMA_API(lzma_ret) extern LZMA_API(lzma_ret)
lzma_index_buffer_encode(lzma_index *i, lzma_index_buffer_encode(const lzma_index *i,
uint8_t *out, size_t *out_pos, size_t out_size) uint8_t *out, size_t *out_pos, size_t out_size)
{ {
// Validate the arugments. // Validate the arugments.

View File

@ -17,7 +17,7 @@
extern lzma_ret lzma_index_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_index_encoder_init(lzma_next_coder *next,
lzma_allocator *allocator, lzma_index *i); lzma_allocator *allocator, const lzma_index *i);
#endif #endif

View File

@ -94,11 +94,11 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check,
// Index // Index
{ {
// Create an Index with one Record. // Create an Index with one Record.
lzma_index *i = lzma_index_init(NULL, NULL); lzma_index *i = lzma_index_init(allocator);
if (i == NULL) if (i == NULL)
return LZMA_MEM_ERROR; return LZMA_MEM_ERROR;
lzma_ret ret = lzma_index_append(i, NULL, lzma_ret ret = lzma_index_append(i, allocator,
lzma_block_unpadded_size(&block), lzma_block_unpadded_size(&block),
block.uncompressed_size); block.uncompressed_size);
@ -111,7 +111,7 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check,
stream_flags.backward_size = lzma_index_size(i); stream_flags.backward_size = lzma_index_size(i);
} }
lzma_index_end(i, NULL); lzma_index_end(i, allocator);
if (ret != LZMA_OK) if (ret != LZMA_OK)
return ret; return ret;

View File

@ -292,7 +292,8 @@ lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
next->coder->filters[0].id = LZMA_VLI_UNKNOWN; next->coder->filters[0].id = LZMA_VLI_UNKNOWN;
// Initialize the Index // Initialize the Index
next->coder->index = lzma_index_init(next->coder->index, allocator); lzma_index_end(next->coder->index, allocator);
next->coder->index = lzma_index_init(allocator);
if (next->coder->index == NULL) if (next->coder->index == NULL)
return LZMA_MEM_ERROR; return LZMA_MEM_ERROR;

View File

@ -21,7 +21,7 @@
static lzma_index * static lzma_index *
create_empty(void) create_empty(void)
{ {
lzma_index *i = lzma_index_init(NULL, NULL); lzma_index *i = lzma_index_init(NULL);
expect(i != NULL); expect(i != NULL);
return i; return i;
} }
@ -30,7 +30,7 @@ create_empty(void)
static lzma_index * static lzma_index *
create_small(void) create_small(void)
{ {
lzma_index *i = lzma_index_init(NULL, NULL); lzma_index *i = lzma_index_init(NULL);
expect(i != NULL); expect(i != NULL);
expect(lzma_index_append(i, NULL, 101, 555) == LZMA_OK); expect(lzma_index_append(i, NULL, 101, 555) == LZMA_OK);
expect(lzma_index_append(i, NULL, 602, 777) == LZMA_OK); expect(lzma_index_append(i, NULL, 602, 777) == LZMA_OK);
@ -42,7 +42,7 @@ create_small(void)
static lzma_index * static lzma_index *
create_big(void) create_big(void)
{ {
lzma_index *i = lzma_index_init(NULL, NULL); lzma_index *i = lzma_index_init(NULL);
expect(i != NULL); expect(i != NULL);
lzma_vli total_size = 0; lzma_vli total_size = 0;
@ -58,7 +58,7 @@ create_big(void)
uncompressed_size += n; uncompressed_size += n;
} }
expect(lzma_index_count(i) == BIG_COUNT); expect(lzma_index_block_count(i) == BIG_COUNT);
expect(lzma_index_total_size(i) == total_size); expect(lzma_index_total_size(i) == total_size);
expect(lzma_index_uncompressed_size(i) == uncompressed_size); expect(lzma_index_uncompressed_size(i) == uncompressed_size);
expect(lzma_index_total_size(i) + lzma_index_size(i) expect(lzma_index_total_size(i) + lzma_index_size(i)
@ -69,6 +69,60 @@ create_big(void)
} }
static bool
is_equal(const lzma_index *a, const lzma_index *b)
{
// Compare only the Stream and Block sizes and offsets.
lzma_index_iter ra, rb;
lzma_index_iter_init(&ra, a);
lzma_index_iter_init(&rb, b);
while (true) {
bool reta = lzma_index_iter_next(&ra, LZMA_INDEX_ITER_ANY);
bool retb = lzma_index_iter_next(&rb, LZMA_INDEX_ITER_ANY);
if (reta)
return !(reta ^ retb);
if (ra.stream.number != rb.stream.number
|| ra.stream.block_count
!= rb.stream.block_count
|| ra.stream.compressed_offset
!= rb.stream.compressed_offset
|| ra.stream.uncompressed_offset
!= rb.stream.uncompressed_offset
|| ra.stream.compressed_size
!= rb.stream.compressed_size
|| ra.stream.uncompressed_size
!= rb.stream.uncompressed_size
|| ra.stream.padding
!= rb.stream.padding)
return false;
if (ra.stream.block_count == 0)
continue;
if (ra.block.number_in_file != rb.block.number_in_file
|| ra.block.compressed_file_offset
!= rb.block.compressed_file_offset
|| ra.block.uncompressed_file_offset
!= rb.block.uncompressed_file_offset
|| ra.block.number_in_stream
!= rb.block.number_in_stream
|| ra.block.compressed_stream_offset
!= rb.block.compressed_stream_offset
|| ra.block.uncompressed_stream_offset
!= rb.block.uncompressed_stream_offset
|| ra.block.uncompressed_size
!= rb.block.uncompressed_size
|| ra.block.unpadded_size
!= rb.block.unpadded_size
|| ra.block.total_size
!= rb.block.total_size)
return false;
}
}
static void static void
test_equal(void) test_equal(void)
{ {
@ -77,13 +131,13 @@ test_equal(void)
lzma_index *c = create_big(); lzma_index *c = create_big();
expect(a && b && c); expect(a && b && c);
expect(lzma_index_equal(a, a)); expect(is_equal(a, a));
expect(lzma_index_equal(b, b)); expect(is_equal(b, b));
expect(lzma_index_equal(c, c)); expect(is_equal(c, c));
expect(!lzma_index_equal(a, b)); expect(!is_equal(a, b));
expect(!lzma_index_equal(a, c)); expect(!is_equal(a, c));
expect(!lzma_index_equal(b, c)); expect(!is_equal(b, c));
lzma_index_end(a, NULL); lzma_index_end(a, NULL);
lzma_index_end(b, NULL); lzma_index_end(b, NULL);
@ -111,6 +165,7 @@ test_copy(const lzma_index *i)
{ {
lzma_index *d = lzma_index_dup(i, NULL); lzma_index *d = lzma_index_dup(i, NULL);
expect(d != NULL); expect(d != NULL);
expect(is_equal(i, d));
lzma_index_end(d, NULL); lzma_index_end(d, NULL);
} }
@ -118,7 +173,8 @@ test_copy(const lzma_index *i)
static void static void
test_read(lzma_index *i) test_read(lzma_index *i)
{ {
lzma_index_record record; lzma_index_iter r;
lzma_index_iter_init(&r, i);
// Try twice so we see that rewinding works. // Try twice so we see that rewinding works.
for (size_t j = 0; j < 2; ++j) { for (size_t j = 0; j < 2; ++j) {
@ -128,25 +184,26 @@ test_read(lzma_index *i)
lzma_vli uncompressed_offset = 0; lzma_vli uncompressed_offset = 0;
uint32_t count = 0; uint32_t count = 0;
while (!lzma_index_read(i, &record)) { while (!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)) {
++count; ++count;
total_size += record.total_size; total_size += r.block.total_size;
uncompressed_size += record.uncompressed_size; uncompressed_size += r.block.uncompressed_size;
expect(record.stream_offset == stream_offset); expect(r.block.compressed_file_offset
expect(record.uncompressed_offset == stream_offset);
expect(r.block.uncompressed_file_offset
== uncompressed_offset); == uncompressed_offset);
stream_offset += record.total_size; stream_offset += r.block.total_size;
uncompressed_offset += record.uncompressed_size; uncompressed_offset += r.block.uncompressed_size;
} }
expect(lzma_index_total_size(i) == total_size); expect(lzma_index_total_size(i) == total_size);
expect(lzma_index_uncompressed_size(i) == uncompressed_size); expect(lzma_index_uncompressed_size(i) == uncompressed_size);
expect(lzma_index_count(i) == count); expect(lzma_index_block_count(i) == count);
lzma_index_rewind(i); lzma_index_iter_rewind(&r);
} }
} }
@ -171,7 +228,7 @@ test_code(lzma_index *i)
expect(d == NULL); expect(d == NULL);
succeed(decoder_loop(&strm, buf, index_size)); succeed(decoder_loop(&strm, buf, index_size));
expect(lzma_index_equal(i, d)); expect(is_equal(i, d));
lzma_index_end(d, NULL); lzma_index_end(d, NULL);
lzma_end(&strm); lzma_end(&strm);
@ -179,11 +236,11 @@ test_code(lzma_index *i)
// Decode with hashing // Decode with hashing
lzma_index_hash *h = lzma_index_hash_init(NULL, NULL); lzma_index_hash *h = lzma_index_hash_init(NULL, NULL);
expect(h != NULL); expect(h != NULL);
lzma_index_rewind(i); lzma_index_iter r;
lzma_index_record r; lzma_index_iter_init(&r, i);
while (!lzma_index_read(i, &r)) while (!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK))
expect(lzma_index_hash_append(h, r.unpadded_size, expect(lzma_index_hash_append(h, r.block.unpadded_size,
r.uncompressed_size) == LZMA_OK); r.block.uncompressed_size) == LZMA_OK);
size_t pos = 0; size_t pos = 0;
while (pos < index_size - 1) while (pos < index_size - 1)
expect(lzma_index_hash_decode(h, buf, &pos, pos + 1) expect(lzma_index_hash_decode(h, buf, &pos, pos + 1)
@ -213,7 +270,7 @@ test_code(lzma_index *i)
succeed(lzma_index_buffer_decode(&d, &memlimit, NULL, buf, &buf_pos, succeed(lzma_index_buffer_decode(&d, &memlimit, NULL, buf, &buf_pos,
index_size + 1)); index_size + 1));
expect(buf_pos == index_size + 1); expect(buf_pos == index_size + 1);
expect(lzma_index_equal(i, d)); expect(is_equal(i, d));
lzma_index_end(d, NULL); lzma_index_end(d, NULL);
@ -234,71 +291,99 @@ static void
test_cat(void) test_cat(void)
{ {
lzma_index *a, *b, *c; lzma_index *a, *b, *c;
lzma_index_record r; lzma_index_iter r;
// Empty Indexes // Empty Indexes
a = create_empty(); a = create_empty();
b = create_empty(); b = create_empty();
expect(lzma_index_cat(a, b, NULL, 0) == LZMA_OK); expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_count(a) == 0); expect(lzma_index_block_count(a) == 0);
expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8); expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8);
expect(lzma_index_file_size(a) expect(lzma_index_file_size(a)
== 2 * (2 * LZMA_STREAM_HEADER_SIZE + 8)); == 2 * (2 * LZMA_STREAM_HEADER_SIZE + 8));
expect(lzma_index_read(a, &r)); lzma_index_iter_init(&r, a);
expect(lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK));
b = create_empty(); b = create_empty();
expect(lzma_index_cat(a, b, NULL, 0) == LZMA_OK); expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_count(a) == 0); expect(lzma_index_block_count(a) == 0);
expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8); expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8);
expect(lzma_index_file_size(a) expect(lzma_index_file_size(a)
== 3 * (2 * LZMA_STREAM_HEADER_SIZE + 8)); == 3 * (2 * LZMA_STREAM_HEADER_SIZE + 8));
b = create_empty(); b = create_empty();
c = create_empty(); c = create_empty();
expect(lzma_index_cat(b, c, NULL, 4) == LZMA_OK); expect(lzma_index_stream_padding(b, 4) == LZMA_OK);
expect(lzma_index_count(b) == 0); expect(lzma_index_cat(b, c, NULL) == LZMA_OK);
expect(lzma_index_block_count(b) == 0);
expect(lzma_index_stream_size(b) == 2 * LZMA_STREAM_HEADER_SIZE + 8); expect(lzma_index_stream_size(b) == 2 * LZMA_STREAM_HEADER_SIZE + 8);
expect(lzma_index_file_size(b) expect(lzma_index_file_size(b)
== 2 * (2 * LZMA_STREAM_HEADER_SIZE + 8) + 4); == 2 * (2 * LZMA_STREAM_HEADER_SIZE + 8) + 4);
expect(lzma_index_cat(a, b, NULL, 8) == LZMA_OK); expect(lzma_index_stream_padding(a, 8) == LZMA_OK);
expect(lzma_index_count(a) == 0); expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_block_count(a) == 0);
expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8); expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8);
expect(lzma_index_file_size(a) expect(lzma_index_file_size(a)
== 5 * (2 * LZMA_STREAM_HEADER_SIZE + 8) + 4 + 8); == 5 * (2 * LZMA_STREAM_HEADER_SIZE + 8) + 4 + 8);
expect(lzma_index_read(a, &r)); expect(lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK));
lzma_index_rewind(a); lzma_index_iter_rewind(&r);
expect(lzma_index_read(a, &r)); expect(lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK));
lzma_index_end(a, NULL); lzma_index_end(a, NULL);
// Small Indexes // Small Indexes
a = create_small(); a = create_small();
lzma_vli stream_size = lzma_index_stream_size(a); lzma_vli stream_size = lzma_index_stream_size(a);
lzma_index_iter_init(&r, a);
for (int i = SMALL_COUNT; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
b = create_small(); b = create_small();
expect(lzma_index_cat(a, b, NULL, 4) == LZMA_OK); expect(lzma_index_stream_padding(a, 4) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_file_size(a) == stream_size * 2 + 4); expect(lzma_index_file_size(a) == stream_size * 2 + 4);
expect(lzma_index_stream_size(a) > stream_size); expect(lzma_index_stream_size(a) > stream_size);
expect(lzma_index_stream_size(a) < stream_size * 2); expect(lzma_index_stream_size(a) < stream_size * 2);
for (int i = SMALL_COUNT; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_iter_rewind(&r);
for (int i = SMALL_COUNT * 2; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
b = create_small(); b = create_small();
c = create_small(); c = create_small();
expect(lzma_index_cat(b, c, NULL, 8) == LZMA_OK); expect(lzma_index_stream_padding(b, 8) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL, 12) == LZMA_OK); expect(lzma_index_cat(b, c, NULL) == LZMA_OK);
expect(lzma_index_stream_padding(a, 12) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_file_size(a) == stream_size * 4 + 4 + 8 + 12); expect(lzma_index_file_size(a) == stream_size * 4 + 4 + 8 + 12);
expect(lzma_index_count(a) == SMALL_COUNT * 4); expect(lzma_index_block_count(a) == SMALL_COUNT * 4);
for (int i = SMALL_COUNT * 2; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_iter_rewind(&r);
for (int i = SMALL_COUNT * 4; i >= 0; --i) for (int i = SMALL_COUNT * 4; i >= 0; --i)
expect(!lzma_index_read(a, &r) ^ (i == 0)); expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_end(a, NULL); lzma_index_end(a, NULL);
// Mix of empty and small // Mix of empty and small
a = create_empty(); a = create_empty();
b = create_small(); b = create_small();
expect(lzma_index_cat(a, b, NULL, 4) == LZMA_OK); expect(lzma_index_stream_padding(a, 4) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
lzma_index_iter_init(&r, a);
for (int i = SMALL_COUNT; i >= 0; --i) for (int i = SMALL_COUNT; i >= 0; --i)
expect(!lzma_index_read(a, &r) ^ (i == 0)); expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_end(a, NULL); lzma_index_end(a, NULL);
@ -306,19 +391,24 @@ test_cat(void)
a = create_big(); a = create_big();
stream_size = lzma_index_stream_size(a); stream_size = lzma_index_stream_size(a);
b = create_big(); b = create_big();
expect(lzma_index_cat(a, b, NULL, 4) == LZMA_OK); expect(lzma_index_stream_padding(a, 4) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_file_size(a) == stream_size * 2 + 4); expect(lzma_index_file_size(a) == stream_size * 2 + 4);
expect(lzma_index_stream_size(a) > stream_size); expect(lzma_index_stream_size(a) > stream_size);
expect(lzma_index_stream_size(a) < stream_size * 2); expect(lzma_index_stream_size(a) < stream_size * 2);
b = create_big(); b = create_big();
c = create_big(); c = create_big();
expect(lzma_index_cat(b, c, NULL, 8) == LZMA_OK); expect(lzma_index_stream_padding(b, 8) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL, 12) == LZMA_OK); expect(lzma_index_cat(b, c, NULL) == LZMA_OK);
expect(lzma_index_stream_padding(a, 12) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_file_size(a) == stream_size * 4 + 4 + 8 + 12); expect(lzma_index_file_size(a) == stream_size * 4 + 4 + 8 + 12);
lzma_index_iter_init(&r, a);
for (int i = BIG_COUNT * 4; i >= 0; --i) for (int i = BIG_COUNT * 4; i >= 0; --i)
expect(!lzma_index_read(a, &r) ^ (i == 0)); expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_end(a, NULL); lzma_index_end(a, NULL);
} }
@ -327,102 +417,113 @@ test_cat(void)
static void static void
test_locate(void) test_locate(void)
{ {
lzma_index_record r; lzma_index *i = lzma_index_init(NULL);
lzma_index *i = lzma_index_init(NULL, NULL);
expect(i != NULL); expect(i != NULL);
lzma_index_iter r;
lzma_index_iter_init(&r, i);
// Cannot locate anything from an empty Index. // Cannot locate anything from an empty Index.
expect(lzma_index_locate(i, &r, 0)); expect(lzma_index_iter_locate(&r, 0));
expect(lzma_index_locate(i, &r, 555)); expect(lzma_index_iter_locate(&r, 555));
// One empty Record: nothing is found since there's no uncompressed // One empty Record: nothing is found since there's no uncompressed
// data. // data.
expect(lzma_index_append(i, NULL, 16, 0) == LZMA_OK); expect(lzma_index_append(i, NULL, 16, 0) == LZMA_OK);
expect(lzma_index_locate(i, &r, 0)); expect(lzma_index_iter_locate(&r, 0));
// Non-empty Record and we can find something. // Non-empty Record and we can find something.
expect(lzma_index_append(i, NULL, 32, 5) == LZMA_OK); expect(lzma_index_append(i, NULL, 32, 5) == LZMA_OK);
expect(!lzma_index_locate(i, &r, 0)); expect(!lzma_index_iter_locate(&r, 0));
expect(r.total_size == 32); expect(r.block.total_size == 32);
expect(r.uncompressed_size == 5); expect(r.block.uncompressed_size == 5);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 16); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 0); == LZMA_STREAM_HEADER_SIZE + 16);
expect(r.block.uncompressed_file_offset == 0);
// Still cannot find anything past the end. // Still cannot find anything past the end.
expect(lzma_index_locate(i, &r, 5)); expect(lzma_index_iter_locate(&r, 5));
// Add the third Record. // Add the third Record.
expect(lzma_index_append(i, NULL, 40, 11) == LZMA_OK); expect(lzma_index_append(i, NULL, 40, 11) == LZMA_OK);
expect(!lzma_index_locate(i, &r, 0)); expect(!lzma_index_iter_locate(&r, 0));
expect(r.total_size == 32); expect(r.block.total_size == 32);
expect(r.uncompressed_size == 5); expect(r.block.uncompressed_size == 5);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 16); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 0); == LZMA_STREAM_HEADER_SIZE + 16);
expect(r.block.uncompressed_file_offset == 0);
expect(!lzma_index_read(i, &r)); expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK));
expect(r.total_size == 40); expect(r.block.total_size == 40);
expect(r.uncompressed_size == 11); expect(r.block.uncompressed_size == 11);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 16 + 32); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 5); == LZMA_STREAM_HEADER_SIZE + 16 + 32);
expect(r.block.uncompressed_file_offset == 5);
expect(!lzma_index_locate(i, &r, 2)); expect(!lzma_index_iter_locate(&r, 2));
expect(r.total_size == 32); expect(r.block.total_size == 32);
expect(r.uncompressed_size == 5); expect(r.block.uncompressed_size == 5);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 16); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 0); == LZMA_STREAM_HEADER_SIZE + 16);
expect(r.block.uncompressed_file_offset == 0);
expect(!lzma_index_locate(i, &r, 5)); expect(!lzma_index_iter_locate(&r, 5));
expect(r.total_size == 40); expect(r.block.total_size == 40);
expect(r.uncompressed_size == 11); expect(r.block.uncompressed_size == 11);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 16 + 32); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 5); == LZMA_STREAM_HEADER_SIZE + 16 + 32);
expect(r.block.uncompressed_file_offset == 5);
expect(!lzma_index_locate(i, &r, 5 + 11 - 1)); expect(!lzma_index_iter_locate(&r, 5 + 11 - 1));
expect(r.total_size == 40); expect(r.block.total_size == 40);
expect(r.uncompressed_size == 11); expect(r.block.uncompressed_size == 11);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 16 + 32); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 5); == LZMA_STREAM_HEADER_SIZE + 16 + 32);
expect(r.block.uncompressed_file_offset == 5);
expect(lzma_index_locate(i, &r, 5 + 11)); expect(lzma_index_iter_locate(&r, 5 + 11));
expect(lzma_index_locate(i, &r, 5 + 15)); expect(lzma_index_iter_locate(&r, 5 + 15));
// Large Index // Large Index
i = lzma_index_init(i, NULL); lzma_index_end(i, NULL);
i = lzma_index_init(NULL);
expect(i != NULL); expect(i != NULL);
lzma_index_iter_init(&r, i);
for (size_t n = 4; n <= 4 * 5555; n += 4) for (size_t n = 4; n <= 4 * 5555; n += 4)
expect(lzma_index_append(i, NULL, n + 8, n) == LZMA_OK); expect(lzma_index_append(i, NULL, n + 8, n) == LZMA_OK);
expect(lzma_index_count(i) == 5555); expect(lzma_index_block_count(i) == 5555);
// First Record // First Record
expect(!lzma_index_locate(i, &r, 0)); expect(!lzma_index_iter_locate(&r, 0));
expect(r.total_size == 4 + 8); expect(r.block.total_size == 4 + 8);
expect(r.uncompressed_size == 4); expect(r.block.uncompressed_size == 4);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE);
expect(r.uncompressed_offset == 0); expect(r.block.uncompressed_file_offset == 0);
expect(!lzma_index_locate(i, &r, 3)); expect(!lzma_index_iter_locate(&r, 3));
expect(r.total_size == 4 + 8); expect(r.block.total_size == 4 + 8);
expect(r.uncompressed_size == 4); expect(r.block.uncompressed_size == 4);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE);
expect(r.uncompressed_offset == 0); expect(r.block.uncompressed_file_offset == 0);
// Second Record // Second Record
expect(!lzma_index_locate(i, &r, 4)); expect(!lzma_index_iter_locate(&r, 4));
expect(r.total_size == 2 * 4 + 8); expect(r.block.total_size == 2 * 4 + 8);
expect(r.uncompressed_size == 2 * 4); expect(r.block.uncompressed_size == 2 * 4);
expect(r.stream_offset == LZMA_STREAM_HEADER_SIZE + 4 + 8); expect(r.block.compressed_file_offset
expect(r.uncompressed_offset == 4); == LZMA_STREAM_HEADER_SIZE + 4 + 8);
expect(r.block.uncompressed_file_offset == 4);
// Last Record // Last Record
expect(!lzma_index_locate(i, &r, lzma_index_uncompressed_size(i) - 1)); expect(!lzma_index_iter_locate(
expect(r.total_size == 4 * 5555 + 8); &r, lzma_index_uncompressed_size(i) - 1));
expect(r.uncompressed_size == 4 * 5555); expect(r.block.total_size == 4 * 5555 + 8);
expect(r.stream_offset == lzma_index_total_size(i) expect(r.block.uncompressed_size == 4 * 5555);
expect(r.block.compressed_file_offset == lzma_index_total_size(i)
+ LZMA_STREAM_HEADER_SIZE - 4 * 5555 - 8); + LZMA_STREAM_HEADER_SIZE - 4 * 5555 - 8);
expect(r.uncompressed_offset expect(r.block.uncompressed_file_offset
== lzma_index_uncompressed_size(i) - 4 * 5555); == lzma_index_uncompressed_size(i) - 4 * 5555);
// Allocation chunk boundaries. See INDEX_GROUP_SIZE in // Allocation chunk boundaries. See INDEX_GROUP_SIZE in
@ -439,48 +540,50 @@ test_locate(void)
} }
while (n < start + 2 * radius) { while (n < start + 2 * radius) {
expect(!lzma_index_locate(i, &r, ubase + n * 4)); expect(!lzma_index_iter_locate(&r, ubase + n * 4));
expect(r.stream_offset == tbase + n * 4 + 8 expect(r.block.compressed_file_offset == tbase + n * 4 + 8
+ LZMA_STREAM_HEADER_SIZE); + LZMA_STREAM_HEADER_SIZE);
expect(r.uncompressed_offset == ubase + n * 4); expect(r.block.uncompressed_file_offset == ubase + n * 4);
tbase += n * 4 + 8; tbase += n * 4 + 8;
ubase += n * 4; ubase += n * 4;
++n; ++n;
expect(r.total_size == n * 4 + 8); expect(r.block.total_size == n * 4 + 8);
expect(r.uncompressed_size == n * 4); expect(r.block.uncompressed_size == n * 4);
} }
// Do it also backwards since lzma_index_locate() uses relative search. // Do it also backwards.
while (n > start) { while (n > start) {
expect(!lzma_index_locate(i, &r, ubase + (n - 1) * 4)); expect(!lzma_index_iter_locate(&r, ubase + (n - 1) * 4));
expect(r.total_size == n * 4 + 8); expect(r.block.total_size == n * 4 + 8);
expect(r.uncompressed_size == n * 4); expect(r.block.uncompressed_size == n * 4);
--n; --n;
tbase -= n * 4 + 8; tbase -= n * 4 + 8;
ubase -= n * 4; ubase -= n * 4;
expect(r.stream_offset == tbase + n * 4 + 8 expect(r.block.compressed_file_offset == tbase + n * 4 + 8
+ LZMA_STREAM_HEADER_SIZE); + LZMA_STREAM_HEADER_SIZE);
expect(r.uncompressed_offset == ubase + n * 4); expect(r.block.uncompressed_file_offset == ubase + n * 4);
} }
// Test locating in concatend Index. // Test locating in concatend Index.
i = lzma_index_init(i, NULL); lzma_index_end(i, NULL);
i = lzma_index_init(NULL);
expect(i != NULL); expect(i != NULL);
lzma_index_iter_init(&r, i);
for (n = 0; n < group_multiple; ++n) for (n = 0; n < group_multiple; ++n)
expect(lzma_index_append(i, NULL, 8, 0) == LZMA_OK); expect(lzma_index_append(i, NULL, 8, 0) == LZMA_OK);
expect(lzma_index_append(i, NULL, 16, 1) == LZMA_OK); expect(lzma_index_append(i, NULL, 16, 1) == LZMA_OK);
expect(!lzma_index_locate(i, &r, 0)); expect(!lzma_index_iter_locate(&r, 0));
expect(r.total_size == 16); expect(r.block.total_size == 16);
expect(r.uncompressed_size == 1); expect(r.block.uncompressed_size == 1);
expect(r.stream_offset expect(r.block.compressed_file_offset
== LZMA_STREAM_HEADER_SIZE + group_multiple * 8); == LZMA_STREAM_HEADER_SIZE + group_multiple * 8);
expect(r.uncompressed_offset == 0); expect(r.block.uncompressed_file_offset == 0);
lzma_index_end(i, NULL); lzma_index_end(i, NULL);
} }